From c74813260fc80be2ae9017f2a70b68fd2df2d7df Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 29 Sep 2025 16:34:45 -0400 Subject: [PATCH 01/60] tdd initial tests --- .../models/async/test_submission_async.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/integration/synapseclient/models/async/test_submission_async.py diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py new file mode 100644 index 000000000..2b856d4e5 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -0,0 +1,199 @@ +def test_create_submission_async(): + # WHEN an evaluation is retrieved + evaluation = Evaluation(id=evaluation_id).get() + + # AND an entity is retrieved + file = File(name="test.txt", parentId=project.id).get() + + # THEN the entity can be submitted to the evaluation + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + +def test_get_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is retrieved by ID + retrieved_submission = Submission(id=submission.id).get() + + # THEN the retrieved submission matches the created one + assert retrieved_submission.id == submission.id + assert retrieved_submission.name == submission.name + + # AND the user_id matches the current user + current_user = syn.getUserProfile()().id + assert retrieved_submission.user_id == current_user + +def test_get_evaluation_submissions_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN submissions are retrieved for the evaluation + submissions = Submission.get_evaluation_submissions(evaluation.id) + + # THEN the submissions list is not empty + assert len(submissions) > 0 + + # AND each submission belongs to the evaluation + for submission in submissions: + assert submission.evaluation_id == evaluation.id + +def test_get_user_submissions_async(): + # GIVEN a user has made submissions + current_user = syn.getUserProfile()().id + + # WHEN submissions are retrieved for the user + submissions = Submission.get_user_submissions(current_user) + + # THEN the submissions list is not empty + assert len(submissions) > 0 + + # AND each submission belongs to the user + for submission in submissions: + assert submission.user_id == current_user + +def test_get_submission_count_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission count is retrieved for the evaluation + count = Submission.get_submission_count(evaluation.id) + + # THEN the count is greater than zero + assert count > 0 + +def test_delete_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is deleted + submission.delete() + + # THEN retrieving the submission should raise an error + try: + Submission(id=submission.id).get() + assert False, "Expected an error when retrieving a deleted submission" + except SynapseError as e: + assert e.response.status_code == 404 + +def test_cancel_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is canceled + submission.cancel() + + # THEN the submission status should be 'CANCELED' + updated_submission = Submission(id=submission.id).get() + assert updated_submission.status == 'CANCELED' + +def test_get_submission_status_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission status is retrieved + status = submission.get_status() + + # THEN the status should be 'RECEIVED' + assert status == 'RECEIVED' + +def test_update_submission_status_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission status is retrieved + status = submission.get_status() + assert status != 'SCORED' + + # AND the submission status is updated to 'SCORED' + submission.update_status('SCORED') + + # THEN the submission status should be 'SCORED' + updated_submission = Submission(id=submission.id).get() + assert updated_submission.status == 'SCORED' + +def test_get_evaluation_submission_statuses_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission statuses are retrieved for the evaluation + statuses = Submission.get_evaluation_submission_statuses(evaluation.id) + + # THEN the statuses list is not empty + assert len(statuses) > 0 + +def test_batch_update_statuses_async(): + # GIVEN multiple submissions have been created + submission1 = Submission( + name="Test Submission 1", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + submission2 = Submission( + name="Test Submission 2", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the statuses of the submissions are batch updated to 'SCORED' + Submission.batch_update_statuses( + [submission1.id, submission2.id], + 'SCORED' + ) + + # THEN each submission status should be 'SCORED' + updated_submission1 = Submission(id=submission1.id).get() + updated_submission2 = Submission(id=submission2.id).get() + assert updated_submission1.status == 'SCORED' + assert updated_submission2.status == 'SCORED' + +def test_get_evaluation_submission_bundles_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission bundles are retrieved for the evaluation + bundles = Submission.get_evaluation_submission_bundles(evaluation.id) + + # THEN the bundles list is not empty + assert len(bundles) > 0 + +def test_get_user_submission_bundles_async(): + # GIVEN a user has made submissions + current_user = syn.getUserProfile()().id + + # WHEN the submission bundles are retrieved for the user + bundles = Submission.get_user_submission_bundles(current_user) + + # THEN the bundles list is not empty + assert len(bundles) > 0 From 33643ae9efa9125931dfe236dcc244fbbb4224cf Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 29 Sep 2025 16:43:57 -0400 Subject: [PATCH 02/60] initial intro of dataclasses --- synapseclient/models/submission.py | 209 +++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 synapseclient/models/submission.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py new file mode 100644 index 000000000..0af9359db --- /dev/null +++ b/synapseclient/models/submission.py @@ -0,0 +1,209 @@ +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Dict, List, Optional, Protocol, TypeVar, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.constants import concrete_types +from synapseclient.core.utils import delete_none_keys +from synapseclient.models import Activity, Annotations +from synapseclient.models.mixins.access_control import AccessControllable +from synapseclient.models.mixins.table_components import ( + DeleteMixin, + GetMixin, +) + + +class SubmissionSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for Submission operations.""" + + def get( + self, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a Submission from Synapse. + + Arguments: + include_activity: Whether to include the activity in the returned submission. + Defaults to False. Setting this to True will include the activity + record associated with this submission. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Example: Retrieving a submission by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234").get() + print(submission) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a submission. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + submission.delete() + print("Deleted Submission.") + ``` + """ + pass + + +@dataclass +@async_to_sync +class Submission( + SubmissionSynchronousProtocol, + AccessControllable, + GetMixin, + DeleteMixin, +): + """A `Submission` object represents a Synapse Submission, which is created when a user + submits an entity to an evaluation queue. + + + Attributes: + id: The unique ID of this Submission. + user_id: The ID of the user that submitted this Submission. + submitter_alias: The name of the user that submitted this Submission. + entity_id: The ID of the entity being submitted. + version_number: The version number of the entity at submission. + evaluation_id: The ID of the Evaluation to which this Submission belongs. + name: The name of this Submission. + created_on: The date this Submission was created. + team_id: The ID of the team that submitted this submission (if it's a team submission). + contributors: User IDs of team members who contributed to this submission (if it's a team submission). + submission_status: The status of this Submission. + entity_bundle_json: The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + docker_repository_name: For Docker repositories, the repository name. + docker_digest: For Docker repositories, the digest of the submitted Docker image. + activity: The Activity model represents the main record of Provenance in Synapse. + + Example: Retrieve a Submission. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn123456").get() + print(submission) + ``` + """ + + id: Optional[str] = None + """ + The unique ID of this Submission. + """ + + user_id: Optional[str] = None + """ + The ID of the user that submitted this Submission. + """ + + submitter_alias: Optional[str] = None + """ + The name of the user that submitted this Submission. + """ + + entity_id: Optional[str] = None + """ + The ID of the entity being submitted. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the entity at submission. + """ + + evaluation_id: Optional[str] = None + """ + The ID of the Evaluation to which this Submission belongs. + """ + + name: Optional[str] = None + """ + The name of this Submission. + """ + + created_on: Optional[str] = field(default=None, compare=False) + """ + The date this Submission was created. + """ + + team_id: Optional[str] = None + """ + The ID of the team that submitted this submission (if it's a team submission). + """ + + contributors: List[str] = field(default_factory=list) + """ + User IDs of team members who contributed to this submission (if it's a team submission). + """ + + submission_status: Optional[Dict] = None + """ + The status of this Submission. + """ + + entity_bundle_json: Optional[str] = None + """ + The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + """ + + docker_repository_name: Optional[str] = None + """ + For Docker repositories, the repository name. + """ + + docker_digest: Optional[str] = None + """ + For Docker repositories, the digest of the submitted Docker image. + """ + + activity: Optional[Activity] = field(default=None, compare=False) + """The Activity model represents the main record of Provenance in Synapse. It is + analogous to the Activity defined in the + [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" + + _last_persistent_instance: Optional["Submission"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" From 8a074e689ad2296b9bbcfd32a2a73de40613a654 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 30 Sep 2025 17:58:11 -0400 Subject: [PATCH 03/60] expose api services for submission object --- synapseclient/api/submission_services.py | 228 +++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 synapseclient/api/submission_services.py diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py new file mode 100644 index 000000000..226b04056 --- /dev/null +++ b/synapseclient/api/submission_services.py @@ -0,0 +1,228 @@ +# TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. + +import json +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: + from synapseclient import Synapse + + +async def create_submission(request_body: dict, synapse_client: Optional["Synapse"] = None) -> dict: + """ + Creates a Submission and sends a submission notification email to the submitter's team members. + + + + Arguments: + request_body: The request body to send to the server. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = "/evaluation/submission" + + response = await client.rest_post_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] = None) -> dict: + """ + Retrieves a Submission by its ID. + + + + Arguments: + submission_id: The ID of the submission to fetch. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + response = await client.rest_get_async(uri) + + return response + + +async def get_evaluation_submissions( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + # TODO: Support pagination in the return type. + A response JSON containing a paginated list of submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/all" + query_params = { + "limit": limit, + "offset": offset + } + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission" + query_params = { + "limit": limit, + "offset": offset + } + + if user_id: + query_params["userId"] = user_id + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/count" + query_params = {} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def delete_submission( + submission_id: str, + synapse_client: Optional["Synapse"] = None +) -> None: + """ + Deletes a Submission and its SubmissionStatus. + + + + Arguments: + submission_id: The ID of the submission to delete. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + await client.rest_delete_async(uri) + + +async def cancel_submission( + submission_id: str, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Cancels a Submission. Only the user who created the Submission may cancel it. + + + + Arguments: + submission_id: The ID of the submission to cancel. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + The canceled Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/cancellation" + + response = await client.rest_put_async(uri) + + return response \ No newline at end of file From c97b8910de64612d84f626bf55d11b7376795a33 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 30 Sep 2025 17:59:31 -0400 Subject: [PATCH 04/60] style and update docstring --- synapseclient/api/submission_services.py | 86 +++++++++---------- synapseclient/models/submission.py | 7 +- .../models/async/test_submission_async.py | 47 ++++++---- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index 226b04056..385423308 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -7,7 +7,9 @@ from synapseclient import Synapse -async def create_submission(request_body: dict, synapse_client: Optional["Synapse"] = None) -> dict: +async def create_submission( + request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: """ Creates a Submission and sends a submission notification email to the submitter's team members. @@ -23,13 +25,15 @@ async def create_submission(request_body: dict, synapse_client: Optional["Synaps client = Synapse.get_client(synapse_client=synapse_client) uri = "/evaluation/submission" - + response = await client.rest_post_async(uri, body=json.dumps(request_body)) return response -async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] = None) -> dict: +async def get_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: """ Retrieves a Submission by its ID. @@ -39,7 +43,7 @@ async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] submission_id: The ID of the submission to fetch. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: The requested Submission. """ @@ -48,18 +52,18 @@ async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}" - + response = await client.rest_get_async(uri) return response async def get_evaluation_submissions( - evaluation_id: str, + evaluation_id: str, status: Optional[str] = None, limit: int = 20, offset: int = 0, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Retrieves all Submissions for a specified Evaluation queue. @@ -68,14 +72,14 @@ async def get_evaluation_submissions( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, + status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. + offset: The offset index determines where this page will start from. An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: # TODO: Support pagination in the return type. A response JSON containing a paginated list of submissions for the evaluation queue. @@ -85,14 +89,11 @@ async def get_evaluation_submissions( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = { - "limit": limit, - "offset": offset - } - + query_params = {"limit": limit, "offset": offset} + if status: query_params["status"] = status - + response = await client.rest_get_async(uri, **query_params) return response @@ -103,7 +104,7 @@ async def get_user_submissions( user_id: Optional[str] = None, limit: int = 20, offset: int = 0, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Retrieves Submissions for a specified Evaluation queue and user. @@ -113,14 +114,14 @@ async def get_user_submissions( Arguments: evaluation_id: The ID of the evaluation queue. - user_id: Optionally specify the ID of the user whose submissions will be returned. + user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. + offset: The offset index determines where this page will start from. An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: A response JSON containing a paginated list of user submissions for the evaluation queue. """ @@ -129,14 +130,11 @@ async def get_user_submissions( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission" - query_params = { - "limit": limit, - "offset": offset - } - + query_params = {"limit": limit, "offset": offset} + if user_id: query_params["userId"] = user_id - + response = await client.rest_get_async(uri, **query_params) return response @@ -145,7 +143,7 @@ async def get_user_submissions( async def get_submission_count( evaluation_id: str, status: Optional[str] = None, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. @@ -154,11 +152,11 @@ async def get_submission_count( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, + status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: A response JSON containing the submission count. """ @@ -168,18 +166,17 @@ async def get_submission_count( uri = f"/evaluation/{evaluation_id}/submission/count" query_params = {} - + if status: query_params["status"] = status - + response = await client.rest_get_async(uri, **query_params) return response async def delete_submission( - submission_id: str, - synapse_client: Optional["Synapse"] = None + submission_id: str, synapse_client: Optional["Synapse"] = None ) -> None: """ Deletes a Submission and its SubmissionStatus. @@ -188,7 +185,7 @@ async def delete_submission( Arguments: submission_id: The ID of the submission to delete. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. """ from synapseclient import Synapse @@ -196,13 +193,12 @@ async def delete_submission( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}" - + await client.rest_delete_async(uri) async def cancel_submission( - submission_id: str, - synapse_client: Optional["Synapse"] = None + submission_id: str, synapse_client: Optional["Synapse"] = None ) -> dict: """ Cancels a Submission. Only the user who created the Submission may cancel it. @@ -211,18 +207,18 @@ async def cancel_submission( Arguments: submission_id: The ID of the submission to cancel. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: - The canceled Submission. + The Submission response object for the canceled submission as a JSON dict. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/cancellation" - + response = await client.rest_put_async(uri) - return response \ No newline at end of file + return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 0af9359db..4e65fc07b 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -11,10 +11,7 @@ from synapseclient.core.utils import delete_none_keys from synapseclient.models import Activity, Annotations from synapseclient.models.mixins.access_control import AccessControllable -from synapseclient.models.mixins.table_components import ( - DeleteMixin, - GetMixin, -) +from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin class SubmissionSynchronousProtocol(Protocol): @@ -112,7 +109,7 @@ class Submission( docker_repository_name: For Docker repositories, the repository name. docker_digest: For Docker repositories, the digest of the submitted Docker image. activity: The Activity model represents the main record of Provenance in Synapse. - + Example: Retrieve a Submission. ```python from synapseclient import Synapse diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 2b856d4e5..da5fdc46f 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -10,16 +10,17 @@ def test_create_submission_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() + def test_get_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is retrieved by ID @@ -33,6 +34,7 @@ def test_get_submission_async(): current_user = syn.getUserProfile()().id assert retrieved_submission.user_id == current_user + def test_get_evaluation_submissions_async(): # GIVEN an evaluation has submissions evaluation = Evaluation(id=evaluation_id).get() @@ -47,6 +49,7 @@ def test_get_evaluation_submissions_async(): for submission in submissions: assert submission.evaluation_id == evaluation.id + def test_get_user_submissions_async(): # GIVEN a user has made submissions current_user = syn.getUserProfile()().id @@ -61,6 +64,7 @@ def test_get_user_submissions_async(): for submission in submissions: assert submission.user_id == current_user + def test_get_submission_count_async(): # GIVEN an evaluation has submissions evaluation = Evaluation(id=evaluation_id).get() @@ -71,13 +75,14 @@ def test_get_submission_count_async(): # THEN the count is greater than zero assert count > 0 + def test_delete_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is deleted @@ -90,13 +95,14 @@ def test_delete_submission_async(): except SynapseError as e: assert e.response.status_code == 404 + def test_cancel_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is canceled @@ -104,7 +110,8 @@ def test_cancel_submission_async(): # THEN the submission status should be 'CANCELED' updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'CANCELED' + assert updated_submission.status == "CANCELED" + def test_get_submission_status_async(): # GIVEN a submission has been created @@ -112,14 +119,15 @@ def test_get_submission_status_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission status is retrieved status = submission.get_status() # THEN the status should be 'RECEIVED' - assert status == 'RECEIVED' + assert status == "RECEIVED" + def test_update_submission_status_async(): # GIVEN a submission has been created @@ -127,19 +135,20 @@ def test_update_submission_status_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission status is retrieved status = submission.get_status() - assert status != 'SCORED' + assert status != "SCORED" # AND the submission status is updated to 'SCORED' - submission.update_status('SCORED') + submission.update_status("SCORED") # THEN the submission status should be 'SCORED' updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'SCORED' + assert updated_submission.status == "SCORED" + def test_get_evaluation_submission_statuses_async(): # GIVEN an evaluation has submissions @@ -151,32 +160,31 @@ def test_get_evaluation_submission_statuses_async(): # THEN the statuses list is not empty assert len(statuses) > 0 + def test_batch_update_statuses_async(): # GIVEN multiple submissions have been created submission1 = Submission( name="Test Submission 1", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() submission2 = Submission( name="Test Submission 2", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the statuses of the submissions are batch updated to 'SCORED' - Submission.batch_update_statuses( - [submission1.id, submission2.id], - 'SCORED' - ) + Submission.batch_update_statuses([submission1.id, submission2.id], "SCORED") # THEN each submission status should be 'SCORED' updated_submission1 = Submission(id=submission1.id).get() updated_submission2 = Submission(id=submission2.id).get() - assert updated_submission1.status == 'SCORED' - assert updated_submission2.status == 'SCORED' + assert updated_submission1.status == "SCORED" + assert updated_submission2.status == "SCORED" + def test_get_evaluation_submission_bundles_async(): # GIVEN an evaluation has submissions @@ -188,6 +196,7 @@ def test_get_evaluation_submission_bundles_async(): # THEN the bundles list is not empty assert len(bundles) > 0 + def test_get_user_submission_bundles_async(): # GIVEN a user has made submissions current_user = syn.getUserProfile()().id From f79f6837f53f0c1af365ff30aaf9eb712420df02 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 10 Oct 2025 10:32:41 -0400 Subject: [PATCH 05/60] add submission and submissionstatus models --- synapseclient/models/submission_status.py | 637 ++++++++++++++++++++++ synapseclient/models/submissionstatus.py | 0 2 files changed, 637 insertions(+) create mode 100644 synapseclient/models/submission_status.py create mode 100644 synapseclient/models/submissionstatus.py diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py new file mode 100644 index 000000000..30cfc0568 --- /dev/null +++ b/synapseclient/models/submission_status.py @@ -0,0 +1,637 @@ +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Dict, List, Optional, Protocol, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.core.utils import delete_none_keys +from synapseclient.models import Annotations +from synapseclient.models.mixins.access_control import AccessControllable + + +class SubmissionStatusSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionStatus operations.""" + + def get( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Example: Retrieving a submission status by ID. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + print(status) + ``` + """ + return self + + def store( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus instance. + + Example: Update a submission status. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + status.status = "SCORED" + status = status.store() + print("Updated SubmissionStatus.") + ``` + """ + return self + + @staticmethod + def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return {} + + @staticmethod + def batch_update_submission_statuses( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + return {} + + +@dataclass +@async_to_sync +class SubmissionStatus( + SubmissionStatusSynchronousProtocol, + AccessControllable, +): + """A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + + Attributes: + id: The unique, immutable Synapse ID of the Submission. + etag: Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + modified_on: The date on which this SubmissionStatus was last modified. + status: The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + score: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + report: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + annotations: Primary container object for Annotations on a Synapse object. + submission_annotations: Annotations are additional key-value pair metadata that are associated with an object. + entity_id: The Synapse ID of the Entity in this Submission. + version_number: The version number of the Entity in this Submission. + status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + cancel_requested: Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + + Example: Retrieve and update a SubmissionStatus. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get a submission status + status = SubmissionStatus(id="syn123456").get() + + # Update the status + status.status = "SCORED" + status.submission_annotations = {"score": [85.5], "feedback": ["Good work!"]} + status = status.store() + print(status) + ``` + """ + + id: Optional[str] = None + """ + The unique, immutable Synapse ID of the Submission. + """ + + etag: Optional[str] = field(default=None, compare=False) + """ + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + """ + + modified_on: Optional[str] = field(default=None, compare=False) + """ + The date on which this SubmissionStatus was last modified. + """ + + status: Optional[str] = None + """ + The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + """ + + score: Optional[float] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + report: Optional[str] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Primary container object for Annotations on a Synapse object.""" + + submission_annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Annotations are additional key-value pair metadata that are associated with an object.""" + + entity_id: Optional[str] = None + """ + The Synapse ID of the Entity in this Submission. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the Entity in this Submission. + """ + + status_version: Optional[int] = field(default=None, compare=False) + """ + A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + """ + + can_cancel: Optional[bool] = field(default=False, compare=False) + """ + Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + """ + + cancel_requested: Optional[bool] = field(default=False, compare=False) + """ + Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + """ + + _last_persistent_instance: Optional["SubmissionStatus"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + def has_changed(self) -> bool: + """Determines if the object has been changed and needs to be updated in Synapse.""" + return ( + not self._last_persistent_instance or self._last_persistent_instance != self + ) + + def _set_last_persistent_instance(self) -> None: + """Stash the last time this object interacted with Synapse. This is used to + determine if the object has been changed and needs to be updated in Synapse.""" + import dataclasses + del self._last_persistent_instance + self._last_persistent_instance = dataclasses.replace(self) + + def fill_from_dict( + self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] + ) -> "SubmissionStatus": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_status: The response from the REST API. + + Returns: + The SubmissionStatus object. + """ + self.id = synapse_submission_status.get("id", None) + self.etag = synapse_submission_status.get("etag", None) + self.modified_on = synapse_submission_status.get("modifiedOn", None) + self.status = synapse_submission_status.get("status", None) + self.score = synapse_submission_status.get("score", None) + self.report = synapse_submission_status.get("report", None) + self.entity_id = synapse_submission_status.get("entityId", None) + self.version_number = synapse_submission_status.get("versionNumber", None) + self.status_version = synapse_submission_status.get("statusVersion", None) + self.can_cancel = synapse_submission_status.get("canCancel", False) + self.cancel_requested = synapse_submission_status.get("cancelRequested", False) + + # Handle annotations + annotations_dict = synapse_submission_status.get("annotations", {}) + if annotations_dict: + self.annotations = Annotations.from_dict(annotations_dict) + + # Handle submission annotations + submission_annotations_dict = synapse_submission_status.get("submissionAnnotations", {}) + if submission_annotations_dict: + self.submission_annotations = Annotations.from_dict(submission_annotations_dict) + + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Get: {self.id}" + ) + async def get_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Raises: + ValueError: If the submission status does not have an ID to get. + + Example: Retrieving a submission status by ID + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = await SubmissionStatus(id="syn1234").get_async() + print(status) + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to get.") + + response = await submission_services.get_submission_status( + submission_id=self.id, + synapse_client=synapse_client + ) + + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Store: {self.id if self.id else 'new_status'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object. + + Raises: + ValueError: If the submission status is missing required fields. + + Example: Update a submission status + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get existing status + status = await SubmissionStatus(id="syn1234").get_async() + + # Update fields + status.status = "SCORED" + status.submission_annotations = {"score": [85.5]} + + # Store the update + status = await status.store_async() + print(f"Updated status: {status.status}") + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to update.") + + # Prepare request body + request_body = delete_none_keys({ + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + }) + + # Add annotations if present + if self.annotations: + # Convert annotations to the format expected by the API + request_body["annotations"] = self.annotations + + # Add submission annotations if present + if self.submission_annotations: + # Convert submission annotations to the format expected by the API + request_body["submissionAnnotations"] = self.submission_annotations + + # Update the submission status using the service + response = await submission_services.update_submission_status( + submission_id=self.id, + request_body=request_body, + synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @staticmethod + async def get_all_submission_statuses_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return await submission_services.get_all_submission_statuses( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client + ) + + @staticmethod + async def batch_update_submission_statuses_async( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + # Convert SubmissionStatus objects to dictionaries + status_dicts = [] + for status in statuses: + status_dict = delete_none_keys({ + "id": status.id, + "etag": status.etag, + "status": status.status, + "score": status.score, + "report": status.report, + "entityId": status.entity_id, + "versionNumber": status.version_number, + "canCancel": status.can_cancel, + "cancelRequested": status.cancel_requested, + }) + + # Add annotations if present + if status.annotations: + status_dict["annotations"] = status.annotations + + # Add submission annotations if present + if status.submission_annotations: + status_dict["submissionAnnotations"] = status.submission_annotations + + status_dicts.append(status_dict) + + # Prepare the batch request body + request_body = { + "statuses": status_dicts, + "isFirstBatch": is_first_batch, + "isLastBatch": is_last_batch, + } + + # Add batch token if provided (required for all but first batch) + if batch_token: + request_body["batchToken"] = batch_token + + return await submission_services.batch_update_submission_statuses( + evaluation_id=evaluation_id, + request_body=request_body, + synapse_client=synapse_client + ) diff --git a/synapseclient/models/submissionstatus.py b/synapseclient/models/submissionstatus.py new file mode 100644 index 000000000..e69de29bb From 99ebaa176f3e8679336f98646e6d594cf12f43c3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 10 Oct 2025 10:43:27 -0400 Subject: [PATCH 06/60] add submission status retrieval and update methods; remove empty submissionstatus file --- synapseclient/api/submission_services.py | 258 +++++++++++++- synapseclient/models/submission.py | 397 +++++++++++++++++++++- synapseclient/models/submission_status.py | 72 ++-- synapseclient/models/submissionstatus.py | 0 4 files changed, 687 insertions(+), 40 deletions(-) delete mode 100644 synapseclient/models/submissionstatus.py diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index 385423308..feea626fc 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -1,7 +1,7 @@ # TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. import json -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from synapseclient import Synapse @@ -222,3 +222,259 @@ async def cancel_submission( response = await client.rest_put_async(uri) return response + + +async def get_submission_status( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the SubmissionStatus object associated with a specified Submission. + + + + Arguments: + submission_id: The ID of the submission to get the status for. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus object as a JSON dict. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatus. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_get_async(uri) + + return response + + +async def update_submission_status( + submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Updates a SubmissionStatus object. + + + + Arguments: + submission_id: The ID of the SubmissionStatus being updated. + request_body: The SubmissionStatus object to update as a dictionary. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object as a JSON dict. + + Note: + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Each time a SubmissionStatus is updated a new etag will be + issued to the SubmissionStatus. When an update is requested, Synapse will compare + the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. + If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED + (412) response. When this occurs, the caller should fetch the latest copy of the + SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatuses. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/status/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def batch_update_submission_statuses( + evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + request_body: The SubmissionStatusBatch object as a dictionary containing: + - statuses: List of SubmissionStatus objects to update + - isFirstBatch: Boolean indicating if this is the first batch in the series + - isLastBatch: Boolean indicating if this is the last batch in the series + - batchToken: Token from previous batch response (required for all but first batch) + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Note: + To allow upload of more than the maximum batch size (500), the system supports + uploading a series of batches. Synapse employs optimistic concurrency on the + series in the form of a batch token. Each request (except the first) must include + the 'batch token' returned in the response to the previous batch. If another client + begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be + generated and upload must restart from the first batch. + + After the final batch is uploaded, the data for the Evaluation queue will be + mirrored to the tables which support querying. Therefore uploaded data will not + appear in Evaluation queries until after the final batch is successfully uploaded. + + It is the client's responsibility to note in each batch request: + 1. Whether it is the first batch in the series (isFirstBatch) + 2. Whether it is the last batch (isLastBatch) + + For a single batch both flags are set to 'true'. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/statusBatch" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission bundles for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of the requesting user's submission bundles for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle" + query_params = {"limit": limit, "offset": offset} + + response = await client.rest_get_async(uri, **query_params) + + return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 4e65fc07b..4aa538131 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -1,15 +1,12 @@ -from collections import OrderedDict from dataclasses import dataclass, field -from datetime import date, datetime -from typing import Dict, List, Optional, Protocol, TypeVar, Union +from typing import Dict, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse -from synapseclient.core.async_utils import async_to_sync -from synapseclient.core.constants import concrete_types +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import delete_none_keys -from synapseclient.models import Activity, Annotations from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin @@ -194,7 +191,8 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ - activity: Optional[Activity] = field(default=None, compare=False) + # TODO + activity: Optional[Dict] = field(default=None, compare=False) """The Activity model represents the main record of Provenance in Synapse. It is analogous to the Activity defined in the [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" @@ -204,3 +202,388 @@ class Submission( ) """The last persistent instance of this object. This is used to determine if the object has been changed and needs to be updated in Synapse.""" + + def fill_from_dict( + self, synapse_submission: Dict[str, Union[bool, str, int, List]] + ) -> "Submission": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission: The response from the REST API. + + Returns: + The Submission object. + """ + self.id = synapse_submission.get("id", None) + self.user_id = synapse_submission.get("userId", None) + self.submitter_alias = synapse_submission.get("submitterAlias", None) + self.entity_id = synapse_submission.get("entityId", None) + self.version_number = synapse_submission.get("versionNumber", None) + self.evaluation_id = synapse_submission.get("evaluationId", None) + self.name = synapse_submission.get("name", None) + self.created_on = synapse_submission.get("createdOn", None) + self.team_id = synapse_submission.get("teamId", None) + self.contributors = synapse_submission.get("contributors", []) + self.submission_status = synapse_submission.get("submissionStatus", None) + self.entity_bundle_json = synapse_submission.get("entityBundleJSON", None) + self.docker_repository_name = synapse_submission.get( + "dockerRepositoryName", None + ) + self.docker_digest = synapse_submission.get("dockerDigest", None) + + activity_dict = synapse_submission.get("activity", None) + if activity_dict: + # TODO: Implement Activity class and its fill_from_dict method + self.activity = {} + + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Store: {self.id if self.id else 'new_submission'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Store the submission in Synapse. This creates a new submission in an evaluation queue. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission object with the ID set. + + Raises: + ValueError: If the submission is missing required fields. + + Example: Creating a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ) + submission = await submission.store_async() + print(submission.id) + ``` + """ + if not self.entity_id: + raise ValueError("The submission must have an entity_id to store.") + if not self.evaluation_id: + raise ValueError("The submission must have an evaluation_id to store.") + + # Prepare request body + request_body = delete_none_keys( + { + "entityId": self.entity_id, + "evaluationId": self.evaluation_id, + "name": self.name, + "teamId": self.team_id, + "contributors": self.contributors if self.contributors else None, + "dockerRepositoryName": self.docker_repository_name, + "dockerDigest": self.docker_digest, + } + ) + + # Create the submission using the service + response = await submission_services.create_submission( + request_body=request_body, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Get: {self.id}" + ) + async def get_async( + self, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Retrieve a Submission from Synapse. + + Arguments: + include_activity: Whether to include the activity in the returned submission. + Defaults to False. Setting this to True will include the activity + record associated with this submission. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Raises: + ValueError: If the submission does not have an ID to get. + + Example: Retrieving a submission by ID + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = await Submission(id="syn1234").get_async() + print(submission) + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to get.") + + # Get the submission using the service + response = await submission_services.get_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + + # Handle activity if requested + if include_activity and self.activity: + # The activity should be included in the response by default + # but if we need to fetch it separately, we would do it here + pass + + return self + + @staticmethod + async def get_evaluation_submissions_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of submissions for the evaluation queue. + + Example: Getting submissions for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_evaluation_submissions_async( + evaluation_id="9614543", + status="SCORED", + limit=10 + ) + print(f"Found {len(response['results'])} submissions") + ``` + """ + return await submission_services.get_evaluation_submissions( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_user_submissions_async( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + + Example: Getting user submissions + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_user_submissions_async( + evaluation_id="9614543", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + ``` + """ + return await submission_services.get_user_submissions( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_submission_count_async( + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + + Example: Getting submission count + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_submission_count_async( + evaluation_id="9614543", + status="SCORED" + ) + print(f"Found {response['count']} submissions") + ``` + """ + return await submission_services.get_submission_count( + evaluation_id=evaluation_id, status=status, synapse_client=synapse_client + ) + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Delete: {self.id}" + ) + async def delete_submission_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If the submission does not have an ID to delete. + + Example: Delete a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + await submission.delete_submission_async() + print("Deleted Submission.") + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to delete.") + + await submission_services.delete_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Cancel: {self.id}" + ) + async def cancel_submission_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Cancel a Submission. Only the user who created the Submission may cancel it. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated Submission object. + + Raises: + ValueError: If the submission does not have an ID to cancel. + + Example: Cancel a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + canceled_submission = await submission.cancel_submission_async() + print(f"Canceled submission: {canceled_submission.id}") + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to cancel.") + + response = await submission_services.cancel_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + return self diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 30cfc0568..c468f0e79 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -214,7 +214,7 @@ class SubmissionStatus( # Get a submission status status = SubmissionStatus(id="syn123456").get() - + # Update the status status.status = "SCORED" status.submission_annotations = {"score": [85.5], "feedback": ["Good work!"]} @@ -328,6 +328,7 @@ def _set_last_persistent_instance(self) -> None: """Stash the last time this object interacted with Synapse. This is used to determine if the object has been changed and needs to be updated in Synapse.""" import dataclasses + del self._last_persistent_instance self._last_persistent_instance = dataclasses.replace(self) @@ -361,9 +362,13 @@ def fill_from_dict( self.annotations = Annotations.from_dict(annotations_dict) # Handle submission annotations - submission_annotations_dict = synapse_submission_status.get("submissionAnnotations", {}) + submission_annotations_dict = synapse_submission_status.get( + "submissionAnnotations", {} + ) if submission_annotations_dict: - self.submission_annotations = Annotations.from_dict(submission_annotations_dict) + self.submission_annotations = Annotations.from_dict( + submission_annotations_dict + ) return self @@ -405,8 +410,7 @@ async def get_async( raise ValueError("The submission status must have an ID to get.") response = await submission_services.get_submission_status( - submission_id=self.id, - synapse_client=synapse_client + submission_id=self.id, synapse_client=synapse_client ) self.fill_from_dict(response) @@ -445,11 +449,11 @@ async def store_async( # Get existing status status = await SubmissionStatus(id="syn1234").get_async() - + # Update fields status.status = "SCORED" status.submission_annotations = {"score": [85.5]} - + # Store the update status = await status.store_async() print(f"Updated status: {status.status}") @@ -459,17 +463,19 @@ async def store_async( raise ValueError("The submission status must have an ID to update.") # Prepare request body - request_body = delete_none_keys({ - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - }) + request_body = delete_none_keys( + { + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + } + ) # Add annotations if present if self.annotations: @@ -485,7 +491,7 @@ async def store_async( response = await submission_services.update_submission_status( submission_id=self.id, request_body=request_body, - synapse_client=synapse_client + synapse_client=synapse_client, ) # Update this object with the response @@ -541,7 +547,7 @@ async def get_all_submission_statuses_async( status=status, limit=limit, offset=offset, - synapse_client=synapse_client + synapse_client=synapse_client, ) @staticmethod @@ -597,17 +603,19 @@ async def batch_update_submission_statuses_async( # Convert SubmissionStatus objects to dictionaries status_dicts = [] for status in statuses: - status_dict = delete_none_keys({ - "id": status.id, - "etag": status.etag, - "status": status.status, - "score": status.score, - "report": status.report, - "entityId": status.entity_id, - "versionNumber": status.version_number, - "canCancel": status.can_cancel, - "cancelRequested": status.cancel_requested, - }) + status_dict = delete_none_keys( + { + "id": status.id, + "etag": status.etag, + "status": status.status, + "score": status.score, + "report": status.report, + "entityId": status.entity_id, + "versionNumber": status.version_number, + "canCancel": status.can_cancel, + "cancelRequested": status.cancel_requested, + } + ) # Add annotations if present if status.annotations: @@ -633,5 +641,5 @@ async def batch_update_submission_statuses_async( return await submission_services.batch_update_submission_statuses( evaluation_id=evaluation_id, request_body=request_body, - synapse_client=synapse_client + synapse_client=synapse_client, ) diff --git a/synapseclient/models/submissionstatus.py b/synapseclient/models/submissionstatus.py deleted file mode 100644 index e69de29bb..000000000 From 33c651fdbe4018fe36f357c3f9b6b11a9a9a6be8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 14 Oct 2025 10:12:07 -0400 Subject: [PATCH 07/60] pipe query params directly into restAPI httpx requests --- synapseclient/api/submission_services.py | 12 ++++++------ synapseclient/models/__init__.py | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index feea626fc..cc9202954 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -94,7 +94,7 @@ async def get_evaluation_submissions( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -135,7 +135,7 @@ async def get_user_submissions( if user_id: query_params["userId"] = user_id - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -170,7 +170,7 @@ async def get_submission_count( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -339,7 +339,7 @@ async def get_all_submission_statuses( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -438,7 +438,7 @@ async def get_evaluation_submission_bundles( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -475,6 +475,6 @@ async def get_user_submission_bundles( uri = f"/evaluation/{evaluation_id}/submission/bundle" query_params = {"limit": limit, "offset": offset} - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index 9a5322727..700087ab5 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -25,6 +25,9 @@ from synapseclient.models.recordset import RecordSet from synapseclient.models.schema_organization import JSONSchema, SchemaOrganization from synapseclient.models.services import FailureStrategy +from synapseclient.models.submission import Submission +from synapseclient.models.submission_bundle import SubmissionBundle +from synapseclient.models.submission_status import SubmissionStatus from synapseclient.models.submissionview import SubmissionView from synapseclient.models.table import Table from synapseclient.models.table_components import ( @@ -128,6 +131,9 @@ "EntityRef", "DatasetCollection", # Submission models + "Submission", + "SubmissionBundle", + "SubmissionStatus", "SubmissionView", # JSON Schema models "SchemaOrganization", From ae05b916498f46977554da27ce88fc552d111058 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 14 Oct 2025 10:12:30 -0400 Subject: [PATCH 08/60] new dataclass object submission_bundle --- synapseclient/models/submission_bundle.py | 313 ++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 synapseclient/models/submission_bundle.py diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py new file mode 100644 index 000000000..7476bad18 --- /dev/null +++ b/synapseclient/models/submission_bundle.py @@ -0,0 +1,313 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Protocol, Union, TYPE_CHECKING + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.models.mixins.access_control import AccessControllable + +if TYPE_CHECKING: + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + +class SubmissionBundleSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionBundle operations.""" + + @staticmethod + def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + @staticmethod + def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + +@dataclass +@async_to_sync +class SubmissionBundle( + SubmissionBundleSynchronousProtocol, + AccessControllable, +): + """A `SubmissionBundle` object represents a bundle containing a Synapse Submission + and its accompanying SubmissionStatus. This bundle provides convenient access to both + the submission data and its current status in a single object. + + + Attributes: + submission: A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + submission_status: A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + Example: Retrieve submission bundles for an evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + # Get all submission bundles for an evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED" + ) + + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") + ``` + """ + + submission: Optional["Submission"] = None + """ + A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + """ + + submission_status: Optional["SubmissionStatus"] = None + """ + A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + """ + + _last_persistent_instance: Optional["SubmissionBundle"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + def fill_from_dict( + self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] + ) -> "SubmissionBundle": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_bundle: The response from the REST API. + + Returns: + The SubmissionBundle object. + """ + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + submission_dict = synapse_submission_bundle.get("submission", None) + if submission_dict: + self.submission = Submission().fill_from_dict(submission_dict) + else: + self.submission = None + + submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) + if submission_status_dict: + self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + else: + self.submission_status = None + + return self + + @staticmethod + async def get_evaluation_submission_bundles_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await submission_services.get_evaluation_submission_bundles( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles + + @staticmethod + async def get_user_submission_bundles_async( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await submission_services.get_user_submission_bundles( + evaluation_id=evaluation_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + # Convert response to list of SubmissionBundle objects + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles \ No newline at end of file From f51188dae7a97ad372e8e89d4771a60aaad0ad01 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 5 Nov 2025 10:08:58 -0500 Subject: [PATCH 09/60] move submission services functions to evaluation_services.py --- synapseclient/api/evaluation_services.py | 472 ++++++++++++++++++++++ synapseclient/api/submission_services.py | 480 ----------------------- 2 files changed, 472 insertions(+), 480 deletions(-) delete mode 100644 synapseclient/api/submission_services.py diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 8149618b3..ce39105d6 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -386,3 +386,475 @@ async def get_evaluation_permissions( uri = f"/evaluation/{evaluation_id}/permissions" return await client.rest_get_async(uri) + +async def create_submission( + request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Creates a Submission and sends a submission notification email to the submitter's team members. + + + + Arguments: + request_body: The request body to send to the server. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = "/evaluation/submission" + + response = await client.rest_post_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves a Submission by its ID. + + + + Arguments: + submission_id: The ID of the submission to fetch. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + response = await client.rest_get_async(uri) + + return response + + +async def get_evaluation_submissions( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + # TODO: Support pagination in the return type. + A response JSON containing a paginated list of submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission" + query_params = {"limit": limit, "offset": offset} + + if user_id: + query_params["userId"] = user_id + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/count" + query_params = {} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def delete_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> None: + """ + Deletes a Submission and its SubmissionStatus. + + + + Arguments: + submission_id: The ID of the submission to delete. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + await client.rest_delete_async(uri) + + +async def cancel_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Cancels a Submission. Only the user who created the Submission may cancel it. + + + + Arguments: + submission_id: The ID of the submission to cancel. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + The Submission response object for the canceled submission as a JSON dict. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/cancellation" + + response = await client.rest_put_async(uri) + + return response + + +async def get_submission_status( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the SubmissionStatus object associated with a specified Submission. + + + + Arguments: + submission_id: The ID of the submission to get the status for. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus object as a JSON dict. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatus. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_get_async(uri) + + return response + + +async def update_submission_status( + submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Updates a SubmissionStatus object. + + + + Arguments: + submission_id: The ID of the SubmissionStatus being updated. + request_body: The SubmissionStatus object to update as a dictionary. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object as a JSON dict. + + Note: + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Each time a SubmissionStatus is updated a new etag will be + issued to the SubmissionStatus. When an update is requested, Synapse will compare + the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. + If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED + (412) response. When this occurs, the caller should fetch the latest copy of the + SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatuses. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/status/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def batch_update_submission_statuses( + evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + request_body: The SubmissionStatusBatch object as a dictionary containing: + - statuses: List of SubmissionStatus objects to update + - isFirstBatch: Boolean indicating if this is the first batch in the series + - isLastBatch: Boolean indicating if this is the last batch in the series + - batchToken: Token from previous batch response (required for all but first batch) + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Note: + To allow upload of more than the maximum batch size (500), the system supports + uploading a series of batches. Synapse employs optimistic concurrency on the + series in the form of a batch token. Each request (except the first) must include + the 'batch token' returned in the response to the previous batch. If another client + begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be + generated and upload must restart from the first batch. + + After the final batch is uploaded, the data for the Evaluation queue will be + mirrored to the tables which support querying. Therefore uploaded data will not + appear in Evaluation queries until after the final batch is successfully uploaded. + + It is the client's responsibility to note in each batch request: + 1. Whether it is the first batch in the series (isFirstBatch) + 2. Whether it is the last batch (isLastBatch) + + For a single batch both flags are set to 'true'. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/statusBatch" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission bundles for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of the requesting user's submission bundles for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle" + query_params = {"limit": limit, "offset": offset} + + response = await client.rest_get_async(uri, params=query_params) + + return response diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py deleted file mode 100644 index cc9202954..000000000 --- a/synapseclient/api/submission_services.py +++ /dev/null @@ -1,480 +0,0 @@ -# TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. - -import json -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from synapseclient import Synapse - - -async def create_submission( - request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Creates a Submission and sends a submission notification email to the submitter's team members. - - - - Arguments: - request_body: The request body to send to the server. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = "/evaluation/submission" - - response = await client.rest_post_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Retrieves a Submission by its ID. - - - - Arguments: - submission_id: The ID of the submission to fetch. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The requested Submission. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}" - - response = await client.rest_get_async(uri) - - return response - - -async def get_evaluation_submissions( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 20, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Retrieves all Submissions for a specified Evaluation queue. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - # TODO: Support pagination in the return type. - A response JSON containing a paginated list of submissions for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_user_submissions( - evaluation_id: str, - user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Retrieves Submissions for a specified Evaluation queue and user. - If user_id is omitted, this returns the submissions of the caller. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - user_id: Optionally specify the ID of the user whose submissions will be returned. - If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission" - query_params = {"limit": limit, "offset": offset} - - if user_id: - query_params["userId"] = user_id - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_submission_count( - evaluation_id: str, - status: Optional[str] = None, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - A response JSON containing the submission count. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/count" - query_params = {} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def delete_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> None: - """ - Deletes a Submission and its SubmissionStatus. - - - - Arguments: - submission_id: The ID of the submission to delete. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}" - - await client.rest_delete_async(uri) - - -async def cancel_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Cancels a Submission. Only the user who created the Submission may cancel it. - - - - Arguments: - submission_id: The ID of the submission to cancel. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - The Submission response object for the canceled submission as a JSON dict. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/cancellation" - - response = await client.rest_put_async(uri) - - return response - - -async def get_submission_status( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Gets the SubmissionStatus object associated with a specified Submission. - - - - Arguments: - submission_id: The ID of the submission to get the status for. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The SubmissionStatus object as a JSON dict. - - Note: - The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. - Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION - to see all data marked as "private" in the SubmissionStatus. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/status" - - response = await client.rest_get_async(uri) - - return response - - -async def update_submission_status( - submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Updates a SubmissionStatus object. - - - - Arguments: - submission_id: The ID of the SubmissionStatus being updated. - request_body: The SubmissionStatus object to update as a dictionary. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The updated SubmissionStatus object as a JSON dict. - - Note: - Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle - concurrent updates. Each time a SubmissionStatus is updated a new etag will be - issued to the SubmissionStatus. When an update is requested, Synapse will compare - the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. - If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED - (412) response. When this occurs, the caller should fetch the latest copy of the - SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. - - The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/status" - - response = await client.rest_put_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_all_submission_statuses( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets a collection of SubmissionStatuses to a specified Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - status: Optionally filter submission statuses by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. - - Note: - The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. - Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION - to see all data marked as "private" in the SubmissionStatuses. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/status/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def batch_update_submission_statuses( - evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Update multiple SubmissionStatuses. The maximum batch size is 500. - - - - Arguments: - evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. - request_body: The SubmissionStatusBatch object as a dictionary containing: - - statuses: List of SubmissionStatus objects to update - - isFirstBatch: Boolean indicating if this is the first batch in the series - - isLastBatch: Boolean indicating if this is the last batch in the series - - batchToken: Token from previous batch response (required for all but first batch) - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A BatchUploadResponse object as a JSON dict containing the batch token - and other response information. - - Note: - To allow upload of more than the maximum batch size (500), the system supports - uploading a series of batches. Synapse employs optimistic concurrency on the - series in the form of a batch token. Each request (except the first) must include - the 'batch token' returned in the response to the previous batch. If another client - begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be - generated and upload must restart from the first batch. - - After the final batch is uploaded, the data for the Evaluation queue will be - mirrored to the tables which support querying. Therefore uploaded data will not - appear in Evaluation queries until after the final batch is successfully uploaded. - - It is the client's responsibility to note in each batch request: - 1. Whether it is the first batch in the series (isFirstBatch) - 2. Whether it is the last batch (isLastBatch) - - For a single batch both flags are set to 'true'. - - The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/statusBatch" - - response = await client.rest_put_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_evaluation_submission_bundles( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission bundles for the evaluation queue. - - Note: - The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/bundle/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_user_submission_bundles( - evaluation_id: str, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of the requesting user's submission bundles for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/bundle" - query_params = {"limit": limit, "offset": offset} - - response = await client.rest_get_async(uri, params=query_params) - - return response From 33f7a6a8e17749f212a0176f1a44c9cfa6f2c144 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 10:26:06 -0500 Subject: [PATCH 10/60] renaming imports, to_synapse_request, request body refactor --- synapseclient/api/__init__.py | 27 ++++++++ synapseclient/api/evaluation_services.py | 1 + synapseclient/models/submission.py | 75 +++++++++++++++-------- synapseclient/models/submission_bundle.py | 36 +++++------ synapseclient/models/submission_status.py | 10 +-- 5 files changed, 100 insertions(+), 49 deletions(-) diff --git a/synapseclient/api/__init__.py b/synapseclient/api/__init__.py index 09c578ed2..1e02bf24c 100644 --- a/synapseclient/api/__init__.py +++ b/synapseclient/api/__init__.py @@ -64,15 +64,28 @@ update_entity_acl, ) from .evaluation_services import ( + batch_update_submission_statuses, + cancel_submission, create_or_update_evaluation, + create_submission, delete_evaluation, + delete_submission, get_all_evaluations, + get_all_submission_statuses, get_available_evaluations, get_evaluation, get_evaluation_acl, get_evaluation_permissions, + get_evaluation_submission_bundles, + get_evaluation_submissions, get_evaluations_by_project, + get_submission, + get_submission_count, + get_submission_status, + get_user_submission_bundles, + get_user_submissions, update_evaluation_acl, + update_submission_status, ) from .file_services import ( AddPartResponse, @@ -282,4 +295,18 @@ "get_evaluation_acl", "update_evaluation_acl", "get_evaluation_permissions", + # submission-related evaluation services + "create_submission", + "get_submission", + "get_evaluation_submissions", + "get_user_submissions", + "get_submission_count", + "delete_submission", + "cancel_submission", + "get_submission_status", + "update_submission_status", + "get_all_submission_statuses", + "batch_update_submission_statuses", + "get_evaluation_submission_bundles", + "get_user_submission_bundles", ] diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index ce39105d6..ae06932d6 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -387,6 +387,7 @@ async def get_evaluation_permissions( return await client.rest_get_async(uri) + async def create_submission( request_body: dict, synapse_client: Optional["Synapse"] = None ) -> dict: diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 4aa538131..f8ff4b206 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -4,9 +4,8 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.api import submission_services +from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin @@ -239,6 +238,44 @@ def fill_from_dict( return self + def to_synapse_request(self) -> Dict: + """Creates a request body expected of the Synapse REST API for the Submission model. + + Returns: + A dictionary containing the request body for creating a submission. + + Raises: + ValueError: If any required attributes are missing. + """ + # These attributes are required for creating a submission + required_attributes = ["entity_id", "evaluation_id"] + + for attribute in required_attributes: + if not getattr(self, attribute): + raise ValueError( + f"Your submission object is missing the '{attribute}' attribute. This attribute is required to create a submission" + ) + + # Build a request body for creating a submission + request_body = { + "entityId": self.entity_id, + "evaluationId": self.evaluation_id, + } + + # Add optional fields if they are set + if self.name is not None: + request_body["name"] = self.name + if self.team_id is not None: + request_body["teamId"] = self.team_id + if self.contributors: + request_body["contributors"] = self.contributors + if self.docker_repository_name is not None: + request_body["dockerRepositoryName"] = self.docker_repository_name + if self.docker_digest is not None: + request_body["dockerDigest"] = self.docker_digest + + return request_body + @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Store: {self.id if self.id else 'new_submission'}" ) @@ -262,6 +299,7 @@ async def store_async( ValueError: If the submission is missing required fields. Example: Creating a submission +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -278,26 +316,11 @@ async def store_async( print(submission.id) ``` """ - if not self.entity_id: - raise ValueError("The submission must have an entity_id to store.") - if not self.evaluation_id: - raise ValueError("The submission must have an evaluation_id to store.") - - # Prepare request body - request_body = delete_none_keys( - { - "entityId": self.entity_id, - "evaluationId": self.evaluation_id, - "name": self.name, - "teamId": self.team_id, - "contributors": self.contributors if self.contributors else None, - "dockerRepositoryName": self.docker_repository_name, - "dockerDigest": self.docker_digest, - } - ) + # Create the submission using the new to_synapse_request method + request_body = self.to_synapse_request() # Create the submission using the service - response = await submission_services.create_submission( + response = await evaluation_services.create_submission( request_body=request_body, synapse_client=synapse_client ) @@ -347,7 +370,7 @@ async def get_async( raise ValueError("The submission must have an ID to get.") # Get the submission using the service - response = await submission_services.get_submission( + response = await evaluation_services.get_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -404,7 +427,7 @@ async def get_evaluation_submissions_async( print(f"Found {len(response['results'])} submissions") ``` """ - return await submission_services.get_evaluation_submissions( + return await evaluation_services.get_evaluation_submissions( evaluation_id=evaluation_id, status=status, limit=limit, @@ -455,7 +478,7 @@ async def get_user_submissions_async( print(f"Found {len(response['results'])} user submissions") ``` """ - return await submission_services.get_user_submissions( + return await evaluation_services.get_user_submissions( evaluation_id=evaluation_id, user_id=user_id, limit=limit, @@ -499,7 +522,7 @@ async def get_submission_count_async( print(f"Found {response['count']} submissions") ``` """ - return await submission_services.get_submission_count( + return await evaluation_services.get_submission_count( evaluation_id=evaluation_id, status=status, synapse_client=synapse_client ) @@ -538,7 +561,7 @@ async def delete_submission_async( if not self.id: raise ValueError("The submission must have an ID to delete.") - await submission_services.delete_submission( + await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -580,7 +603,7 @@ async def cancel_submission_async( if not self.id: raise ValueError("The submission must have an ID to cancel.") - response = await submission_services.cancel_submission( + response = await evaluation_services.cancel_submission( submission_id=self.id, synapse_client=synapse_client ) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 7476bad18..ce4c736de 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,11 +1,9 @@ from dataclasses import dataclass, field -from typing import Dict, List, Optional, Protocol, Union, TYPE_CHECKING - -from typing_extensions import Self +from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union from synapseclient import Synapse -from synapseclient.api import submission_services -from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.api import evaluation_services +from synapseclient.core.async_utils import async_to_sync from synapseclient.models.mixins.access_control import AccessControllable if TYPE_CHECKING: @@ -85,7 +83,7 @@ def get_user_submission_bundles( instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's + A list of SubmissionBundle objects containing the requesting user's submission bundles for the evaluation queue. Example: Getting user submission bundles @@ -138,7 +136,7 @@ class SubmissionBundle( evaluation_id="9614543", status="SCORED" ) - + for bundle in bundles: print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") @@ -177,16 +175,18 @@ def fill_from_dict( """ from synapseclient.models.submission import Submission from synapseclient.models.submission_status import SubmissionStatus - + submission_dict = synapse_submission_bundle.get("submission", None) if submission_dict: self.submission = Submission().fill_from_dict(submission_dict) else: self.submission = None - + submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + self.submission_status = SubmissionStatus().fill_from_dict( + submission_status_dict + ) else: self.submission_status = None @@ -240,19 +240,19 @@ async def get_evaluation_submission_bundles_async( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - response = await submission_services.get_evaluation_submission_bundles( + response = await evaluation_services.get_evaluation_submission_bundles( evaluation_id=evaluation_id, status=status, limit=limit, offset=offset, synapse_client=synapse_client, ) - + bundles = [] for bundle_dict in response.get("results", []): bundle = SubmissionBundle().fill_from_dict(bundle_dict) bundles.append(bundle) - + return bundles @staticmethod @@ -277,7 +277,7 @@ async def get_user_submission_bundles_async( instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's + A list of SubmissionBundle objects containing the requesting user's submission bundles for the evaluation queue. Example: Getting user submission bundles @@ -297,17 +297,17 @@ async def get_user_submission_bundles_async( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - response = await submission_services.get_user_submission_bundles( + response = await evaluation_services.get_user_submission_bundles( evaluation_id=evaluation_id, limit=limit, offset=offset, synapse_client=synapse_client, ) - + # Convert response to list of SubmissionBundle objects bundles = [] for bundle_dict in response.get("results", []): bundle = SubmissionBundle().fill_from_dict(bundle_dict) bundles.append(bundle) - - return bundles \ No newline at end of file + + return bundles diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index c468f0e79..cc0376b7f 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -5,7 +5,7 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.api import submission_services +from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import delete_none_keys from synapseclient.models import Annotations @@ -409,7 +409,7 @@ async def get_async( if not self.id: raise ValueError("The submission status must have an ID to get.") - response = await submission_services.get_submission_status( + response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) @@ -488,7 +488,7 @@ async def store_async( request_body["submissionAnnotations"] = self.submission_annotations # Update the submission status using the service - response = await submission_services.update_submission_status( + response = await evaluation_services.update_submission_status( submission_id=self.id, request_body=request_body, synapse_client=synapse_client, @@ -542,7 +542,7 @@ async def get_all_submission_statuses_async( print(f"Found {len(response['results'])} submission statuses") ``` """ - return await submission_services.get_all_submission_statuses( + return await evaluation_services.get_all_submission_statuses( evaluation_id=evaluation_id, status=status, limit=limit, @@ -638,7 +638,7 @@ async def batch_update_submission_statuses_async( if batch_token: request_body["batchToken"] = batch_token - return await submission_services.batch_update_submission_statuses( + return await evaluation_services.batch_update_submission_statuses( evaluation_id=evaluation_id, request_body=request_body, synapse_client=synapse_client, From 4a91a2efc12032c95a715cc506273677a407684e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 11:37:44 -0500 Subject: [PATCH 11/60] patching up store method signature --- synapseclient/api/evaluation_services.py | 10 +++- synapseclient/models/submission.py | 61 +++++++++++++++++++++--- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index ae06932d6..478976137 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -389,7 +389,7 @@ async def get_evaluation_permissions( async def create_submission( - request_body: dict, synapse_client: Optional["Synapse"] = None + request_body: dict, etag: str, synapse_client: Optional["Synapse"] = None ) -> dict: """ Creates a Submission and sends a submission notification email to the submitter's team members. @@ -398,6 +398,7 @@ async def create_submission( Arguments: request_body: The request body to send to the server. + etag: The current eTag of the Entity being submitted. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. """ @@ -407,7 +408,12 @@ async def create_submission( uri = "/evaluation/submission" - response = await client.rest_post_async(uri, body=json.dumps(request_body)) + # Add etag as query parameter if provided + params = {"etag": etag} + + response = await client.rest_post_async( + uri, body=json.dumps(request_body), params=params + ) return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index f8ff4b206..95ec2d294 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -141,7 +141,7 @@ class Submission( version_number: Optional[int] = field(default=None, compare=False) """ - The version number of the entity at submission. + The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. """ evaluation_id: Optional[str] = None @@ -196,6 +196,9 @@ class Submission( analogous to the Activity defined in the [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" + etag: Optional[str] = None + """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" + _last_persistent_instance: Optional["Submission"] = field( default=None, repr=False, compare=False ) @@ -238,6 +241,38 @@ def fill_from_dict( return self + async def _fetch_latest_entity( + self, *, synapse_client: Optional[Synapse] = None + ) -> Dict: + """ + Fetch the latest entity information from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Dictionary containing entity information from the REST API. + + Raises: + ValueError: If entity_id is not set or if unable to fetch entity information. + """ + if not self.entity_id: + raise ValueError("entity_id must be set to fetch entity information") + + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + try: + entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") + return entity_info + except Exception as e: + raise ValueError( + f"Unable to fetch entity information for {self.entity_id}: {e}" + ) + def to_synapse_request(self) -> Dict: """Creates a request body expected of the Synapse REST API for the Submission model. @@ -296,7 +331,7 @@ async def store_async( The Submission object with the ID set. Raises: - ValueError: If the submission is missing required fields. + ValueError: If the submission is missing required fields, or if unable to fetch entity etag. Example: Creating a submission   @@ -319,12 +354,23 @@ async def store_async( # Create the submission using the new to_synapse_request method request_body = self.to_synapse_request() - # Create the submission using the service + if self.entity_id: + entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + self.entity_etag = entity_info.get("etag") + self.version_number = entity_info.get("versionNumber") + + # version number is required in the request body + if self.version_number is not None: + request_body["versionNumber"] = self.version_number + else: + raise ValueError("entity_id is required to create a submission") + + if not self.entity_etag: + raise ValueError("Unable to fetch etag for entity") + response = await evaluation_services.create_submission( - request_body=request_body, synapse_client=synapse_client + request_body, self.entity_etag, synapse_client=synapse_client ) - - # Update this object with the response self.fill_from_dict(response) return self @@ -355,6 +401,7 @@ async def get_async( ValueError: If the submission does not have an ID to get. Example: Retrieving a submission by ID +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -362,7 +409,7 @@ async def get_async( syn = Synapse() syn.login() - submission = await Submission(id="syn1234").get_async() + submission = await Submission(id="9999999").get_async() print(submission) ``` """ From 65eac2257800b6665fdb29b6b58ad0dda543ebd7 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 12:14:01 -0500 Subject: [PATCH 12/60] update docs --- synapseclient/models/submission.py | 148 +++++++++++++++++------------ 1 file changed, 85 insertions(+), 63 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 95ec2d294..f4c1602ff 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -15,7 +15,6 @@ class SubmissionSynchronousProtocol(Protocol): def get( self, - include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, ) -> "Self": @@ -104,7 +103,6 @@ class Submission( file handles, and other metadata. docker_repository_name: For Docker repositories, the repository name. docker_digest: For Docker repositories, the digest of the submitted Docker image. - activity: The Activity model represents the main record of Provenance in Synapse. Example: Retrieve a Submission. ```python @@ -190,12 +188,6 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ - # TODO - activity: Optional[Dict] = field(default=None, compare=False) - """The Activity model represents the main record of Provenance in Synapse. It is - analogous to the Activity defined in the - [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" - etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" @@ -234,11 +226,6 @@ def fill_from_dict( ) self.docker_digest = synapse_submission.get("dockerDigest", None) - activity_dict = synapse_submission.get("activity", None) - if activity_dict: - # TODO: Implement Activity class and its fill_from_dict method - self.activity = {} - return self async def _fetch_latest_entity( @@ -336,19 +323,24 @@ async def store_async( Example: Creating a submission   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission( - entity_id="syn123456", - evaluation_id="9614543", - name="My Submission" - ) - submission = await submission.store_async() - print(submission.id) + async def create_submission_example(): + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ) + submission = await submission.store_async() + print(submission.id) + + asyncio.run(create_submission_example()) ``` """ # Create the submission using the new to_synapse_request method @@ -379,7 +371,6 @@ async def store_async( ) async def get_async( self, - include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, ) -> "Submission": @@ -387,9 +378,6 @@ async def get_async( Retrieve a Submission from Synapse. Arguments: - include_activity: Whether to include the activity in the returned submission. - Defaults to False. Setting this to True will include the activity - record associated with this submission. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -403,33 +391,30 @@ async def get_async( Example: Retrieving a submission by ID   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = await Submission(id="9999999").get_async() - print(submission) + async def get_submission_example(): + + submission = await Submission(id="9999999").get_async() + print(submission) + + asyncio.run(get_submission_example()) ``` """ if not self.id: raise ValueError("The submission must have an ID to get.") - # Get the submission using the service response = await evaluation_services.get_submission( submission_id=self.id, synapse_client=synapse_client ) - # Update this object with the response self.fill_from_dict(response) - # Handle activity if requested - if include_activity and self.activity: - # The activity should be included in the response by default - # but if we need to fetch it separately, we would do it here - pass - return self @staticmethod @@ -446,11 +431,11 @@ async def get_evaluation_submissions_async( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. + status: Optionally filter submissions by a submission status. + Submission status can be one of + limit: Limits the number of submissions in a single response. Defaults to 20. offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. + An index of 0 is the first submission. Defaults to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -459,19 +444,24 @@ async def get_evaluation_submissions_async( A response JSON containing a paginated list of submissions for the evaluation queue. Example: Getting submissions for an evaluation +   + Get SCORED submissions from a specific evaluation. ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_evaluation_submissions_async( - evaluation_id="9614543", - status="SCORED", - limit=10 - ) - print(f"Found {len(response['results'])} submissions") + async def get_evaluation_submissions_example(): + response = await Submission.get_evaluation_submissions_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {len(response['results'])} submissions") + + asyncio.run(get_evaluation_submissions_example()) ``` """ return await evaluation_services.get_evaluation_submissions( @@ -511,18 +501,22 @@ async def get_user_submissions_async( Example: Getting user submissions ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_user_submissions_async( - evaluation_id="9614543", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + async def get_user_submissions_example(): + response = await Submission.get_user_submissions_async( + evaluation_id="9999999", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + + asyncio.run(get_user_submissions_example()) ``` """ return await evaluation_services.get_user_submissions( @@ -555,18 +549,24 @@ async def get_submission_count_async( A response JSON containing the submission count. Example: Getting submission count +   + Get the total number of SCORED submissions from a specific evaluation. ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_submission_count_async( - evaluation_id="9614543", - status="SCORED" - ) - print(f"Found {response['count']} submissions") + async def get_submission_count_example(): + response = await Submission.get_submission_count_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {response['count']} submissions") + + asyncio.run(get_submission_count_example()) ``` """ return await evaluation_services.get_submission_count( @@ -576,7 +576,7 @@ async def get_submission_count_async( @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Delete: {self.id}" ) - async def delete_submission_async( + async def delete_async( self, *, synapse_client: Optional[Synapse] = None, @@ -593,16 +593,22 @@ async def delete_submission_async( ValueError: If the submission does not have an ID to delete. Example: Delete a submission +   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission(id="syn1234") - await submission.delete_submission_async() - print("Deleted Submission.") + async def delete_submission_example(): + submission = Submission(id="9999999") + await submission.delete_async() + print("Submission deleted successfully") + + # Run the async function + asyncio.run(delete_submission_example()) ``` """ if not self.id: @@ -611,11 +617,16 @@ async def delete_submission_async( await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) + + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been deleted.") @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Cancel: {self.id}" ) - async def cancel_submission_async( + async def cancel_async( self, *, synapse_client: Optional[Synapse] = None, @@ -635,16 +646,22 @@ async def cancel_submission_async( ValueError: If the submission does not have an ID to cancel. Example: Cancel a submission +   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission(id="syn1234") - canceled_submission = await submission.cancel_submission_async() - print(f"Canceled submission: {canceled_submission.id}") + async def cancel_submission_example(): + submission = Submission(id="syn1234") + canceled_submission = await submission.cancel_async() + print(f"Canceled submission: {canceled_submission.id}") + + # Run the async function + asyncio.run(cancel_submission_example()) ``` """ if not self.id: @@ -654,6 +671,11 @@ async def cancel_submission_async( submission_id=self.id, synapse_client=synapse_client ) + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been cancelled.") + # Update this object with the response self.fill_from_dict(response) return self From a5ec33a2b12f9333633a3c9cdb7bc67fe595a972 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 16:55:51 -0500 Subject: [PATCH 13/60] new suite of tests --- synapseclient/models/submission.py | 52 +- .../models/async/test_submission_async.py | 834 +++++++++++++----- .../models/synchronous/test_submission.py | 610 +++++++++++++ .../async/unit_test_submission_async.py | 604 +++++++++++++ .../synchronous/unit_test_submission.py | 805 +++++++++++++++++ 5 files changed, 2680 insertions(+), 225 deletions(-) create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission.py create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_async.py create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index f4c1602ff..b2e0e34f1 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -7,7 +7,6 @@ from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.models.mixins.access_control import AccessControllable -from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin class SubmissionSynchronousProtocol(Protocol): @@ -80,8 +79,6 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: class Submission( SubmissionSynchronousProtocol, AccessControllable, - GetMixin, - DeleteMixin, ): """A `Submission` object represents a Synapse Submission, which is created when a user submits an entity to an evaluation queue. @@ -191,12 +188,6 @@ class Submission( etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" - _last_persistent_instance: Optional["Submission"] = field( - default=None, repr=False, compare=False - ) - """The last persistent instance of this object. This is used to determine if the - object has been changed and needs to be updated in Synapse.""" - def fill_from_dict( self, synapse_submission: Dict[str, Union[bool, str, int, List]] ) -> "Submission": @@ -254,6 +245,25 @@ async def _fetch_latest_entity( try: entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") + + # If this is a DockerRepository, fetch docker image tag & digest, and add it to the entity_info dict + if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": + docker_tag_response = await client.rest_get_async(f"/entity/{self.entity_id}/dockerTag") + + # Get the latest digest from the docker tag results + if "results" in docker_tag_response and docker_tag_response["results"]: + # Sort by createdOn timestamp to get the latest entry + # Convert ISO timestamp strings to datetime objects for comparison + from datetime import datetime + + latest_result = max( + docker_tag_response["results"], + key=lambda x: datetime.fromisoformat(x["createdOn"].replace("Z", "+00:00")) + ) + + # Add the latest result to entity_info + entity_info.update(latest_result) + return entity_info except Exception as e: raise ValueError( @@ -282,6 +292,7 @@ def to_synapse_request(self) -> Dict: request_body = { "entityId": self.entity_id, "evaluationId": self.evaluation_id, + "versionNumber": self.version_number } # Add optional fields if they are set @@ -343,23 +354,27 @@ async def create_submission_example(): asyncio.run(create_submission_example()) ``` """ - # Create the submission using the new to_synapse_request method - request_body = self.to_synapse_request() if self.entity_id: entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + self.entity_etag = entity_info.get("etag") - self.version_number = entity_info.get("versionNumber") - # version number is required in the request body - if self.version_number is not None: - request_body["versionNumber"] = self.version_number + if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.FileEntity": + self.version_number = entity_info.get("versionNumber") + elif entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": + self.version_number = 1 # TODO: Docker repositories do not have version numbers + self.docker_repository_name = entity_info.get("repositoryName") + self.docker_digest = entity_info.get("digest") else: raise ValueError("entity_id is required to create a submission") if not self.entity_etag: raise ValueError("Unable to fetch etag for entity") + # Build the request body now that all the necessary dataclass attributes are set + request_body = self.to_synapse_request() + response = await evaluation_services.create_submission( request_body, self.entity_etag, synapse_client=synapse_client ) @@ -417,6 +432,7 @@ async def get_submission_example(): return self + # TODO: Have all staticmethods return generators for pagination @staticmethod async def get_evaluation_submissions_async( evaluation_id: str, @@ -564,7 +580,7 @@ async def get_submission_count_example(): evaluation_id="9999999", status="SCORED" ) - print(f"Found {response['count']} submissions") + print(f"Found {response} submissions") asyncio.run(get_submission_count_example()) ``` @@ -617,8 +633,9 @@ async def delete_submission_example(): await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) - + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger logger.info(f"Submission {self.id} has successfully been deleted.") @@ -672,6 +689,7 @@ async def cancel_submission_example(): ) from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger logger.info(f"Submission {self.id} has successfully been cancelled.") diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index da5fdc46f..e5fb73631 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -1,208 +1,626 @@ -def test_create_submission_async(): - # WHEN an evaluation is retrieved - evaluation = Evaluation(id=evaluation_id).get() - - # AND an entity is retrieved - file = File(name="test.txt", parentId=project.id).get() - - # THEN the entity can be submitted to the evaluation - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - -def test_get_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is retrieved by ID - retrieved_submission = Submission(id=submission.id).get() - - # THEN the retrieved submission matches the created one - assert retrieved_submission.id == submission.id - assert retrieved_submission.name == submission.name - - # AND the user_id matches the current user - current_user = syn.getUserProfile()().id - assert retrieved_submission.user_id == current_user - - -def test_get_evaluation_submissions_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN submissions are retrieved for the evaluation - submissions = Submission.get_evaluation_submissions(evaluation.id) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the evaluation - for submission in submissions: - assert submission.evaluation_id == evaluation.id - - -def test_get_user_submissions_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN submissions are retrieved for the user - submissions = Submission.get_user_submissions(current_user) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the user - for submission in submissions: - assert submission.user_id == current_user - - -def test_get_submission_count_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission count is retrieved for the evaluation - count = Submission.get_submission_count(evaluation.id) - - # THEN the count is greater than zero - assert count > 0 - - -def test_delete_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is deleted - submission.delete() - - # THEN retrieving the submission should raise an error - try: - Submission(id=submission.id).get() - assert False, "Expected an error when retrieving a deleted submission" - except SynapseError as e: - assert e.response.status_code == 404 - - -def test_cancel_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is canceled - submission.cancel() - - # THEN the submission status should be 'CANCELED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == "CANCELED" - - -def test_get_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - - # THEN the status should be 'RECEIVED' - assert status == "RECEIVED" - - -def test_update_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - assert status != "SCORED" - - # AND the submission status is updated to 'SCORED' - submission.update_status("SCORED") - - # THEN the submission status should be 'SCORED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == "SCORED" - - -def test_get_evaluation_submission_statuses_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission statuses are retrieved for the evaluation - statuses = Submission.get_evaluation_submission_statuses(evaluation.id) - - # THEN the statuses list is not empty - assert len(statuses) > 0 - - -def test_batch_update_statuses_async(): - # GIVEN multiple submissions have been created - submission1 = Submission( - name="Test Submission 1", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - submission2 = Submission( - name="Test Submission 2", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the statuses of the submissions are batch updated to 'SCORED' - Submission.batch_update_statuses([submission1.id, submission2.id], "SCORED") - - # THEN each submission status should be 'SCORED' - updated_submission1 = Submission(id=submission1.id).get() - updated_submission2 = Submission(id=submission2.id).get() - assert updated_submission1.status == "SCORED" - assert updated_submission2.status == "SCORED" - - -def test_get_evaluation_submission_bundles_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission bundles are retrieved for the evaluation - bundles = Submission.get_evaluation_submission_bundles(evaluation.id) - - # THEN the bundles list is not empty - assert len(bundles) > 0 - - -def test_get_user_submission_bundles_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN the submission bundles are retrieved for the user - bundles = Submission.get_user_submission_bundles(current_user) - - # THEN the bundles list is not empty - assert len(bundles) > 0 +"""Async integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest +import pytest_asyncio + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data using async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id_async(self, test_evaluation: Evaluation): + # WHEN I try to create a submission without entity_id using async method + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id is required to create a submission"): + await submission.store_async(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id_async(self, test_file: File): + # WHEN I try to create a submission without evaluation_id using async method + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + await submission.store_async(synapse_client=self.syn) + + # async def test_store_submission_with_docker_repository_async( + # self, test_evaluation: Evaluation + # ): + # # GIVEN we would need a Docker repository entity (mocked for this test) + # # This test demonstrates the expected behavior for Docker repository submissions + + # # WHEN I create a submission for a Docker repository entity using async method + # # TODO: This would require a real Docker repository entity in a full integration test + # submission = Submission( + # entity_id="syn123456789", # Would be a Docker repository ID + # evaluation_id=test_evaluation.id, + # name=f"Docker Submission {uuid.uuid4()}", + # ) + + # # THEN the submission should handle Docker-specific attributes + # # (This test would need to be expanded with actual Docker repository setup) + # assert submission.entity_id == "syn123456789" + # assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrievalAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest_asyncio.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id_async( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID using async method + retrieved_submission = await Submission(id=test_submission.id).get_async( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination_async( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions_async(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user using async method + response = await Submission.get_user_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count_async(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation using async method + response = await Submission.get_submission_count_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + + +class TestSubmissionDeletionAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission created with async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + + # WHEN I delete the submission using async method + await created_submission.delete_async(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + await Submission(id=created_submission.id).get_async(synapse_client=self.syn) + + async def test_delete_submission_without_id_async(self): + # WHEN I try to delete a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + +class TestSubmissionCancelAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # async def test_cancel_submission_successfully_async( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission created with async method + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = await submission.store_async(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission using async method + # cancelled_submission = await created_submission.cancel_async(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id_async(self): + # WHEN I try to cancel a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + +class TestSubmissionValidationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id_async(self): + # WHEN I try to get a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id_async(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id_async(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data_async(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data_async(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + async def test_fetch_latest_entity_success_async(self): + # GIVEN a submission with a valid entity_id + submission = Submission(entity_id="syn123456", evaluation_id="789") + + # Note: This test would need a real entity ID to work in practice + # For now, we test the validation logic + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + async def test_fetch_latest_entity_without_entity_id_async(self): + # GIVEN a submission without entity_id + submission = Submission(evaluation_id="789") + + # WHEN I try to fetch entity information + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id must be set"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + +class TestSubmissionDataMappingAsync: + async def test_fill_from_dict_complete_data_async(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data_async(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py new file mode 100644 index 000000000..c595cc509 --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -0,0 +1,610 @@ +"""Integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id(self, test_evaluation: Evaluation): + # WHEN I try to create a submission without entity_id + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id is required"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id(self, test_file: File): + # WHEN I try to create a submission without evaluation_id + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_with_docker_repository( + self, test_evaluation: Evaluation + ): + # GIVEN we would need a Docker repository entity (mocked for this test) + # This test demonstrates the expected behavior for Docker repository submissions + + # WHEN I create a submission for a Docker repository entity + # TODO: This would require a real Docker repository entity in a full integration test + submission = Submission( + entity_id="syn123456789", # Would be a Docker repository ID + evaluation_id=test_evaluation.id, + name=f"Docker Submission {uuid.uuid4()}", + ) + + # THEN the submission should handle Docker-specific attributes + # (This test would need to be expanded with actual Docker repository setup) + assert submission.entity_id == "syn123456789" + assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrieval: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID + retrieved_submission = Submission(id=test_submission.id).get( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user + response = Submission.get_user_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation + response = Submission.get_submission_count( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + assert response >= 0 + + +class TestSubmissionDeletion: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + + # WHEN I delete the submission + created_submission.delete(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + Submission(id=created_submission.id).get(synapse_client=self.syn) + + async def test_delete_submission_without_id(self): + # WHEN I try to delete a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + submission.delete(synapse_client=self.syn) + + +class TestSubmissionCancel: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # TODO: Add with SubmissionStatus model tests + # async def test_cancel_submission_successfully( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = submission.store(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission + # cancelled_submission = created_submission.cancel(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id(self): + # WHEN I try to cancel a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + submission.cancel(synapse_client=self.syn) + + +class TestSubmissionValidation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id(self): + # WHEN I try to get a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + submission.get(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="Your submission object is missing the 'entity_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + +class TestSubmissionDataMapping: + async def test_fill_from_dict_complete_data(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py new file mode 100644 index 000000000..d02c0e713 --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -0,0 +1,604 @@ +"""Async unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmissionAsync: + """Async tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + } + ] + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z" + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z" + } + ] + } + + def test_fill_from_dict_complete_data_async(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_complete_data_async(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success_async(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag") + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection_async(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update({ + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + }) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert stored_submission.version_number == 1 # Docker repos get version 1 + assert stored_submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_to_synapse_request_minimal_data_async(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py new file mode 100644 index 000000000..77820d3a6 --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -0,0 +1,805 @@ +"""Unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmission: + """Tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + } + ] + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z" + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z" + } + ] + } + + def test_fill_from_dict_complete_data(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_fill_from_dict_minimal_data(self) -> None: + # GIVEN a minimal submission response from the REST API + # WHEN I call fill_from_dict with the minimal submission response + submission = Submission().fill_from_dict(self.get_minimal_submission_response()) + + # THEN the Submission object should be filled with required data and defaults for optional data + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + + def test_to_synapse_request_complete_data(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + def test_to_synapse_request_minimal_data(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + def test_to_synapse_request_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + def test_to_synapse_request_missing_evaluation_id(self) -> None: + # GIVEN a submission without evaluation_id + submission = Submission(entity_id=ENTITY_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag") + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_fetch_latest_entity_without_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id must be set to fetch entity information"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_api_error(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity and the API returns an error + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + side_effect=SynapseHTTPError("Entity not found"), + ) as mock_rest_get: + # THEN it should raise a ValueError with context about the original error + with pytest.raises(ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Check the call arguments to create_submission + call_args = mock_create_submission.call_args + request_body = call_args[0][0] + etag = call_args[0][1] + + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["name"] == SUBMISSION_NAME + assert request_body["versionNumber"] == VERSION_NUMBER + assert etag == ETAG + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update({ + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + }) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert submission.version_number == 1 # Docker repos get version 1 + assert submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID, name=SUBMISSION_NAME) + + # WHEN I call store_async + # THEN it should raise a ValueError during to_synapse_request + with pytest.raises(ValueError, match="entity_id is required to create a submission"): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_entity_fetch_failure(self) -> None: + # GIVEN a submission with valid data but entity fetch fails + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async and entity fetching fails + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + side_effect=ValueError("Unable to fetch entity information"), + ) as mock_fetch_entity: + # THEN it should propagate the ValueError + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call get_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_delete_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call delete_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_cancel_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call cancel_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_default_values(self) -> None: + # GIVEN a new Submission object with no parameters + submission = Submission() + + # THEN all attributes should have their default values + assert submission.id is None + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.entity_id is None + assert submission.version_number is None + assert submission.evaluation_id is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + assert submission.etag is None + + def test_constructor_with_values(self) -> None: + # GIVEN specific values for submission attributes + # WHEN I create a Submission object with those values + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + ) + + # THEN the object should be initialized with those values + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_with_none_values(self) -> None: + # GIVEN a submission with some None values for optional fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=None, # Explicitly None + team_id=None, # Explicitly None + contributors=[], # Empty list (falsy) + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN None and empty values should not be included + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body From a160585779d5924679a71288d39a1b688f143fa4 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 18:07:19 -0500 Subject: [PATCH 14/60] submissionstatus rework as a mutable object --- synapseclient/models/submission.py | 39 ++++-- synapseclient/models/submission_status.py | 125 ++++++++++-------- .../models/async/test_submission_async.py | 86 +++++++----- .../models/synchronous/test_submission.py | 100 ++++++++------ .../async/unit_test_submission_async.py | 73 +++++----- .../synchronous/unit_test_submission.py | 89 +++++++------ 6 files changed, 293 insertions(+), 219 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index b2e0e34f1..5d7390050 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -245,25 +245,32 @@ async def _fetch_latest_entity( try: entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") - + # If this is a DockerRepository, fetch docker image tag & digest, and add it to the entity_info dict - if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": - docker_tag_response = await client.rest_get_async(f"/entity/{self.entity_id}/dockerTag") - + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + docker_tag_response = await client.rest_get_async( + f"/entity/{self.entity_id}/dockerTag" + ) + # Get the latest digest from the docker tag results if "results" in docker_tag_response and docker_tag_response["results"]: # Sort by createdOn timestamp to get the latest entry # Convert ISO timestamp strings to datetime objects for comparison from datetime import datetime - + latest_result = max( docker_tag_response["results"], - key=lambda x: datetime.fromisoformat(x["createdOn"].replace("Z", "+00:00")) + key=lambda x: datetime.fromisoformat( + x["createdOn"].replace("Z", "+00:00") + ), ) - + # Add the latest result to entity_info entity_info.update(latest_result) - + return entity_info except Exception as e: raise ValueError( @@ -292,7 +299,7 @@ def to_synapse_request(self) -> Dict: request_body = { "entityId": self.entity_id, "evaluationId": self.evaluation_id, - "versionNumber": self.version_number + "versionNumber": self.version_number, } # Add optional fields if they are set @@ -360,10 +367,18 @@ async def create_submission_example(): self.entity_etag = entity_info.get("etag") - if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.FileEntity": + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.FileEntity" + ): self.version_number = entity_info.get("versionNumber") - elif entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": - self.version_number = 1 # TODO: Docker repositories do not have version numbers + elif ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + self.version_number = ( + 1 # TODO: Docker repositories do not have version numbers + ) self.docker_repository_name = entity_info.get("repositoryName") self.docker_digest = entity_info.get("digest") else: diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index cc0376b7f..740351504 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,13 +1,14 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse +from synapseclient.annotations import to_submission_status_annotations from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys +from synapseclient.core.utils import delete_none_keys, merge_dataclass_entities from synapseclient.models import Annotations from synapseclient.models.mixins.access_control import AccessControllable @@ -318,8 +319,9 @@ class SubmissionStatus( """The last persistent instance of this object. This is used to determine if the object has been changed and needs to be updated in Synapse.""" + @property def has_changed(self) -> bool: - """Determines if the object has been changed and needs to be updated in Synapse.""" + """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( not self._last_persistent_instance or self._last_persistent_instance != self ) @@ -327,10 +329,7 @@ def has_changed(self) -> bool: def _set_last_persistent_instance(self) -> None: """Stash the last time this object interacted with Synapse. This is used to determine if the object has been changed and needs to be updated in Synapse.""" - import dataclasses - - del self._last_persistent_instance - self._last_persistent_instance = dataclasses.replace(self) + self._last_persistent_instance = replace(self) def fill_from_dict( self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] @@ -372,6 +371,44 @@ def fill_from_dict( return self + def to_synapse_request(self) -> Dict: + """ + Creates a request body expected by the Synapse REST API for the SubmissionStatus model. + + Returns: + A dictionary containing the request body for updating a submission status. + """ + # Prepare request body with basic fields + request_body = delete_none_keys( + { + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + } + ) + + # Add annotations if present + if self.annotations and len(self.annotations) > 0: + # Convert annotations to the format expected by the API + request_body["annotations"] = to_submission_status_annotations( + self.annotations + ) + + # Add submission annotations if present + if self.submission_annotations and len(self.submission_annotations) > 0: + # Convert submission annotations to the format expected by the API + request_body["submissionAnnotations"] = to_submission_status_annotations( + self.submission_annotations + ) + + return request_body + @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Get: {self.id}" ) @@ -462,39 +499,40 @@ async def store_async( if not self.id: raise ValueError("The submission status must have an ID to update.") - # Prepare request body - request_body = delete_none_keys( - { - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - } - ) - - # Add annotations if present - if self.annotations: - # Convert annotations to the format expected by the API - request_body["annotations"] = self.annotations + # Get the client for logging + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + + # Check if there are changes to apply + if self._last_persistent_instance and self.has_changed: + # Merge with the last persistent instance to preserve system-managed fields + merge_dataclass_entities( + source=self._last_persistent_instance, + destination=self, + fields_to_preserve_from_source=[ + "id", + "etag", + "modified_on", + "entity_id", + "version_number", + "status_version", + ], + logger=logger, + ) + elif self._last_persistent_instance and not self.has_changed: + logger.warning( + f"SubmissionStatus (ID: {self.id}) has not changed since last 'store' or 'get' event, so it will not be updated in Synapse. Please get the submission status again if you want to refresh its state." + ) + return self - # Add submission annotations if present - if self.submission_annotations: - # Convert submission annotations to the format expected by the API - request_body["submissionAnnotations"] = self.submission_annotations + request_body = self.to_synapse_request() - # Update the submission status using the service response = await evaluation_services.update_submission_status( submission_id=self.id, request_body=request_body, synapse_client=synapse_client, ) - # Update this object with the response self.fill_from_dict(response) self._set_last_persistent_instance() return self @@ -603,28 +641,7 @@ async def batch_update_submission_statuses_async( # Convert SubmissionStatus objects to dictionaries status_dicts = [] for status in statuses: - status_dict = delete_none_keys( - { - "id": status.id, - "etag": status.etag, - "status": status.status, - "score": status.score, - "report": status.report, - "entityId": status.entity_id, - "versionNumber": status.version_number, - "canCancel": status.can_cancel, - "cancelRequested": status.cancel_requested, - } - ) - - # Add annotations if present - if status.annotations: - status_dict["annotations"] = status.annotations - - # Add submission annotations if present - if status.submission_annotations: - status_dict["submissionAnnotations"] = status.submission_annotations - + status_dict = status.to_synapse_request() status_dicts.append(status_dict) # Prepare the batch request body diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index e5fb73631..745a60960 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -49,22 +49,27 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - + import tempfile + # Create a temporary file - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -93,7 +98,9 @@ async def test_store_submission_successfully_async( assert created_submission.created_on is not None assert created_submission.version_number is not None - async def test_store_submission_without_entity_id_async(self, test_evaluation: Evaluation): + async def test_store_submission_without_entity_id_async( + self, test_evaluation: Evaluation + ): # WHEN I try to create a submission without entity_id using async method submission = Submission( evaluation_id=test_evaluation.id, @@ -101,7 +108,9 @@ async def test_store_submission_without_entity_id_async(self, test_evaluation: E ) # THEN it should raise a ValueError - with pytest.raises(ValueError, match="entity_id is required to create a submission"): + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): await submission.store_async(synapse_client=self.syn) async def test_store_submission_without_evaluation_id_async(self, test_file: File): @@ -120,7 +129,7 @@ async def test_store_submission_without_evaluation_id_async(self, test_file: Fil # ): # # GIVEN we would need a Docker repository entity (mocked for this test) # # This test demonstrates the expected behavior for Docker repository submissions - + # # WHEN I create a submission for a Docker repository entity using async method # # TODO: This would require a real Docker repository entity in a full integration test # submission = Submission( @@ -128,7 +137,7 @@ async def test_store_submission_without_evaluation_id_async(self, test_file: Fil # evaluation_id=test_evaluation.id, # name=f"Docker Submission {uuid.uuid4()}", # ) - + # # THEN the submission should handle Docker-specific attributes # # (This test would need to be expanded with actual Docker repository setup) # assert submission.entity_id == "syn123456789" @@ -173,21 +182,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -239,7 +253,7 @@ async def test_get_evaluation_submissions_async( # THEN I should get a response with submissions assert "results" in response assert len(response["results"]) > 0 - + # AND the submission should be in the results submission_ids = [sub.get("id") for sub in response["results"]] assert test_submission.id in submission_ids @@ -336,21 +350,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -373,7 +392,9 @@ async def test_delete_submission_successfully_async( # THEN attempting to retrieve it should raise an error with pytest.raises(SynapseHTTPError): - await Submission(id=created_submission.id).get_async(synapse_client=self.syn) + await Submission(id=created_submission.id).get_async( + synapse_client=self.syn + ) async def test_delete_submission_without_id_async(self): # WHEN I try to delete a submission without an ID using async method @@ -422,21 +443,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index c595cc509..e6943a006 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -21,9 +21,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -48,22 +46,27 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - + import tempfile + # Create a temporary file - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -92,7 +95,9 @@ async def test_store_submission_successfully( assert created_submission.created_on is not None assert created_submission.version_number is not None - async def test_store_submission_without_entity_id(self, test_evaluation: Evaluation): + async def test_store_submission_without_entity_id( + self, test_evaluation: Evaluation + ): # WHEN I try to create a submission without entity_id submission = Submission( evaluation_id=test_evaluation.id, @@ -119,7 +124,7 @@ async def test_store_submission_with_docker_repository( ): # GIVEN we would need a Docker repository entity (mocked for this test) # This test demonstrates the expected behavior for Docker repository submissions - + # WHEN I create a submission for a Docker repository entity # TODO: This would require a real Docker repository entity in a full integration test submission = Submission( @@ -127,7 +132,7 @@ async def test_store_submission_with_docker_repository( evaluation_id=test_evaluation.id, name=f"Docker Submission {uuid.uuid4()}", ) - + # THEN the submission should handle Docker-specific attributes # (This test would need to be expanded with actual Docker repository setup) assert submission.entity_id == "syn123456789" @@ -145,9 +150,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -172,21 +175,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -238,7 +246,7 @@ async def test_get_evaluation_submissions( # THEN I should get a response with submissions assert "results" in response assert len(response["results"]) > 0 - + # AND the submission should be in the results submission_ids = [sub.get("id") for sub in response["results"]] assert test_submission.id in submission_ids @@ -309,9 +317,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -336,21 +342,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -395,9 +406,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -422,21 +431,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -490,7 +504,10 @@ async def test_to_synapse_request_missing_entity_id(self): submission = Submission(evaluation_id="456", name="Test") # THEN it should raise a ValueError - with pytest.raises(ValueError, match="Your submission object is missing the 'entity_id' attribute"): + with pytest.raises( + ValueError, + match="Your submission object is missing the 'entity_id' attribute", + ): submission.to_synapse_request() async def test_to_synapse_request_missing_evaluation_id(self): @@ -607,4 +624,3 @@ async def test_fill_from_dict_minimal_data(self): assert submission.entity_bundle_json is None assert submission.docker_repository_name is None assert submission.docker_digest is None - diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index d02c0e713..8a0ca25ac 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -88,14 +88,14 @@ def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:older123def456", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - } - ] + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], } def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: @@ -106,24 +106,24 @@ def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:version1", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v3.0", + "tag": "v3.0", "digest": "sha256:version3", - "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:version2", - "createdOn": "2024-06-01T15:30:00.000Z" + "createdOn": "2024-06-01T15:30:00.000Z", }, { "tag": "v1.5", "digest": "sha256:version1_5", - "createdOn": "2024-03-15T08:45:00.000Z" - } - ] + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], } def test_fill_from_dict_complete_data_async(self) -> None: @@ -210,9 +210,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_example_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information with latest docker tag info @@ -223,11 +223,11 @@ def side_effect(url): assert entity_info["tag"] == "v2.0" assert entity_info["digest"] == "sha256:latest456abc789" assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" - + # Verify both API calls were made expected_calls = [ call(f"/entity/{ENTITY_ID}"), - call(f"/entity/{ENTITY_ID}/dockerTag") + call(f"/entity/{ENTITY_ID}/dockerTag"), ] mock_rest_get.assert_has_calls(expected_calls) @@ -248,9 +248,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return {"totalNumberOfResults": 0, "results": []} - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information without docker tag info @@ -279,9 +279,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_complex_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should select the tag with the latest createdOn timestamp (v3.0) @@ -309,13 +309,12 @@ async def test_store_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should fetch entity information, create the submission, and fill the object mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify the submission is filled with response data assert stored_submission.id == SUBMISSION_ID assert stored_submission.entity_id == ENTITY_ID @@ -332,12 +331,14 @@ async def test_store_async_docker_repository_success(self) -> None: # WHEN I call store_async with mocked Docker repository entity docker_entity_with_tag = self.get_example_docker_entity_response() - docker_entity_with_tag.update({ - "tag": "v2.0", - "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - }) - + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + with patch.object( submission, "_fetch_latest_entity", @@ -348,13 +349,12 @@ async def test_store_async_docker_repository_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should handle Docker repository specific logic mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify Docker repository attributes are set correctly assert stored_submission.version_number == 1 # Docker repos get version 1 assert stored_submission.docker_repository_name == "test/repository" @@ -382,13 +382,12 @@ async def test_store_async_with_team_data_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should preserve team information in the stored submission mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify team data is preserved assert stored_submission.team_id == TEAM_ID assert stored_submission.contributors == CONTRIBUTORS @@ -407,7 +406,6 @@ async def test_get_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_get_submission: - retrieved_submission = await submission.get_async(synapse_client=self.syn) # THEN it should call the API and fill the object @@ -463,7 +461,9 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( @@ -496,7 +496,6 @@ async def test_get_evaluation_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, @@ -534,7 +533,6 @@ async def test_get_user_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, @@ -567,7 +565,6 @@ async def test_get_submission_count_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_count: - response = await Submission.get_submission_count_async( evaluation_id=evaluation_id, status=status, diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 77820d3a6..038045f11 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -88,14 +88,14 @@ def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:older123def456", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - } - ] + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], } def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: @@ -106,24 +106,24 @@ def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:version1", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v3.0", + "tag": "v3.0", "digest": "sha256:version3", - "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:version2", - "createdOn": "2024-06-01T15:30:00.000Z" + "createdOn": "2024-06-01T15:30:00.000Z", }, { "tag": "v1.5", "digest": "sha256:version1_5", - "createdOn": "2024-03-15T08:45:00.000Z" - } - ] + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], } def test_fill_from_dict_complete_data(self) -> None: @@ -270,9 +270,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_example_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information with latest docker tag info @@ -283,11 +283,11 @@ def side_effect(url): assert entity_info["tag"] == "v2.0" assert entity_info["digest"] == "sha256:latest456abc789" assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" - + # Verify both API calls were made expected_calls = [ call(f"/entity/{ENTITY_ID}"), - call(f"/entity/{ENTITY_ID}/dockerTag") + call(f"/entity/{ENTITY_ID}/dockerTag"), ] mock_rest_get.assert_has_calls(expected_calls) @@ -308,9 +308,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return {"totalNumberOfResults": 0, "results": []} - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information without docker tag info @@ -339,9 +339,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_complex_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should select the tag with the latest createdOn timestamp (v3.0) @@ -356,7 +356,9 @@ async def test_fetch_latest_entity_without_entity_id(self) -> None: # WHEN I call _fetch_latest_entity # THEN it should raise a ValueError - with pytest.raises(ValueError, match="entity_id must be set to fetch entity information"): + with pytest.raises( + ValueError, match="entity_id must be set to fetch entity information" + ): await submission._fetch_latest_entity(synapse_client=self.syn) @pytest.mark.asyncio @@ -372,7 +374,9 @@ async def test_fetch_latest_entity_api_error(self) -> None: side_effect=SynapseHTTPError("Entity not found"), ) as mock_rest_get: # THEN it should raise a ValueError with context about the original error - with pytest.raises(ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}"): + with pytest.raises( + ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}" + ): await submission._fetch_latest_entity(synapse_client=self.syn) @pytest.mark.asyncio @@ -395,24 +399,23 @@ async def test_store_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should fetch entity information, create the submission, and fill the object mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Check the call arguments to create_submission call_args = mock_create_submission.call_args request_body = call_args[0][0] etag = call_args[0][1] - + assert request_body["entityId"] == ENTITY_ID assert request_body["evaluationId"] == EVALUATION_ID assert request_body["name"] == SUBMISSION_NAME assert request_body["versionNumber"] == VERSION_NUMBER assert etag == ETAG - + # Verify the submission is filled with response data assert stored_submission.id == SUBMISSION_ID assert stored_submission.entity_id == ENTITY_ID @@ -429,12 +432,14 @@ async def test_store_async_docker_repository_success(self) -> None: # WHEN I call store_async with mocked Docker repository entity docker_entity_with_tag = self.get_example_docker_entity_response() - docker_entity_with_tag.update({ - "tag": "v2.0", - "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - }) - + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + with patch.object( submission, "_fetch_latest_entity", @@ -445,13 +450,12 @@ async def test_store_async_docker_repository_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should handle Docker repository specific logic mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify Docker repository attributes are set correctly assert submission.version_number == 1 # Docker repos get version 1 assert submission.docker_repository_name == "test/repository" @@ -479,13 +483,12 @@ async def test_store_async_with_team_data_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should preserve team information in the stored submission mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify team data is preserved assert stored_submission.team_id == TEAM_ID assert stored_submission.contributors == CONTRIBUTORS @@ -500,7 +503,9 @@ async def test_store_async_missing_entity_id(self) -> None: # WHEN I call store_async # THEN it should raise a ValueError during to_synapse_request - with pytest.raises(ValueError, match="entity_id is required to create a submission"): + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): await submission.store_async(synapse_client=self.syn) @pytest.mark.asyncio @@ -534,7 +539,6 @@ async def test_get_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_get_submission: - retrieved_submission = await submission.get_async(synapse_client=self.syn) # THEN it should call the API and fill the object @@ -610,7 +614,9 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( @@ -653,7 +659,6 @@ async def test_get_evaluation_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, @@ -691,7 +696,6 @@ async def test_get_user_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, @@ -724,7 +728,6 @@ async def test_get_submission_count_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_count: - response = await Submission.get_submission_count_async( evaluation_id=evaluation_id, status=status, From 660259ec9d84296891993fc55f5272c767ea7196 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:16:50 -0500 Subject: [PATCH 15/60] bug fix for Statuses: updated to_synapse_request to follow same pattern as evaluations design --- synapseclient/models/submission_status.py | 50 ++++++++++++++++------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 740351504..132be79c2 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -8,7 +8,7 @@ from synapseclient.annotations import to_submission_status_annotations from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys, merge_dataclass_entities +from synapseclient.core.utils import merge_dataclass_entities from synapseclient.models import Annotations from synapseclient.models.mixins.access_control import AccessControllable @@ -377,21 +377,41 @@ def to_synapse_request(self) -> Dict: Returns: A dictionary containing the request body for updating a submission status. + + Raises: + ValueError: If any required attributes are missing. """ - # Prepare request body with basic fields - request_body = delete_none_keys( - { - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - } - ) + # These attributes are required for updating a submission status + required_attributes = ["id", "etag", "status_version"] + + for attribute in required_attributes: + if getattr(self, attribute) is None: + raise ValueError( + f"Your submission status object is missing the '{attribute}' attribute. This attribute is required to update a submission status" + ) + + # Build request body with required fields + request_body = { + "id": self.id, + "etag": self.etag, + "statusVersion": self.status_version, + } + + # Add optional fields only if they have values + if self.status is not None: + request_body["status"] = self.status + if self.score is not None: + request_body["score"] = self.score + if self.report is not None: + request_body["report"] = self.report + if self.entity_id is not None: + request_body["entityId"] = self.entity_id + if self.version_number is not None: + request_body["versionNumber"] = self.version_number + if self.can_cancel is not None: + request_body["canCancel"] = self.can_cancel + if self.cancel_requested is not None: + request_body["cancelRequested"] = self.cancel_requested # Add annotations if present if self.annotations and len(self.annotations) > 0: From 55a229c3689b368d9b6104b1db6bc10610839bd3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:39:12 -0500 Subject: [PATCH 16/60] replace != with is not for full object comparison (not just keys) --- synapseclient/api/evaluation_services.py | 3 ++- synapseclient/models/submission_status.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 478976137..8cd34a942 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -678,7 +678,8 @@ async def update_submission_status( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/status" - + print("request body") + print(request_body) response = await client.rest_put_async(uri, body=json.dumps(request_body)) return response diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 132be79c2..b1dae7dee 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -323,7 +323,7 @@ class SubmissionStatus( def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( - not self._last_persistent_instance or self._last_persistent_instance != self + not self._last_persistent_instance or self._last_persistent_instance is not self ) def _set_last_persistent_instance(self) -> None: @@ -368,6 +368,7 @@ def fill_from_dict( self.submission_annotations = Annotations.from_dict( submission_annotations_dict ) + print(self.submission_annotations) return self From 347904675330530a4e1443d30f949906b456528d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:43:50 -0500 Subject: [PATCH 17/60] expose the is_private arg for to_submission_status_annotations ONLY FOR submission annotations --- synapseclient/models/submission_status.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index b1dae7dee..08815709c 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -286,6 +286,9 @@ class SubmissionStatus( ] = field(default_factory=dict, compare=False) """Annotations are additional key-value pair metadata that are associated with an object.""" + is_private: Optional[bool] = field(default=True, compare=False) + """Indicates whether the submission annotations are private (True) or public (False). Default is True.""" + entity_id: Optional[str] = None """ The Synapse ID of the Entity in this Submission. @@ -425,7 +428,7 @@ def to_synapse_request(self) -> Dict: if self.submission_annotations and len(self.submission_annotations) > 0: # Convert submission annotations to the format expected by the API request_body["submissionAnnotations"] = to_submission_status_annotations( - self.submission_annotations + self.submission_annotations, is_private=self.is_private ) return request_body From 896a3c12684800ebdbabd4443941b3edf684ffcd Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 12 Nov 2025 10:53:40 -0500 Subject: [PATCH 18/60] fixed submission status/submission annotations --- synapseclient/annotations.py | 122 ++++++++++++++++++++++ synapseclient/models/submission_status.py | 40 ++++--- 2 files changed, 149 insertions(+), 13 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index 2b7143f71..fd422cb89 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -228,6 +228,128 @@ def to_submission_status_annotations(annotations, is_private=True): return synapseAnnos +def to_submission_annotations( + id: typing.Union[str, int], + etag: str, + annotations: typing.Dict[str, typing.Any], + is_private: bool = True, + logger: typing.Optional[typing.Any] = None, +) -> typing.Dict[str, typing.Any]: + """ + Converts a normal dictionary to the format used for submission annotations, which is different from the format + used to annotate entities. + + This function creates the proper nested structure that includes id, etag, and annotations in the format + expected by the submissionAnnotations field of a SubmissionStatus request body. + + Arguments: + id: The unique ID of the submission being annotated. + etag: The etag of the submission status for optimistic concurrency control. + annotations: A normal Python dictionary comprised of the annotations to be added. + logger: An optional logger instance. If not provided, a default logger will be used. + + Returns: + A dictionary in the format expected by submissionAnnotations with nested structure containing + id, etag, and annotations object with type/value format. + + Example: Using this function + Converting annotations to submission format + + from synapseclient.annotations import to_submission_annotations + + # Input annotations + my_annotations = { + "score": 85, + "feedback": "Good work!" + } + + # Convert to submission annotations format + submission_annos = to_submission_annotations( + id="9999999", + etag="abc123", + annotations=my_annotations, + is_private=True + ) + + # Result: + # { + # "id": "9999999", + # "etag": "abc123", + # "annotations": { + # "score": {"type": "INTEGER", "value": [85]}, + # "feedback": {"type": "STRING", "value": ["Good work!"]}, + # } + # } + + Note: + This function is designed specifically for the submissionAnnotations field format, + which is part of the creation of a SubmissionStatus request body: + + + """ + # Create the base structure + submission_annos = {"id": str(id), "etag": str(etag), "annotations": {}} + + # Convert each annotation to the proper nested format + for key, value in annotations.items(): + # Ensure value is a list + if not isinstance(value, list): + value_list = [value] + else: + value_list = value + + # Warn about empty annotation values and skip them + if not value_list: + if logger: + logger.warning( + f"Annotation '{key}' has an empty value list and will be skipped" + ) + else: + from synapseclient import Synapse + + client = Synapse.get_client() + client.logger.warning( + f"Annotation '{key}' has an empty value list and will be skipped" + ) + continue + + # Determine type based on the first element + first_element = value_list[0] + + if isinstance(first_element, str): + submission_annos["annotations"][key] = { + "type": "STRING", + "value": value_list, + } + elif isinstance(first_element, bool): + # Convert booleans to lowercase strings + submission_annos["annotations"][key] = { + "type": "STRING", + "value": [str(v).lower() for v in value_list], + } + elif isinstance(first_element, int): + submission_annos["annotations"][key] = {"type": "LONG", "value": value_list} + elif isinstance(first_element, float): + submission_annos["annotations"][key] = { + "type": "DOUBLE", + "value": value_list, + } + elif is_date(first_element): + # Convert dates to unix timestamps + submission_annos["annotations"][key] = { + "type": "LONG", + "value": [to_unix_epoch_time(v) for v in value_list], + } + else: + # Default to string representation + submission_annos["annotations"][key] = { + "type": "STRING", + "value": [str(v) for v in value_list], + } + + return submission_annos + + # TODO: this should accept a status object and return its annotations or an empty dict if there are none def from_submission_status_annotations(annotations) -> dict: """ diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 08815709c..8fedbbde8 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,3 +1,4 @@ +from calendar import c from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union @@ -5,7 +6,10 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.annotations import to_submission_status_annotations +from synapseclient.annotations import ( + to_submission_annotations, + to_submission_status_annotations, +) from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import merge_dataclass_entities @@ -286,8 +290,8 @@ class SubmissionStatus( ] = field(default_factory=dict, compare=False) """Annotations are additional key-value pair metadata that are associated with an object.""" - is_private: Optional[bool] = field(default=True, compare=False) - """Indicates whether the submission annotations are private (True) or public (False). Default is True.""" + private_status_annotations: Optional[bool] = field(default=True, compare=False) + """Indicates whether the submission status annotations (NOT to be confused with submission annotations) are private (True) or public (False). Default is True.""" entity_id: Optional[str] = None """ @@ -326,7 +330,8 @@ class SubmissionStatus( def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( - not self._last_persistent_instance or self._last_persistent_instance is not self + not self._last_persistent_instance + or self._last_persistent_instance is not self ) def _set_last_persistent_instance(self) -> None: @@ -375,16 +380,25 @@ def fill_from_dict( return self - def to_synapse_request(self) -> Dict: + def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: """ Creates a request body expected by the Synapse REST API for the SubmissionStatus model. + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + Returns: A dictionary containing the request body for updating a submission status. Raises: ValueError: If any required attributes are missing. """ + # Get the client for logging + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + # These attributes are required for updating a submission status required_attributes = ["id", "etag", "status_version"] @@ -417,18 +431,17 @@ def to_synapse_request(self) -> Dict: if self.cancel_requested is not None: request_body["cancelRequested"] = self.cancel_requested - # Add annotations if present if self.annotations and len(self.annotations) > 0: - # Convert annotations to the format expected by the API request_body["annotations"] = to_submission_status_annotations( - self.annotations + self.annotations, self.private_status_annotations ) - # Add submission annotations if present if self.submission_annotations and len(self.submission_annotations) > 0: - # Convert submission annotations to the format expected by the API - request_body["submissionAnnotations"] = to_submission_status_annotations( - self.submission_annotations, is_private=self.is_private + request_body["submissionAnnotations"] = to_submission_annotations( + id=self.id, + etag=self.etag, + annotations=self.submission_annotations, + logger=logger, ) return request_body @@ -473,7 +486,8 @@ async def get_async( response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) - + print("raw response") + print(response) self.fill_from_dict(response) self._set_last_persistent_instance() return self From 26a02f1220b7defea9d8c3acf746be960cb771e3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 10:53:15 -0500 Subject: [PATCH 19/60] add support for legacy annotations --- synapseclient/annotations.py | 1 - synapseclient/models/submission_status.py | 38 ++++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index fd422cb89..fb2933b1f 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -232,7 +232,6 @@ def to_submission_annotations( id: typing.Union[str, int], etag: str, annotations: typing.Dict[str, typing.Any], - is_private: bool = True, logger: typing.Optional[typing.Any] = None, ) -> typing.Dict[str, typing.Any]: """ diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 8fedbbde8..1dc8646fa 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,4 +1,3 @@ -from calendar import c from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union @@ -199,9 +198,15 @@ class SubmissionStatus( status: The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). score: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. report: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. - annotations: Primary container object for Annotations on a Synapse object. - submission_annotations: Annotations are additional key-value pair metadata that are associated with an object. + annotations: Primary container object for Annotations on a Synapse object. These annotations use the legacy + format and do not show up in a submission view. The visibility is controlled by private_status_annotations. + submission_annotations: Submission Annotations are additional key-value pair metadata that are associated with an object. + These annotations use the modern nested format and show up in a submission view. + private_status_annotations: Indicates whether the annotations (not to be confused with submission annotations) are private (True) or public (False). + Default is True. This controls the visibility of the 'annotations' field. entity_id: The Synapse ID of the Entity in this Submission. + evaluation_id: The ID of the Evaluation to which this Submission belongs. This field is automatically + populated when retrieving a SubmissionStatus via get() and is required when updating annotations. version_number: The version number of the Entity in this Submission. status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. @@ -298,6 +303,11 @@ class SubmissionStatus( The Synapse ID of the Entity in this Submission. """ + evaluation_id: Optional[str] = None + """ + The ID of the Evaluation to which this Submission belongs. + """ + version_number: Optional[int] = field(default=None, compare=False) """ The version number of the Entity in this Submission. @@ -432,8 +442,20 @@ def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: request_body["cancelRequested"] = self.cancel_requested if self.annotations and len(self.annotations) > 0: + # evaluation_id is required when annotations are provided for scopeId + if self.evaluation_id is None: + raise ValueError( + "Your submission status object is missing the 'evaluation_id' attribute. This attribute is required when submissions are updated with annotations. Please retrieve your submission status with .get() to populate this field." + ) + + # Add required objectId and scopeId to annotations dict as per Synapse API requirements + # https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/annotation/Annotations.html + annotations_with_metadata = self.annotations.copy() + annotations_with_metadata["objectId"] = self.id + annotations_with_metadata["scopeId"] = self.evaluation_id + request_body["annotations"] = to_submission_status_annotations( - self.annotations, self.private_status_annotations + annotations_with_metadata, self.private_status_annotations ) if self.submission_annotations and len(self.submission_annotations) > 0: @@ -489,6 +511,14 @@ async def get_async( print("raw response") print(response) self.fill_from_dict(response) + + # Fetch evaluation_id from the associated submission since it's not in the SubmissionStatus response + if not self.evaluation_id: + submission_response = await evaluation_services.get_submission( + submission_id=self.id, synapse_client=synapse_client + ) + self.evaluation_id = submission_response.get("evaluationId", None) + self._set_last_persistent_instance() return self From 271181ae3f196b0dbbd7fedaaa1eb1cb4fcbfc72 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 13:59:42 -0500 Subject: [PATCH 20/60] remove debug prints --- synapseclient/api/evaluation_services.py | 3 +-- synapseclient/models/submission_status.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 8cd34a942..478976137 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -678,8 +678,7 @@ async def update_submission_status( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/status" - print("request body") - print(request_body) + response = await client.rest_put_async(uri, body=json.dumps(request_body)) return response diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 1dc8646fa..402c330be 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -508,8 +508,6 @@ async def get_async( response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) - print("raw response") - print(response) self.fill_from_dict(response) # Fetch evaluation_id from the associated submission since it's not in the SubmissionStatus response From c8c1041661455dc4356551f5e5cf31f068f683ba Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 14:12:59 -0500 Subject: [PATCH 21/60] get_all_submission_statuses now returns a list of substat objects --- synapseclient/models/submission_status.py | 54 +++++++++++++++-------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 402c330be..6585f8a68 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -89,7 +89,7 @@ def get_all_submission_statuses( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> List["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -105,8 +105,7 @@ def get_all_submission_statuses( instance from the Synapse class constructor. Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. + A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation ```python @@ -116,15 +115,17 @@ def get_all_submission_statuses( syn = Synapse() syn.login() - response = SubmissionStatus.get_all_submission_statuses( + statuses = SubmissionStatus.get_all_submission_statuses( evaluation_id="9614543", status="SCORED", limit=50 ) - print(f"Found {len(response['results'])} submission statuses") + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") ``` """ - return {} + return [] @staticmethod def batch_update_submission_statuses( @@ -386,7 +387,6 @@ def fill_from_dict( self.submission_annotations = Annotations.from_dict( submission_annotations_dict ) - print(self.submission_annotations) return self @@ -611,7 +611,7 @@ async def get_all_submission_statuses_async( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> List["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -627,8 +627,7 @@ async def get_all_submission_statuses_async( instance from the Synapse class constructor. Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. + A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation ```python @@ -638,15 +637,17 @@ async def get_all_submission_statuses_async( syn = Synapse() syn.login() - response = await SubmissionStatus.get_all_submission_statuses_async( + statuses = await SubmissionStatus.get_all_submission_statuses_async( evaluation_id="9614543", status="SCORED", limit=50 ) - print(f"Found {len(response['results'])} submission statuses") + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") ``` """ - return await evaluation_services.get_all_submission_statuses( + response = await evaluation_services.get_all_submission_statuses( evaluation_id=evaluation_id, status=status, limit=limit, @@ -654,6 +655,18 @@ async def get_all_submission_statuses_async( synapse_client=synapse_client, ) + # Convert each result to a SubmissionStatus object + submission_statuses = [] + for status_dict in response.get("results", []): + submission_status = SubmissionStatus() + submission_status.fill_from_dict(status_dict) + # Manually set evaluation_id since it's not part of the response + submission_status.evaluation_id = evaluation_id + submission_status._set_last_persistent_instance() + submission_statuses.append(submission_status) + + return submission_statuses + @staticmethod async def batch_update_submission_statuses_async( evaluation_id: str, @@ -689,12 +702,17 @@ async def batch_update_submission_statuses_async( syn = Synapse() syn.login() - # Prepare list of status updates - statuses = [ - SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), - SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) - ] + # Retrieve existing statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="RECEIVED" + ) + + # Modify statuses as needed + for status in statuses: + status.status = "SCORED" + # Update statuses in batch response = await SubmissionStatus.batch_update_submission_statuses_async( evaluation_id="9614543", statuses=statuses, From 043693129641bd08cb2f1684da5255cbe1af50f3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 14:24:11 -0500 Subject: [PATCH 22/60] docstring updates --- synapseclient/models/submission_status.py | 59 +++++++++++++++++------ 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 6585f8a68..4d4360ae9 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -35,7 +35,11 @@ def get( Returns: The SubmissionStatus instance retrieved from Synapse. - Example: Retrieving a submission status by ID. + Raises: + ValueError: If the submission status does not have an ID to get. + + Example: Retrieving a submission status by ID +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -43,7 +47,7 @@ def get( syn = Synapse() syn.login() - status = SubmissionStatus(id="syn1234").get() + status = SubmissionStatus(id="9999999").get() print(status) ``` """ @@ -63,9 +67,14 @@ def store( instance from the Synapse class constructor. Returns: - The updated SubmissionStatus instance. + The updated SubmissionStatus object. - Example: Update a submission status. + Raises: + ValueError: If the submission status is missing required fields. + + Example: Update a submission status +   + Update an existing submission status by first retrieving it, then modifying fields and storing the changes. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -73,10 +82,17 @@ def store( syn = Synapse() syn.login() - status = SubmissionStatus(id="syn1234").get() + # Get existing status + status = SubmissionStatus(id="9999999").get() + + # Update fields status.status = "SCORED" + status.submission_annotations = {"score": [85.5]} + + # Store the update status = status.store() - print("Updated SubmissionStatus.") + print(f"Updated status:") + print(status) ``` """ return self @@ -108,6 +124,8 @@ def get_all_submission_statuses( A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation +   + Retrieve a list of submission statuses for a specific evaluation, optionally filtered by status. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -155,6 +173,8 @@ def batch_update_submission_statuses( and other response information. Example: Batch update submission statuses +   + Update multiple submission statuses in a single batch operation for efficiency. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -162,12 +182,17 @@ def batch_update_submission_statuses( syn = Synapse() syn.login() - # Prepare list of status updates - statuses = [ - SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), - SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) - ] + # Retrieve existing statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="RECEIVED" + ) + + # Modify statuses as needed + for status in statuses: + status.status = "SCORED" + # Update statuses in batch response = SubmissionStatus.batch_update_submission_statuses( evaluation_id="9614543", statuses=statuses, @@ -216,6 +241,8 @@ class SubmissionStatus( Submission owner can read and request to change this value. Example: Retrieve and update a SubmissionStatus. +   + This example demonstrates the basic workflow of retrieving an existing submission status, updating its fields, and storing the changes back to Synapse. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -224,7 +251,7 @@ class SubmissionStatus( syn.login() # Get a submission status - status = SubmissionStatus(id="syn123456").get() + status = SubmissionStatus(id="9999999").get() # Update the status status.status = "SCORED" @@ -491,6 +518,7 @@ async def get_async( ValueError: If the submission status does not have an ID to get. Example: Retrieving a submission status by ID +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -498,7 +526,7 @@ async def get_async( syn = Synapse() syn.login() - status = await SubmissionStatus(id="syn1234").get_async() + status = await SubmissionStatus(id="9999999").get_async() print(status) ``` """ @@ -543,6 +571,7 @@ async def store_async( ValueError: If the submission status is missing required fields. Example: Update a submission status +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -551,7 +580,7 @@ async def store_async( syn.login() # Get existing status - status = await SubmissionStatus(id="syn1234").get_async() + status = await SubmissionStatus(id="9999999").get_async() # Update fields status.status = "SCORED" @@ -630,6 +659,7 @@ async def get_all_submission_statuses_async( A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -695,6 +725,7 @@ async def batch_update_submission_statuses_async( and other response information. Example: Batch update submission statuses +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus From cb48df223b4181996a81457935690d39674a1244 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 10:22:54 -0500 Subject: [PATCH 23/60] update submissionbundle docstrings, add more examples --- synapseclient/models/submission_bundle.py | 56 +++++++++++++++-------- synapseclient/models/submission_status.py | 54 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index ce4c736de..2bf1e8430 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,10 +1,9 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union from synapseclient import Synapse from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync -from synapseclient.models.mixins.access_control import AccessControllable if TYPE_CHECKING: from synapseclient.models.submission import Submission @@ -41,7 +40,11 @@ def get_evaluation_submission_bundles( A list of SubmissionBundle objects containing the submission bundles for the evaluation queue. + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + Example: Getting submission bundles for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -87,6 +90,7 @@ def get_user_submission_bundles( submission bundles for the evaluation queue. Example: Getting user submission bundles +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -95,12 +99,12 @@ def get_user_submission_bundles( syn.login() bundles = SubmissionBundle.get_user_submission_bundles( - evaluation_id="9614543", + evaluation_id="9999999", limit=25 ) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ return [] @@ -108,13 +112,11 @@ def get_user_submission_bundles( @dataclass @async_to_sync -class SubmissionBundle( - SubmissionBundleSynchronousProtocol, - AccessControllable, -): +class SubmissionBundle(SubmissionBundleSynchronousProtocol): """A `SubmissionBundle` object represents a bundle containing a Synapse Submission and its accompanying SubmissionStatus. This bundle provides convenient access to both the submission data and its current status in a single object. + Attributes: @@ -124,6 +126,7 @@ class SubmissionBundle( This object should be used to contain scoring data about the Submission. Example: Retrieve submission bundles for an evaluation. +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -141,6 +144,26 @@ class SubmissionBundle( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") ``` + + Example: Retrieve user submission bundles for an evaluation. +   + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + # Get current user's submission bundles for an evaluation + user_bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id="9999999", + limit=25 + ) + + for bundle in user_bundles: + print(f"Submission ID: {bundle.submission.id}") + print(f"Status: {bundle.submission_status.status}") + ``` """ submission: Optional["Submission"] = None @@ -155,12 +178,6 @@ class SubmissionBundle( This object should be used to contain scoring data about the Submission. """ - _last_persistent_instance: Optional["SubmissionBundle"] = field( - default=None, repr=False, compare=False - ) - """The last persistent instance of this object. This is used to determine if the - object has been changed and needs to be updated in Synapse.""" - def fill_from_dict( self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] ) -> "SubmissionBundle": @@ -223,6 +240,7 @@ async def get_evaluation_submission_bundles_async( The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. Example: Getting submission bundles for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -231,13 +249,13 @@ async def get_evaluation_submission_bundles_async( syn.login() bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id="9614543", + evaluation_id="9999999", status="SCORED", limit=50 ) print(f"Found {len(bundles)} submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ response = await evaluation_services.get_evaluation_submission_bundles( @@ -281,6 +299,8 @@ async def get_user_submission_bundles_async( submission bundles for the evaluation queue. Example: Getting user submission bundles +   + ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -289,12 +309,12 @@ async def get_user_submission_bundles_async( syn.login() bundles = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id="9614543", + evaluation_id="9999999", limit=25 ) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ response = await evaluation_services.get_user_submission_bundles( diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 4d4360ae9..93cc9e516 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -259,6 +259,60 @@ class SubmissionStatus( status = status.store() print(status) ``` + + Example: Get all submission statuses for an evaluation. +   + Retrieve multiple submission statuses for an evaluation queue with optional filtering. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get all RECEIVED statuses for an evaluation + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9999999", + status="RECEIVED", + limit=100 + ) + + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") + ``` + + Example: Batch update multiple submission statuses. +   + Efficiently update multiple submission statuses in a single operation. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Retrieve statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9999999", + status="RECEIVED" + ) + + # Update each status + for status in statuses: + status.status = "SCORED" + status.submission_annotations = { + "validation_score": 95.0, + "comments": "Passed validation" + } + + # Batch update all statuses + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id="9614543", + statuses=statuses + ) + print(f"Batch update completed: {response}") + ``` """ id: Optional[str] = None From dbe5f760f6eb816d06c3d5c36373e9991eac5add Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 11:27:54 -0500 Subject: [PATCH 24/60] initial sync test for status. moved evaluation_id retrieval to fill_from_dict for submissionstatus calls --- .../synchronous/test_submission_status.py | 748 ++++++++++++++++++ 1 file changed, 748 insertions(+) create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission_status.py diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py new file mode 100644 index 000000000..30b70e4bc --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -0,0 +1,748 @@ +"""Integration tests for the synapseclient.models.SubmissionStatus class.""" + +import os +import tempfile +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus + + +class TestSubmissionStatusRetrieval: + """Tests for retrieving SubmissionStatus objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_status_by_id( + self, test_submission: Submission, test_evaluation: Evaluation + ): + """Test retrieving a submission status by ID.""" + # WHEN I get a submission status by ID + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + + # THEN the submission status should be retrieved correctly + assert submission_status.id == test_submission.id + assert submission_status.entity_id == test_submission.entity_id + assert submission_status.evaluation_id == test_evaluation.id + assert submission_status.status is not None # Should have some status (e.g., "RECEIVED") + assert submission_status.etag is not None + assert submission_status.status_version is not None + assert submission_status.modified_on is not None + + async def test_get_submission_status_without_id(self): + """Test that getting a submission status without ID raises ValueError.""" + # WHEN I try to get a submission status without an ID + submission_status = SubmissionStatus() + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + submission_status.get(synapse_client=self.syn) + + async def test_get_submission_status_with_invalid_id(self): + """Test that getting a submission status with invalid ID raises exception.""" + # WHEN I try to get a submission status with an invalid ID + submission_status = SubmissionStatus(id="syn999999999999") + + # THEN it should raise a SynapseHTTPError (404) + with pytest.raises(SynapseHTTPError): + submission_status.get(synapse_client=self.syn) + + +class TestSubmissionStatusUpdates: + """Tests for updating SubmissionStatus objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + @pytest.fixture(scope="function") + async def test_submission_status( + self, test_submission: Submission + ) -> SubmissionStatus: + """Create a test submission status by getting the existing one.""" + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + return submission_status + + async def test_store_submission_status_with_status_change( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with a status change.""" + # GIVEN a submission status that exists + original_status = test_submission_status.status + original_etag = test_submission_status.etag + + # WHEN I update the status + test_submission_status.status = "VALIDATED" + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the submission status should be updated + assert updated_status.id == test_submission_status.id + assert updated_status.status == "VALIDATED" + assert updated_status.status != original_status + assert updated_status.etag != original_etag # etag should change + assert updated_status.status_version > test_submission_status.status_version + + async def test_store_submission_status_with_submission_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with submission annotations.""" + # WHEN I add submission annotations and store + test_submission_status.submission_annotations = { + "score": 85.5, + "validation_passed": True, + "feedback": "Good work!", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the submission annotations should be saved + assert updated_status.submission_annotations is not None + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == 85.5 + assert updated_status.submission_annotations["validation_passed"] == True + assert updated_status.submission_annotations["feedback"] == "Good work!" + + async def test_store_submission_status_with_legacy_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with legacy annotations.""" + # WHEN I add legacy annotations and store + test_submission_status.annotations = { + "internal_score": 92.3, + "reviewer_notes": "Excellent submission", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the legacy annotations should be saved + assert updated_status.annotations is not None + assert "internal_score" in updated_status.annotations + assert updated_status.annotations["internal_score"] == 92.3 + assert updated_status.annotations["reviewer_notes"] == "Excellent submission" + + async def test_store_submission_status_with_combined_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with both types of annotations.""" + # WHEN I add both submission and legacy annotations + test_submission_status.submission_annotations = { + "public_score": 78.0, + "category": "Bronze", + } + test_submission_status.annotations = { + "internal_review": True, + "notes": "Needs minor improvements", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN both types of annotations should be saved + assert updated_status.submission_annotations is not None + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == 78.0 + + assert updated_status.annotations is not None + assert "internal_review" in updated_status.annotations + assert updated_status.annotations["internal_review"] == True + + async def test_store_submission_status_with_private_annotations_false( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to False.""" + # WHEN I add legacy annotations with private_status_annotations set to False + test_submission_status.annotations = { + "public_internal_score": 88.5, + "public_notes": "This should be visible", + } + test_submission_status.private_status_annotations = False + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the annotations should be marked as not private in the request + assert "annotations" in request_body + annotations_data = request_body["annotations"] + assert "isPrivate" in annotations_data + assert annotations_data["isPrivate"] is False + + async def test_store_submission_status_with_private_annotations_true( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to True (default).""" + # WHEN I add legacy annotations with private_status_annotations set to True (default) + test_submission_status.annotations = { + "private_internal_score": 95.0, + "private_notes": "This should be private", + } + test_submission_status.private_status_annotations = True + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the annotations should be marked as private in the request + assert "annotations" in request_body + annotations_data = request_body["annotations"] + assert "isPrivate" in annotations_data + assert annotations_data["isPrivate"] is True + + async def test_store_submission_status_without_id(self): + """Test that storing a submission status without ID raises ValueError.""" + # WHEN I try to store a submission status without an ID + submission_status = SubmissionStatus(status="SCORED") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to update"): + submission_status.store(synapse_client=self.syn) + + async def test_store_submission_status_without_changes( + self, test_submission_status: SubmissionStatus + ): + """Test that storing a submission status without changes shows warning.""" + # GIVEN a submission status that hasn't been modified + # (it already has _last_persistent_instance set from get()) + + # WHEN I try to store it without making changes + result = test_submission_status.store(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is test_submission_status + + async def test_store_submission_status_change_tracking( + self, test_submission_status: SubmissionStatus + ): + """Test that change tracking works correctly.""" + # GIVEN a submission status that was retrieved (has_changed should be False) + assert not test_submission_status.has_changed + + # WHEN I make a change + test_submission_status.status = "SCORED" + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store the changes + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN has_changed should be False again + assert not updated_status.has_changed + + +class TestSubmissionStatusBulkOperations: + """Tests for bulk SubmissionStatus operations.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_files( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[File]: + """Create multiple test files for submission status tests.""" + files = [] + for i in range(3): + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write(f"This is test content {i} for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{i}_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + files.append(file) + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + return files + + @pytest.fixture(scope="function") + async def test_submissions( + self, + test_evaluation: Evaluation, + test_files: list[File], + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple test submissions for status tests.""" + submissions = [] + for i, file in enumerate(test_files): + submission = Submission( + entity_id=file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {i} {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + submissions.append(created_submission) + return submissions + + async def test_get_all_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting all submission statuses for an evaluation.""" + # WHEN I get all submission statuses for the evaluation + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should get submission statuses for all submissions + assert len(statuses) >= len(test_submissions) + status_ids = [status.id for status in statuses] + + # AND all test submissions should have their statuses in the results + for submission in test_submissions: + assert submission.id in status_ids + + # AND each status should have proper attributes + for status in statuses: + assert status.id is not None + assert status.evaluation_id == test_evaluation.id + assert status.status is not None + assert status.etag is not None + + async def test_get_all_submission_statuses_with_status_filter( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with status filter.""" + # WHEN I get submission statuses filtered by status + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should only get statuses with the specified status + for status in statuses: + assert status.status == "RECEIVED" + assert status.evaluation_id == test_evaluation.id + + async def test_get_all_submission_statuses_with_pagination( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with pagination.""" + # WHEN I get submission statuses with pagination + statuses_page1 = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 statuses + assert len(statuses_page1) <= 2 + + # WHEN I get the next page + statuses_page2 = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN the results should be different (assuming more than 2 submissions exist) + if len(statuses_page1) == 2 and len(statuses_page2) > 0: + page1_ids = {status.id for status in statuses_page1} + page2_ids = {status.id for status in statuses_page2} + assert page1_ids != page2_ids # Should be different sets + + async def test_batch_update_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating multiple submission statuses.""" + # GIVEN multiple submission statuses + statuses = [] + for submission in test_submissions: + status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) + # Update each status + status.status = "VALIDATED" + status.submission_annotations = { + "batch_score": 90.0 + (len(statuses) * 2), + "batch_processed": True, + } + statuses.append(status) + + # WHEN I batch update the statuses + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the batch update should succeed + assert response is not None + assert "batchToken" in response or response == {} # Response format may vary + + # AND I should be able to verify the updates by retrieving the statuses + for original_status in statuses: + updated_status = SubmissionStatus(id=original_status.id).get( + synapse_client=self.syn + ) + assert updated_status.status == "VALIDATED" + assert "batch_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["batch_processed"] == True + + async def test_batch_update_submission_statuses_large_batch( + self, test_evaluation: Evaluation + ): + """Test batch update behavior with larger batch (approaching limits).""" + # Note: This test demonstrates the pattern but doesn't create 500 submissions + # as that would be too expensive for regular test runs + + # GIVEN I have a list of statuses (simulated for this test) + statuses = [] + + # WHEN I try to batch update (even with empty list) + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the operation should complete without error + assert response is not None + + async def test_batch_update_submission_statuses_with_batch_tokens( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating with batch tokens for multi-batch operations.""" + if len(test_submissions) < 2: + pytest.skip("Need at least 2 submissions for batch token test") + + # GIVEN multiple statuses split into batches + all_statuses = [] + for submission in test_submissions: + status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) + status.status = "SCORED" + all_statuses.append(status) + + # Split into batches + batch1 = all_statuses[:1] + batch2 = all_statuses[1:] + + # WHEN I update the first batch + response1 = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=batch1, + is_first_batch=True, + is_last_batch=len(batch2) == 0, + synapse_client=self.syn, + ) + + # THEN I should get a response (possibly with batch token) + assert response1 is not None + + # IF there's a second batch and we got a batch token + if len(batch2) > 0: + batch_token = response1.get("batchToken") if isinstance(response1, dict) else None + + # WHEN I update the second batch with the token + response2 = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=batch2, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the second batch should also succeed + assert response2 is not None + + +class TestSubmissionStatusValidation: + """Tests for SubmissionStatus validation and error handling.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_to_synapse_request_missing_required_attributes(self): + """Test that to_synapse_request validates required attributes.""" + # WHEN I try to create a request with missing required attributes + submission_status = SubmissionStatus(id="123") # Missing etag, status_version + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + # WHEN I add etag but still missing status_version + submission_status.etag = "some-etag" + + # THEN it should raise a ValueError for status_version + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): + """Test that annotations require evaluation_id.""" + # WHEN I try to create a request with annotations but no evaluation_id + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + annotations={"test": "value"} + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_valid_attributes(self): + """Test that to_synapse_request works with valid attributes.""" + # WHEN I create a request with all required attributes + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + status="SCORED", + evaluation_id="eval123", + submission_annotations={"score": 85.5} + ) + + # THEN it should create a valid request body + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # AND the request should have the required fields + assert request_body["id"] == "123" + assert request_body["etag"] == "some-etag" + assert request_body["statusVersion"] == 1 + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + + async def test_fill_from_dict_with_complete_response(self): + """Test filling a SubmissionStatus from a complete API response.""" + # GIVEN a complete API response + api_response = { + "id": "123456", + "etag": "abcd-1234", + "modifiedOn": "2023-01-01T00:00:00.000Z", + "status": "SCORED", + "entityId": "syn789", + "versionNumber": 1, + "statusVersion": 2, + "canCancel": False, + "cancelRequested": False, + "annotations": { + "stringAnnos": { + "internal_note": ["This is internal"] + }, + "doubleAnnos": {}, + "longAnnos": {} + }, + "submissionAnnotations": { + "stringAnnos": { + "feedback": ["Great work!"] + }, + "doubleAnnos": { + "score": [92.5] + }, + "longAnnos": {} + } + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN all fields should be populated correctly + assert result.id == "123456" + assert result.etag == "abcd-1234" + assert result.modified_on == "2023-01-01T00:00:00.000Z" + assert result.status == "SCORED" + assert result.entity_id == "syn789" + assert result.version_number == 1 + assert result.status_version == 2 + assert result.can_cancel is False + assert result.cancel_requested is False + assert "internal_note" in result.annotations + assert result.annotations["internal_note"] == "This is internal" + assert "feedback" in result.submission_annotations + assert "score" in result.submission_annotations + assert result.submission_annotations["score"] == 92.5 + + async def test_fill_from_dict_with_minimal_response(self): + """Test filling a SubmissionStatus from a minimal API response.""" + # GIVEN a minimal API response + api_response = { + "id": "123456", + "status": "RECEIVED" + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN basic fields should be populated + assert result.id == "123456" + assert result.status == "RECEIVED" + # AND optional fields should have default values + assert result.etag is None + assert result.can_cancel is False + assert result.cancel_requested is False From 7c07af97063367685414a39ed1ae9fd761ecd7ab Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 12:29:56 -0500 Subject: [PATCH 25/60] update submissionBundle submissionstatus with evaluation_id --- synapseclient/models/submission_bundle.py | 16 ++-- synapseclient/models/submission_status.py | 5 +- .../synchronous/test_submission_status.py | 90 ++++++++++--------- 3 files changed, 59 insertions(+), 52 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 2bf1e8430..89f4fc50c 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -179,7 +179,8 @@ class SubmissionBundle(SubmissionBundleSynchronousProtocol): """ def fill_from_dict( - self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] + self, + synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]], ) -> "SubmissionBundle": """ Converts a response from the REST API into this dataclass. @@ -201,9 +202,10 @@ def fill_from_dict( submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict( - submission_status_dict - ) + self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + # Manually set evaluation_id from the submission data if available + if self.submission_status and self.submission and self.submission.evaluation_id: + self.submission_status.evaluation_id = self.submission.evaluation_id else: self.submission_status = None @@ -268,7 +270,8 @@ async def get_evaluation_submission_bundles_async( bundles = [] for bundle_dict in response.get("results", []): - bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundle = SubmissionBundle() + bundle.fill_from_dict(bundle_dict) bundles.append(bundle) return bundles @@ -327,7 +330,8 @@ async def get_user_submission_bundles_async( # Convert response to list of SubmissionBundle objects bundles = [] for bundle_dict in response.get("results", []): - bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundle = SubmissionBundle() + bundle.fill_from_dict(bundle_dict) bundles.append(bundle) return bundles diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 93cc9e516..d19ac9494 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -432,7 +432,8 @@ def _set_last_persistent_instance(self) -> None: self._last_persistent_instance = replace(self) def fill_from_dict( - self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] + self, + synapse_submission_status: Dict[str, Union[bool, str, int, float, List]], ) -> "SubmissionStatus": """ Converts a response from the REST API into this dataclass. @@ -744,7 +745,7 @@ async def get_all_submission_statuses_async( for status_dict in response.get("results", []): submission_status = SubmissionStatus() submission_status.fill_from_dict(status_dict) - # Manually set evaluation_id since it's not part of the response + # Manually set evaluation_id since it's not in the SubmissionStatus response submission_status.evaluation_id = evaluation_id submission_status._set_last_persistent_instance() submission_statuses.append(submission_status) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index 30b70e4bc..e68574f51 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -97,7 +97,9 @@ async def test_get_submission_status_by_id( assert submission_status.id == test_submission.id assert submission_status.entity_id == test_submission.entity_id assert submission_status.evaluation_id == test_evaluation.id - assert submission_status.status is not None # Should have some status (e.g., "RECEIVED") + assert ( + submission_status.status is not None + ) # Should have some status (e.g., "RECEIVED") assert submission_status.etag is not None assert submission_status.status_version is not None assert submission_status.modified_on is not None @@ -108,7 +110,9 @@ async def test_get_submission_status_without_id(self): submission_status = SubmissionStatus() # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): submission_status.get(synapse_client=self.syn) async def test_get_submission_status_with_invalid_id(self): @@ -229,7 +233,7 @@ async def test_store_submission_status_with_submission_annotations( # WHEN I add submission annotations and store test_submission_status.submission_annotations = { "score": 85.5, - "validation_passed": True, + "validation_passed": [True], "feedback": "Good work!", } updated_status = test_submission_status.store(synapse_client=self.syn) @@ -237,9 +241,9 @@ async def test_store_submission_status_with_submission_annotations( # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None assert "score" in updated_status.submission_annotations - assert updated_status.submission_annotations["score"] == 85.5 - assert updated_status.submission_annotations["validation_passed"] == True - assert updated_status.submission_annotations["feedback"] == "Good work!" + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["validation_passed"] == [True] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -255,8 +259,8 @@ async def test_store_submission_status_with_legacy_annotations( # THEN the legacy annotations should be saved assert updated_status.annotations is not None assert "internal_score" in updated_status.annotations - assert updated_status.annotations["internal_score"] == 92.3 - assert updated_status.annotations["reviewer_notes"] == "Excellent submission" + assert updated_status.annotations["internal_score"] == [92.3] + assert updated_status.annotations["reviewer_notes"] == ["Excellent submission"] async def test_store_submission_status_with_combined_annotations( self, test_submission_status: SubmissionStatus @@ -276,11 +280,11 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None assert "public_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["public_score"] == 78.0 + assert updated_status.submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None assert "internal_review" in updated_status.annotations - assert updated_status.annotations["internal_review"] == True + assert updated_status.annotations["internal_review"] == [True] async def test_store_submission_status_with_private_annotations_false( self, test_submission_status: SubmissionStatus @@ -292,10 +296,12 @@ async def test_store_submission_status_with_private_annotations_false( "public_notes": "This should be visible", } test_submission_status.private_status_annotations = False - + # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) - + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + # THEN the annotations should be marked as not private in the request assert "annotations" in request_body annotations_data = request_body["annotations"] @@ -312,10 +318,12 @@ async def test_store_submission_status_with_private_annotations_true( "private_notes": "This should be private", } test_submission_status.private_status_annotations = True - + # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) - + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + # THEN the annotations should be marked as private in the request assert "annotations" in request_body annotations_data = request_body["annotations"] @@ -328,7 +336,9 @@ async def test_store_submission_status_without_id(self): submission_status = SubmissionStatus(status="SCORED") # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to update"): + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): submission_status.store(synapse_client=self.syn) async def test_store_submission_status_without_changes( @@ -405,7 +415,9 @@ async def test_files( with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".txt" ) as temp_file: - temp_file.write(f"This is test content {i} for submission status testing.") + temp_file.write( + f"This is test content {i} for submission status testing." + ) temp_file_path = temp_file.name try: @@ -546,7 +558,7 @@ async def test_batch_update_submission_statuses( ) assert updated_status.status == "VALIDATED" assert "batch_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["batch_processed"] == True + assert updated_status.submission_annotations["batch_processed"] == [True] async def test_batch_update_submission_statuses_large_batch( self, test_evaluation: Evaluation @@ -554,10 +566,10 @@ async def test_batch_update_submission_statuses_large_batch( """Test batch update behavior with larger batch (approaching limits).""" # Note: This test demonstrates the pattern but doesn't create 500 submissions # as that would be too expensive for regular test runs - + # GIVEN I have a list of statuses (simulated for this test) statuses = [] - + # WHEN I try to batch update (even with empty list) response = SubmissionStatus.batch_update_submission_statuses( evaluation_id=test_evaluation.id, @@ -600,8 +612,10 @@ async def test_batch_update_submission_statuses_with_batch_tokens( # IF there's a second batch and we got a batch token if len(batch2) > 0: - batch_token = response1.get("batchToken") if isinstance(response1, dict) else None - + batch_token = ( + response1.get("batchToken") if isinstance(response1, dict) else None + ) + # WHEN I update the second batch with the token response2 = SubmissionStatus.batch_update_submission_statuses( evaluation_id=test_evaluation.id, @@ -644,10 +658,7 @@ async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): """Test that annotations require evaluation_id.""" # WHEN I try to create a request with annotations but no evaluation_id submission_status = SubmissionStatus( - id="123", - etag="some-etag", - status_version=1, - annotations={"test": "value"} + id="123", etag="some-etag", status_version=1, annotations={"test": "value"} ) # THEN it should raise a ValueError @@ -663,7 +674,7 @@ async def test_to_synapse_request_valid_attributes(self): status_version=1, status="SCORED", evaluation_id="eval123", - submission_annotations={"score": 85.5} + submission_annotations={"score": 85.5}, ) # THEN it should create a valid request body @@ -690,21 +701,15 @@ async def test_fill_from_dict_with_complete_response(self): "canCancel": False, "cancelRequested": False, "annotations": { - "stringAnnos": { - "internal_note": ["This is internal"] - }, + "stringAnnos": {"internal_note": ["This is internal"]}, "doubleAnnos": {}, - "longAnnos": {} + "longAnnos": {}, }, "submissionAnnotations": { - "stringAnnos": { - "feedback": ["Great work!"] - }, - "doubleAnnos": { - "score": [92.5] - }, - "longAnnos": {} - } + "stringAnnos": {"feedback": ["Great work!"]}, + "doubleAnnos": {"score": [92.5]}, + "longAnnos": {}, + }, } # WHEN I fill a SubmissionStatus from the response @@ -730,10 +735,7 @@ async def test_fill_from_dict_with_complete_response(self): async def test_fill_from_dict_with_minimal_response(self): """Test filling a SubmissionStatus from a minimal API response.""" # GIVEN a minimal API response - api_response = { - "id": "123456", - "status": "RECEIVED" - } + api_response = {"id": "123456", "status": "RECEIVED"} # WHEN I fill a SubmissionStatus from the response submission_status = SubmissionStatus() From 6a5e97b52dac17f86cd0d4c8f5fbb6000e5d8b77 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 16:22:50 -0500 Subject: [PATCH 26/60] patch sync substatus integ tests. style. --- synapseclient/models/submission_bundle.py | 10 +++- synapseclient/models/submission_status.py | 1 - .../synchronous/test_submission_status.py | 48 ++++++++++++------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 89f4fc50c..6c7ae42e4 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -202,9 +202,15 @@ def fill_from_dict( submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + self.submission_status = SubmissionStatus().fill_from_dict( + submission_status_dict + ) # Manually set evaluation_id from the submission data if available - if self.submission_status and self.submission and self.submission.evaluation_id: + if ( + self.submission_status + and self.submission + and self.submission.evaluation_id + ): self.submission_status.evaluation_id = self.submission.evaluation_id else: self.submission_status = None diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index d19ac9494..f50e099fc 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -665,7 +665,6 @@ async def store_async( "modified_on", "entity_id", "version_number", - "status_version", ], logger=logger, ) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index e68574f51..ccc3c4083 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -8,6 +8,7 @@ import pytest from synapseclient import Synapse +from synapseclient.annotations import from_submission_status_annotations from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus @@ -214,6 +215,7 @@ async def test_store_submission_status_with_status_change( # GIVEN a submission status that exists original_status = test_submission_status.status original_etag = test_submission_status.etag + original_status_version = test_submission_status.status_version # WHEN I update the status test_submission_status.status = "VALIDATED" @@ -224,7 +226,7 @@ async def test_store_submission_status_with_status_change( assert updated_status.status == "VALIDATED" assert updated_status.status != original_status assert updated_status.etag != original_etag # etag should change - assert updated_status.status_version > test_submission_status.status_version + assert updated_status.status_version > original_status_version async def test_store_submission_status_with_submission_annotations( self, test_submission_status: SubmissionStatus @@ -233,17 +235,18 @@ async def test_store_submission_status_with_submission_annotations( # WHEN I add submission annotations and store test_submission_status.submission_annotations = { "score": 85.5, - "validation_passed": [True], "feedback": "Good work!", } updated_status = test_submission_status.store(synapse_client=self.syn) # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None - assert "score" in updated_status.submission_annotations - assert updated_status.submission_annotations["score"] == [85.5] - assert updated_status.submission_annotations["validation_passed"] == [True] - assert updated_status.submission_annotations["feedback"] == ["Good work!"] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "score" in converted_submission_annotations + assert converted_submission_annotations["score"] == [85.5] + assert converted_submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -255,12 +258,16 @@ async def test_store_submission_status_with_legacy_annotations( "reviewer_notes": "Excellent submission", } updated_status = test_submission_status.store(synapse_client=self.syn) + assert updated_status.annotations is not None + + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) # THEN the legacy annotations should be saved - assert updated_status.annotations is not None - assert "internal_score" in updated_status.annotations - assert updated_status.annotations["internal_score"] == [92.3] - assert updated_status.annotations["reviewer_notes"] == ["Excellent submission"] + assert "internal_score" in converted_annotations + assert converted_annotations["internal_score"] == 92.3 + assert converted_annotations["reviewer_notes"] == "Excellent submission" async def test_store_submission_status_with_combined_annotations( self, test_submission_status: SubmissionStatus @@ -279,12 +286,18 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None - assert "public_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["public_score"] == [78.0] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "public_score" in converted_submission_annotations + assert converted_submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None - assert "internal_review" in updated_status.annotations - assert updated_status.annotations["internal_review"] == [True] + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "internal_review" in converted_annotations + assert converted_annotations["internal_review"] == "true" async def test_store_submission_status_with_private_annotations_false( self, test_submission_status: SubmissionStatus @@ -557,8 +570,11 @@ async def test_batch_update_submission_statuses( synapse_client=self.syn ) assert updated_status.status == "VALIDATED" - assert "batch_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["batch_processed"] == [True] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "batch_score" in converted_submission_annotations + assert converted_submission_annotations["batch_processed"] == ["true"] async def test_batch_update_submission_statuses_large_batch( self, test_evaluation: Evaluation From 9c2c0d10772b562920ba840465a77d97b319a502 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 17 Nov 2025 16:18:45 -0500 Subject: [PATCH 27/60] fix submissionStatus integ tests and has_changed attribute --- synapseclient/models/submission_status.py | 17 +- .../synchronous/test_submission_status.py | 224 ++++++++++-------- 2 files changed, 135 insertions(+), 106 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index f50e099fc..1e46d61b7 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -421,9 +421,22 @@ class SubmissionStatus( @property def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" + if not self._last_persistent_instance: + return True + + model_attributes_changed = self._last_persistent_instance != self + annotations_changed = ( + self._last_persistent_instance.annotations != self.annotations + ) + submission_annotations_changed = ( + self._last_persistent_instance.submission_annotations + != self.submission_annotations + ) + return ( - not self._last_persistent_instance - or self._last_persistent_instance is not self + model_attributes_changed + or annotations_changed + or submission_annotations_changed ) def _set_last_persistent_instance(self) -> None: diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index ccc3c4083..b2c4bd65c 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -2,6 +2,7 @@ import os import tempfile +import test import uuid from typing import Callable @@ -241,12 +242,9 @@ async def test_store_submission_status_with_submission_annotations( # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None - converted_submission_annotations = from_submission_status_annotations( - updated_status.submission_annotations - ) - assert "score" in converted_submission_annotations - assert converted_submission_annotations["score"] == [85.5] - assert converted_submission_annotations["feedback"] == ["Good work!"] + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -286,11 +284,8 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None - converted_submission_annotations = from_submission_status_annotations( - updated_status.submission_annotations - ) - assert "public_score" in converted_submission_annotations - assert converted_submission_annotations["public_score"] == [78.0] + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None converted_annotations = from_submission_status_annotations( @@ -310,16 +305,22 @@ async def test_store_submission_status_with_private_annotations_false( } test_submission_status.private_status_annotations = False - # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request( - synapse_client=self.syn + # WHEN I store the submission status + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations ) + assert "public_internal_score" in converted_annotations + assert converted_annotations["public_internal_score"] == 88.5 + assert converted_annotations["public_notes"] == "This should be visible" - # THEN the annotations should be marked as not private in the request - assert "annotations" in request_body - annotations_data = request_body["annotations"] - assert "isPrivate" in annotations_data - assert annotations_data["isPrivate"] is False + # AND the annotations should be marked as not private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + assert all(not anno["isPrivate"] for anno in annotations) async def test_store_submission_status_with_private_annotations_true( self, test_submission_status: SubmissionStatus @@ -337,11 +338,23 @@ async def test_store_submission_status_with_private_annotations_true( synapse_client=self.syn ) - # THEN the annotations should be marked as private in the request - assert "annotations" in request_body - annotations_data = request_body["annotations"] - assert "isPrivate" in annotations_data - assert annotations_data["isPrivate"] is True + # WHEN I store the submission status + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "private_internal_score" in converted_annotations + assert converted_annotations["private_internal_score"] == 95.0 + assert converted_annotations["private_notes"] == "This should be private" + + # AND the annotations should be marked as private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + print(annotations) + assert all(anno["isPrivate"] for anno in annotations) async def test_store_submission_status_without_id(self): """Test that storing a submission status without ID raises ValueError.""" @@ -386,6 +399,57 @@ async def test_store_submission_status_change_tracking( # THEN has_changed should be False again assert not updated_status.has_changed + async def test_has_changed_property_edge_cases( + self, test_submission_status: SubmissionStatus + ): + """Test the has_changed property with various edge cases and detailed scenarios.""" + # GIVEN a submission status that was just retrieved + assert not test_submission_status.has_changed + original_annotations = ( + test_submission_status.annotations.copy() + if test_submission_status.annotations + else {} + ) + + # WHEN I modify only annotations (not submission_annotations) + test_submission_status.annotations = {"test_key": "test_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I reset annotations to the original value (should be the same as the persistent instance) + test_submission_status.annotations = original_annotations + + # THEN has_changed should be False (same as original) + assert not test_submission_status.has_changed + + # WHEN I add a different annotation value + test_submission_status.annotations = {"different_key": "different_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store and get a fresh copy + updated_status = test_submission_status.store(synapse_client=self.syn) + fresh_status = SubmissionStatus(id=updated_status.id).get( + synapse_client=self.syn + ) + + # THEN the fresh copy should not have changes + assert not fresh_status.has_changed + + # WHEN I modify only submission_annotations + fresh_status.submission_annotations = {"new_key": ["new_value"]} + + # THEN has_changed should be True + assert fresh_status.has_changed + + # WHEN I modify a scalar field + fresh_status.status = "VALIDATED" + + # THEN has_changed should still be True + assert fresh_status.has_changed + class TestSubmissionStatusBulkOperations: """Tests for bulk SubmissionStatus operations.""" @@ -576,75 +640,6 @@ async def test_batch_update_submission_statuses( assert "batch_score" in converted_submission_annotations assert converted_submission_annotations["batch_processed"] == ["true"] - async def test_batch_update_submission_statuses_large_batch( - self, test_evaluation: Evaluation - ): - """Test batch update behavior with larger batch (approaching limits).""" - # Note: This test demonstrates the pattern but doesn't create 500 submissions - # as that would be too expensive for regular test runs - - # GIVEN I have a list of statuses (simulated for this test) - statuses = [] - - # WHEN I try to batch update (even with empty list) - response = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=statuses, - synapse_client=self.syn, - ) - - # THEN the operation should complete without error - assert response is not None - - async def test_batch_update_submission_statuses_with_batch_tokens( - self, test_evaluation: Evaluation, test_submissions: list[Submission] - ): - """Test batch updating with batch tokens for multi-batch operations.""" - if len(test_submissions) < 2: - pytest.skip("Need at least 2 submissions for batch token test") - - # GIVEN multiple statuses split into batches - all_statuses = [] - for submission in test_submissions: - status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) - status.status = "SCORED" - all_statuses.append(status) - - # Split into batches - batch1 = all_statuses[:1] - batch2 = all_statuses[1:] - - # WHEN I update the first batch - response1 = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=batch1, - is_first_batch=True, - is_last_batch=len(batch2) == 0, - synapse_client=self.syn, - ) - - # THEN I should get a response (possibly with batch token) - assert response1 is not None - - # IF there's a second batch and we got a batch token - if len(batch2) > 0: - batch_token = ( - response1.get("batchToken") if isinstance(response1, dict) else None - ) - - # WHEN I update the second batch with the token - response2 = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=batch2, - is_first_batch=False, - is_last_batch=True, - batch_token=batch_token, - synapse_client=self.syn, - ) - - # THEN the second batch should also succeed - assert response2 is not None - class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling.""" @@ -717,15 +712,26 @@ async def test_fill_from_dict_with_complete_response(self): "canCancel": False, "cancelRequested": False, "annotations": { - "stringAnnos": {"internal_note": ["This is internal"]}, - "doubleAnnos": {}, - "longAnnos": {}, - }, - "submissionAnnotations": { - "stringAnnos": {"feedback": ["Great work!"]}, - "doubleAnnos": {"score": [92.5]}, - "longAnnos": {}, + "objectId": "123456", + "scopeId": "9617645", + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + }, + { + "key": "reviewer_notes", + "isPrivate": True, + "value": "Excellent work", + }, + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, } # WHEN I fill a SubmissionStatus from the response @@ -742,11 +748,21 @@ async def test_fill_from_dict_with_complete_response(self): assert result.status_version == 2 assert result.can_cancel is False assert result.cancel_requested is False - assert "internal_note" in result.annotations - assert result.annotations["internal_note"] == "This is internal" + + # The annotations field should contain the raw submission status format + assert result.annotations is not None + assert "objectId" in result.annotations + assert "scopeId" in result.annotations + assert "stringAnnos" in result.annotations + assert "doubleAnnos" in result.annotations + assert len(result.annotations["stringAnnos"]) == 2 + assert len(result.annotations["doubleAnnos"]) == 1 + + # The submission_annotations should be in simple key-value format assert "feedback" in result.submission_annotations assert "score" in result.submission_annotations - assert result.submission_annotations["score"] == 92.5 + assert result.submission_annotations["feedback"] == ["Great work!"] + assert result.submission_annotations["score"] == [92.5] async def test_fill_from_dict_with_minimal_response(self): """Test filling a SubmissionStatus from a minimal API response.""" From aee4ccda03159402b4a6f9ecb56a77c44f2cc6c6 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 10:57:45 -0500 Subject: [PATCH 28/60] new substatus async integ tests. can_cancel can now be modified by an organizer on the client. cancel request returns no response body. --- synapseclient/models/submission.py | 8 +- synapseclient/models/submission_status.py | 6 +- .../async/test_submission_status_async.py | 785 ++++++++++++++++++ 3 files changed, 790 insertions(+), 9 deletions(-) create mode 100644 tests/integration/synapseclient/models/async/test_submission_status_async.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 5d7390050..7096201c9 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -699,7 +699,7 @@ async def cancel_submission_example(): if not self.id: raise ValueError("The submission must have an ID to cancel.") - response = await evaluation_services.cancel_submission( + await evaluation_services.cancel_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -707,8 +707,4 @@ async def cancel_submission_example(): client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger - logger.info(f"Submission {self.id} has successfully been cancelled.") - - # Update this object with the response - self.fill_from_dict(response) - return self + logger.info(f"A request to cancel Submission {self.id} has been submitted.") diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 1e46d61b7..60787be32 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -400,10 +400,10 @@ class SubmissionStatus( A version of the status, auto-generated and auto-incremented by the system and read-only to the client. """ - can_cancel: Optional[bool] = field(default=False, compare=False) + can_cancel: Optional[bool] = field(default=False) """ - Can this submission be cancelled? By default, this will be set to False. Users can read this value. - Only the queue's scoring application can change this value. + Can this submission be cancelled? By default, this will be set to False. Submission owner can read this value. + Only the queue's organizers can change this value. """ cancel_requested: Optional[bool] = field(default=False, compare=False) diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py new file mode 100644 index 000000000..f46aea107 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -0,0 +1,785 @@ +"""Integration tests for the synapseclient.models.SubmissionStatus class async methods.""" + +import os +import tempfile +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.annotations import from_submission_status_annotations +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus + + +class TestSubmissionStatusRetrieval: + """Tests for retrieving SubmissionStatus objects async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_status_by_id( + self, test_submission: Submission, test_evaluation: Evaluation + ): + """Test retrieving a submission status by ID async.""" + # WHEN I get a submission status by ID + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + + # THEN the submission status should be retrieved correctly + assert submission_status.id == test_submission.id + assert submission_status.entity_id == test_submission.entity_id + assert submission_status.evaluation_id == test_evaluation.id + assert ( + submission_status.status is not None + ) # Should have some status (e.g., "RECEIVED") + assert submission_status.etag is not None + assert submission_status.status_version is not None + assert submission_status.modified_on is not None + + async def test_get_submission_status_without_id(self): + """Test that getting a submission status without ID raises ValueError async.""" + # WHEN I try to get a submission status without an ID + submission_status = SubmissionStatus() + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): + await submission_status.get_async(synapse_client=self.syn) + + async def test_get_submission_status_with_invalid_id(self): + """Test that getting a submission status with invalid ID raises exception async.""" + # WHEN I try to get a submission status with an invalid ID + submission_status = SubmissionStatus(id="syn999999999999") + + # THEN it should raise a SynapseHTTPError (404) + with pytest.raises(SynapseHTTPError): + await submission_status.get_async(synapse_client=self.syn) + + +class TestSubmissionStatusUpdates: + """Tests for updating SubmissionStatus objects async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + @pytest.fixture(scope="function") + async def test_submission_status( + self, test_submission: Submission + ) -> SubmissionStatus: + """Create a test submission status by getting the existing one.""" + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + return submission_status + + async def test_store_submission_status_with_status_change( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with a status change async.""" + # GIVEN a submission status that exists + original_status = test_submission_status.status + original_etag = test_submission_status.etag + original_status_version = test_submission_status.status_version + + # WHEN I update the status + test_submission_status.status = "VALIDATED" + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN the submission status should be updated + assert updated_status.id == test_submission_status.id + assert updated_status.status == "VALIDATED" + assert updated_status.status != original_status + assert updated_status.etag != original_etag # etag should change + assert updated_status.status_version > original_status_version + + async def test_store_submission_status_with_submission_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with submission annotations async.""" + # WHEN I add submission annotations and store + test_submission_status.submission_annotations = { + "score": 85.5, + "feedback": "Good work!", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN the submission annotations should be saved + assert updated_status.submission_annotations is not None + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] + + async def test_store_submission_status_with_legacy_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with legacy annotations async.""" + # WHEN I add legacy annotations and store + test_submission_status.annotations = { + "internal_score": 92.3, + "reviewer_notes": "Excellent submission", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + assert updated_status.annotations is not None + + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + + # THEN the legacy annotations should be saved + assert "internal_score" in converted_annotations + assert converted_annotations["internal_score"] == 92.3 + assert converted_annotations["reviewer_notes"] == "Excellent submission" + + async def test_store_submission_status_with_combined_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with both types of annotations async.""" + # WHEN I add both submission and legacy annotations + test_submission_status.submission_annotations = { + "public_score": 78.0, + "category": "Bronze", + } + test_submission_status.annotations = { + "internal_review": True, + "notes": "Needs minor improvements", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN both types of annotations should be saved + assert updated_status.submission_annotations is not None + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == [78.0] + + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "internal_review" in converted_annotations + assert converted_annotations["internal_review"] == "true" + + async def test_store_submission_status_with_private_annotations_false( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to False async.""" + # WHEN I add legacy annotations with private_status_annotations set to False + test_submission_status.annotations = { + "public_internal_score": 88.5, + "public_notes": "This should be visible", + } + test_submission_status.private_status_annotations = False + + # WHEN I store the submission status + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "public_internal_score" in converted_annotations + assert converted_annotations["public_internal_score"] == 88.5 + assert converted_annotations["public_notes"] == "This should be visible" + + # AND the annotations should be marked as not private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + assert all(not anno["isPrivate"] for anno in annotations) + + async def test_store_submission_status_with_private_annotations_true( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to True (default) async.""" + # WHEN I add legacy annotations with private_status_annotations set to True (default) + test_submission_status.annotations = { + "private_internal_score": 95.0, + "private_notes": "This should be private", + } + test_submission_status.private_status_annotations = True + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + + # WHEN I store the submission status + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "private_internal_score" in converted_annotations + assert converted_annotations["private_internal_score"] == 95.0 + assert converted_annotations["private_notes"] == "This should be private" + + # AND the annotations should be marked as private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + print(annotations) + assert all(anno["isPrivate"] for anno in annotations) + + async def test_store_submission_status_without_id(self): + """Test that storing a submission status without ID raises ValueError async.""" + # WHEN I try to store a submission status without an ID + submission_status = SubmissionStatus(status="SCORED") + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + await submission_status.store_async(synapse_client=self.syn) + + async def test_store_submission_status_without_changes( + self, test_submission_status: SubmissionStatus + ): + """Test that storing a submission status without changes shows warning async.""" + # GIVEN a submission status that hasn't been modified + # (it already has _last_persistent_instance set from get()) + + # WHEN I try to store it without making changes + result = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is test_submission_status + + async def test_store_submission_status_change_tracking( + self, test_submission_status: SubmissionStatus + ): + """Test that change tracking works correctly async.""" + # GIVEN a submission status that was retrieved (has_changed should be False) + assert not test_submission_status.has_changed + + # WHEN I make a change + test_submission_status.status = "SCORED" + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store the changes + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN has_changed should be False again + assert not updated_status.has_changed + + async def test_has_changed_property_edge_cases( + self, test_submission_status: SubmissionStatus + ): + """Test the has_changed property with various edge cases and detailed scenarios async.""" + # GIVEN a submission status that was just retrieved + assert not test_submission_status.has_changed + original_annotations = ( + test_submission_status.annotations.copy() + if test_submission_status.annotations + else {} + ) + + # WHEN I modify only annotations (not submission_annotations) + test_submission_status.annotations = {"test_key": "test_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I reset annotations to the original value (should be the same as the persistent instance) + test_submission_status.annotations = original_annotations + + # THEN has_changed should be False (same as original) + assert not test_submission_status.has_changed + + # WHEN I add a different annotation value + test_submission_status.annotations = {"different_key": "different_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store and get a fresh copy + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + fresh_status = await SubmissionStatus(id=updated_status.id).get_async( + synapse_client=self.syn + ) + + # THEN the fresh copy should not have changes + assert not fresh_status.has_changed + + # WHEN I modify only submission_annotations + fresh_status.submission_annotations = {"new_key": ["new_value"]} + + # THEN has_changed should be True + assert fresh_status.has_changed + + # WHEN I modify a scalar field + fresh_status.status = "VALIDATED" + + # THEN has_changed should still be True + assert fresh_status.has_changed + + +class TestSubmissionStatusBulkOperations: + """Tests for bulk SubmissionStatus operations async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_files( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[File]: + """Create multiple test files for submission status tests.""" + files = [] + for i in range(3): + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write( + f"This is test content {i} for submission status testing." + ) + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{i}_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + files.append(stored_file) + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + return files + + @pytest.fixture(scope="function") + async def test_submissions( + self, + test_evaluation: Evaluation, + test_files: list[File], + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple test submissions for status tests.""" + submissions = [] + for i, file in enumerate(test_files): + submission = Submission( + entity_id=file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {i} {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + submissions.append(created_submission) + return submissions + + async def test_get_all_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting all submission statuses for an evaluation async.""" + # WHEN I get all submission statuses for the evaluation + statuses = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should get submission statuses for all submissions + assert len(statuses) >= len(test_submissions) + status_ids = [status.id for status in statuses] + + # AND all test submissions should have their statuses in the results + for submission in test_submissions: + assert submission.id in status_ids + + # AND each status should have proper attributes + for status in statuses: + assert status.id is not None + assert status.evaluation_id == test_evaluation.id + assert status.status is not None + assert status.etag is not None + + async def test_get_all_submission_statuses_with_status_filter( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with status filter async.""" + # WHEN I get submission statuses filtered by status + statuses = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should only get statuses with the specified status + for status in statuses: + assert status.status == "RECEIVED" + assert status.evaluation_id == test_evaluation.id + + async def test_get_all_submission_statuses_with_pagination( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with pagination async.""" + # WHEN I get submission statuses with pagination + statuses_page1 = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 statuses + assert len(statuses_page1) <= 2 + + # WHEN I get the next page + statuses_page2 = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN the results should be different (assuming more than 2 submissions exist) + if len(statuses_page1) == 2 and len(statuses_page2) > 0: + page1_ids = {status.id for status in statuses_page1} + page2_ids = {status.id for status in statuses_page2} + assert page1_ids != page2_ids # Should be different sets + + async def test_batch_update_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating multiple submission statuses async.""" + # GIVEN multiple submission statuses + statuses = [] + for submission in test_submissions: + status = await SubmissionStatus(id=submission.id).get_async(synapse_client=self.syn) + # Update each status + status.status = "VALIDATED" + status.submission_annotations = { + "batch_score": 90.0 + (len(statuses) * 2), + "batch_processed": True, + } + statuses.append(status) + + # WHEN I batch update the statuses + response = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the batch update should succeed + assert response is not None + assert "batchToken" in response or response == {} # Response format may vary + + # AND I should be able to verify the updates by retrieving the statuses + for original_status in statuses: + updated_status = await SubmissionStatus(id=original_status.id).get_async( + synapse_client=self.syn + ) + assert updated_status.status == "VALIDATED" + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "batch_score" in converted_submission_annotations + assert converted_submission_annotations["batch_processed"] == ["true"] + + +class TestSubmissionStatusValidation: + """Tests for SubmissionStatus validation and error handling async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_to_synapse_request_missing_required_attributes(self): + """Test that to_synapse_request validates required attributes async.""" + # WHEN I try to create a request with missing required attributes + submission_status = SubmissionStatus(id="123") # Missing etag, status_version + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + # WHEN I add etag but still missing status_version + submission_status.etag = "some-etag" + + # THEN it should raise a ValueError for status_version + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): + """Test that annotations require evaluation_id async.""" + # WHEN I try to create a request with annotations but no evaluation_id + submission_status = SubmissionStatus( + id="123", etag="some-etag", status_version=1, annotations={"test": "value"} + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_valid_attributes(self): + """Test that to_synapse_request works with valid attributes async.""" + # WHEN I create a request with all required attributes + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + status="SCORED", + evaluation_id="eval123", + submission_annotations={"score": 85.5}, + ) + + # THEN it should create a valid request body + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # AND the request should have the required fields + assert request_body["id"] == "123" + assert request_body["etag"] == "some-etag" + assert request_body["statusVersion"] == 1 + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + + async def test_fill_from_dict_with_complete_response(self): + """Test filling a SubmissionStatus from a complete API response async.""" + # GIVEN a complete API response + api_response = { + "id": "123456", + "etag": "abcd-1234", + "modifiedOn": "2023-01-01T00:00:00.000Z", + "status": "SCORED", + "entityId": "syn789", + "versionNumber": 1, + "statusVersion": 2, + "canCancel": False, + "cancelRequested": False, + "annotations": { + "objectId": "123456", + "scopeId": "9617645", + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + }, + { + "key": "reviewer_notes", + "isPrivate": True, + "value": "Excellent work", + }, + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN all fields should be populated correctly + assert result.id == "123456" + assert result.etag == "abcd-1234" + assert result.modified_on == "2023-01-01T00:00:00.000Z" + assert result.status == "SCORED" + assert result.entity_id == "syn789" + assert result.version_number == 1 + assert result.status_version == 2 + assert result.can_cancel is False + assert result.cancel_requested is False + + # The annotations field should contain the raw submission status format + assert result.annotations is not None + assert "objectId" in result.annotations + assert "scopeId" in result.annotations + assert "stringAnnos" in result.annotations + assert "doubleAnnos" in result.annotations + assert len(result.annotations["stringAnnos"]) == 2 + assert len(result.annotations["doubleAnnos"]) == 1 + + # The submission_annotations should be in simple key-value format + assert "feedback" in result.submission_annotations + assert "score" in result.submission_annotations + assert result.submission_annotations["feedback"] == ["Great work!"] + assert result.submission_annotations["score"] == [92.5] + + async def test_fill_from_dict_with_minimal_response(self): + """Test filling a SubmissionStatus from a minimal API response async.""" + # GIVEN a minimal API response + api_response = {"id": "123456", "status": "RECEIVED"} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN basic fields should be populated + assert result.id == "123456" + assert result.status == "RECEIVED" + # AND optional fields should have default values + assert result.etag is None + assert result.can_cancel is False + assert result.cancel_requested is False From 4a5fe53bcea55bab2ef3284f49020a52d46fd62d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 11:08:04 -0500 Subject: [PATCH 29/60] new test class for submission cancel functionality --- .../async/test_submission_status_async.py | 104 ++++++++++++++++++ .../synchronous/test_submission_status.py | 103 +++++++++++++++++ 2 files changed, 207 insertions(+) diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index f46aea107..b558d7f08 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -644,6 +644,110 @@ async def test_batch_update_submission_statuses( assert converted_submission_annotations["batch_processed"] == ["true"] +class TestSubmissionStatusCancellation: + """Tests for SubmissionStatus cancellation functionality async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_submission_cancellation_workflow( + self, test_submission: Submission + ): + """Test the complete submission cancellation workflow async.""" + # GIVEN a submission that exists + submission_id = test_submission.id + + # WHEN I get the initial submission status + initial_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + + # THEN initially it should not be cancellable or cancelled + assert initial_status.can_cancel is False + assert initial_status.cancel_requested is False + + # WHEN I update the submission status to allow cancellation + initial_status.can_cancel = True + updated_status = await initial_status.store_async(synapse_client=self.syn) + + # THEN the submission should be marked as cancellable + assert updated_status.can_cancel is True + assert updated_status.cancel_requested is False + + # WHEN I cancel the submission + await test_submission.cancel_async() + + # THEN I should be able to retrieve the updated status showing cancellation was requested + final_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + assert final_status.can_cancel is True + assert final_status.cancel_requested is True + + class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling async.""" diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index b2c4bd65c..b720a6ba3 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -641,6 +641,109 @@ async def test_batch_update_submission_statuses( assert converted_submission_annotations["batch_processed"] == ["true"] +class TestSubmissionStatusCancellation: + """Tests for SubmissionStatus cancellation functionality.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_submission_cancellation_workflow( + self, test_submission: Submission + ): + """Test the complete submission cancellation workflow.""" + # GIVEN a submission that exists + submission_id = test_submission.id + + # WHEN I get the initial submission status + initial_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) + + # THEN initially it should not be cancellable or cancelled + assert initial_status.can_cancel is False + assert initial_status.cancel_requested is False + + # WHEN I update the submission status to allow cancellation + initial_status.can_cancel = True + updated_status = initial_status.store(synapse_client=self.syn) + + # THEN the submission should be marked as cancellable + assert updated_status.can_cancel is True + assert updated_status.cancel_requested is False + + # WHEN I cancel the submission + test_submission.cancel() + + # THEN I should be able to retrieve the updated status showing cancellation was requested + final_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) + assert final_status.can_cancel is True + assert final_status.cancel_requested is True + + class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling.""" From 480d5de7a8e361107132383d61d440879aa2c47d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 11:19:37 -0500 Subject: [PATCH 30/60] substatus async unit tests --- .../unit_test_submission_status_async.py | 593 ++++++++++++++++++ 1 file changed, 593 insertions(+) create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_status_async.py diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py new file mode 100644 index 000000000..3ca7d906d --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py @@ -0,0 +1,593 @@ +"""Unit tests for the synapseclient.models.SubmissionStatus class.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import SubmissionStatus + +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" +SCORE = 85.5 +REPORT = "Test report" +VERSION_NUMBER = 1 +STATUS_VERSION = 1 +CAN_CANCEL = False +CANCEL_REQUESTED = False +PRIVATE_STATUS_ANNOTATIONS = True + + +class TestSubmissionStatus: + """Tests for the synapseclient.models.SubmissionStatus class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "score": SCORE, + "report": REPORT, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "statusVersion": STATUS_VERSION, + "canCancel": CAN_CANCEL, + "cancelRequested": CANCEL_REQUESTED, + "annotations": { + "objectId": SUBMISSION_STATUS_ID, + "scopeId": EVALUATION_ID, + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + } + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + def get_example_submission_dict(self) -> Dict[str, str]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "evaluationId": EVALUATION_ID, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "userId": "123456", + "submitterAlias": "test_user", + "createdOn": "2023-01-01T00:00:00.000Z", + } + + def test_init_submission_status(self) -> None: + """Test creating a SubmissionStatus with basic attributes.""" + # WHEN I create a SubmissionStatus object + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + + # THEN the SubmissionStatus should have the expected attributes + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + assert submission_status.entity_id == ENTITY_ID + assert submission_status.evaluation_id == EVALUATION_ID + assert submission_status.can_cancel is False # default value + assert submission_status.cancel_requested is False # default value + assert submission_status.private_status_annotations is True # default value + + def test_fill_from_dict(self) -> None: + """Test filling a SubmissionStatus from a REST API response.""" + # GIVEN an example submission status response + submission_status_data = self.get_example_submission_status_dict() + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(submission_status_data) + + # THEN all fields should be populated correctly + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.etag == ETAG + assert submission_status.modified_on == MODIFIED_ON + assert submission_status.status == STATUS + assert submission_status.score == SCORE + assert submission_status.report == REPORT + assert submission_status.entity_id == ENTITY_ID + assert submission_status.version_number == VERSION_NUMBER + assert submission_status.status_version == STATUS_VERSION + assert submission_status.can_cancel is CAN_CANCEL + assert submission_status.cancel_requested is CANCEL_REQUESTED + + # Check annotations + assert submission_status.annotations is not None + assert "objectId" in submission_status.annotations + assert "scopeId" in submission_status.annotations + assert "stringAnnos" in submission_status.annotations + assert "doubleAnnos" in submission_status.annotations + + # Check submission annotations + assert "feedback" in submission_status.submission_annotations + assert "score" in submission_status.submission_annotations + assert submission_status.submission_annotations["feedback"] == ["Great work!"] + assert submission_status.submission_annotations["score"] == [92.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionStatus from minimal REST API response.""" + # GIVEN a minimal submission status response + minimal_data = {"id": SUBMISSION_STATUS_ID, "status": STATUS} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(minimal_data) + + # THEN basic fields should be populated + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + # AND optional fields should have default values + assert submission_status.etag is None + assert submission_status.can_cancel is False + assert submission_status.cancel_requested is False + + async def test_get_async(self) -> None: + """Test retrieving a SubmissionStatus by ID.""" + # GIVEN a SubmissionStatus with an ID + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call get_async + with patch( + "synapseclient.api.evaluation_services.get_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_get_status, patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_dict(), + ) as mock_get_submission: + result = await submission_status.get_async(synapse_client=self.syn) + + # THEN the submission status should be retrieved + mock_get_status.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + + # AND the result should have the expected data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS + assert result.evaluation_id == EVALUATION_ID + assert result._last_persistent_instance is not None + + async def test_get_async_without_id(self) -> None: + """Test that getting a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call get_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + await submission_status.get_async(synapse_client=self.syn) + + async def test_store_async(self) -> None: + """Test storing a SubmissionStatus.""" + # GIVEN a SubmissionStatus with required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + ) + submission_status._set_last_persistent_instance() + + # AND I modify the status + submission_status.status = "VALIDATED" + + # WHEN I call store_async + with patch( + "synapseclient.api.evaluation_services.update_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_update: + result = await submission_status.store_async(synapse_client=self.syn) + + # THEN the submission status should be updated + mock_update.assert_called_once() + call_args = mock_update.call_args + assert call_args.kwargs["submission_id"] == SUBMISSION_STATUS_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # AND the result should have updated data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS # from mock response + assert result._last_persistent_instance is not None + + async def test_store_async_without_id(self) -> None: + """Test that storing a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus(status="SCORED") + + # WHEN I call store_async + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + await submission_status.store_async(synapse_client=self.syn) + + async def test_store_async_without_changes(self) -> None: + """Test storing a SubmissionStatus without changes.""" + # GIVEN a SubmissionStatus that hasn't been modified + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status=STATUS, + ) + submission_status._set_last_persistent_instance() + + # WHEN I call store_async without making changes + result = await submission_status.store_async(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is submission_status + + def test_to_synapse_request_missing_id(self) -> None: + """Test to_synapse_request with missing ID.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_etag(self) -> None: + """Test to_synapse_request with missing etag.""" + # GIVEN a SubmissionStatus with ID but no etag + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_status_version(self) -> None: + """Test to_synapse_request with missing status_version.""" + # GIVEN a SubmissionStatus with ID and etag but no status_version + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, etag=ETAG) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: + """Test to_synapse_request with annotations but missing evaluation_id.""" + # GIVEN a SubmissionStatus with annotations but no evaluation_id + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + annotations={"test": "value"}, + ) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_valid(self) -> None: + """Test to_synapse_request with valid attributes.""" + # GIVEN a SubmissionStatus with all required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + submission_annotations={"score": 85.5}, + annotations={"internal_note": "test"}, + ) + + # WHEN I call to_synapse_request + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the request should have the required fields + assert request_body["id"] == SUBMISSION_STATUS_ID + assert request_body["etag"] == ETAG + assert request_body["statusVersion"] == STATUS_VERSION + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + assert "annotations" in request_body + + def test_has_changed_property_new_instance(self) -> None: + """Test has_changed property for a new instance.""" + # GIVEN a new SubmissionStatus instance + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # THEN has_changed should be True (no persistent instance) + assert submission_status.has_changed is True + + def test_has_changed_property_after_get(self) -> None: + """Test has_changed property after retrieving from Synapse.""" + # GIVEN a SubmissionStatus that was retrieved (has persistent instance) + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False + assert submission_status.has_changed is False + + # WHEN I modify a field + submission_status.status = "VALIDATED" + + # THEN has_changed should be True + assert submission_status.has_changed is True + + def test_has_changed_property_annotations(self) -> None: + """Test has_changed property with annotation changes.""" + # GIVEN a SubmissionStatus with annotations + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + annotations={"original": "value"}, + submission_annotations={"score": 85.0}, + ) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False initially + assert submission_status.has_changed is False + + # WHEN I modify annotations + submission_status.annotations = {"modified": "value"} + + # THEN has_changed should be True + assert submission_status.has_changed is True + + # WHEN I reset annotations to original and modify submission_annotations + submission_status.annotations = {"original": "value"} + submission_status.submission_annotations = {"score": 90.0} + + # THEN has_changed should still be True + assert submission_status.has_changed is True + + async def test_get_all_submission_statuses_async(self) -> None: + """Test getting all submission statuses for an evaluation.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + ] + } + + # WHEN I call get_all_submission_statuses_async + with patch( + "synapseclient.api.evaluation_services.get_all_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_all: + result = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_all.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionStatus objects + assert len(result) == 2 + assert all(isinstance(status, SubmissionStatus) for status in result) + assert result[0].id == "123" + assert result[0].status == "RECEIVED" + assert result[1].id == "456" + assert result[1].status == "SCORED" + + async def test_batch_update_submission_statuses_async(self) -> None: + """Test batch updating submission statuses.""" + # GIVEN a list of SubmissionStatus objects + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ), + SubmissionStatus( + id="456", + etag="etag2", + status_version=1, + status="SCORED", + evaluation_id=EVALUATION_ID, + ), + ] + + # AND mock response + mock_response = {"batchToken": "token123"} + + # WHEN I call batch_update_submission_statuses_async + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_batch_update: + result = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=True, + is_last_batch=True, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_batch_update.assert_called_once() + call_args = mock_batch_update.call_args + assert call_args.kwargs["evaluation_id"] == EVALUATION_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # Check request body structure + request_body = call_args.kwargs["request_body"] + assert request_body["isFirstBatch"] is True + assert request_body["isLastBatch"] is True + assert "statuses" in request_body + assert len(request_body["statuses"]) == 2 + + # AND the result should be the mock response + assert result == mock_response + + async def test_batch_update_with_batch_token(self) -> None: + """Test batch update with batch token for subsequent batches.""" + # GIVEN a list of SubmissionStatus objects and a batch token + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ) + ] + batch_token = "previous_batch_token" + + # WHEN I call batch_update_submission_statuses_async with a batch token + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value={}, + ) as mock_batch_update: + await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the batch token should be included in the request + call_args = mock_batch_update.call_args + request_body = call_args.kwargs["request_body"] + assert request_body["batchToken"] == batch_token + assert request_body["isFirstBatch"] is False + + def test_set_last_persistent_instance(self) -> None: + """Test setting the last persistent instance.""" + # GIVEN a SubmissionStatus + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + annotations={"test": "value"}, + ) + + # WHEN I set the last persistent instance + submission_status._set_last_persistent_instance() + + # THEN the persistent instance should be set + assert submission_status._last_persistent_instance is not None + assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID + assert submission_status._last_persistent_instance.status == STATUS + assert submission_status._last_persistent_instance.annotations == {"test": "value"} + + # AND modifying the current instance shouldn't affect the persistent one + submission_status.status = "MODIFIED" + assert submission_status._last_persistent_instance.status == STATUS + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionStatus objects with the same data + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # THEN they should be equal + assert status1 == status2 + + # WHEN I modify one of them + status2.status = "DIFFERENT" + + # THEN they should not be equal + assert status1 != status2 + + def test_dataclass_fields_excluded_from_comparison(self) -> None: + """Test that certain fields are excluded from comparison.""" + # GIVEN two SubmissionStatus objects that differ only in comparison-excluded fields + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag1", + modified_on="2023-01-01", + cancel_requested=False, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag2", # different etag + modified_on="2023-01-02", # different modified_on + cancel_requested=True, # different cancel_requested + ) + + # THEN they should still be equal (these fields are excluded from comparison) + assert status1 == status2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionStatus.""" + # GIVEN a SubmissionStatus with some data + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I get the string representation + repr_str = repr(submission_status) + str_str = str(submission_status) + + # THEN it should contain the relevant information + assert SUBMISSION_STATUS_ID in repr_str + assert STATUS in repr_str + assert ENTITY_ID in repr_str + assert "SubmissionStatus" in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str From 4ca53e091c794e6aed8b89a0557757681c35f0e1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 13:59:14 -0500 Subject: [PATCH 31/60] remove compare=false for some attributes. update sync unit tests --- synapseclient/models/submission_status.py | 6 +- .../unit_test_submission_status.py | 593 ++++++++++++++++++ 2 files changed, 596 insertions(+), 3 deletions(-) create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 60787be32..d8be2c913 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -359,7 +359,7 @@ class SubmissionStatus( List[datetime], ], ] - ] = field(default_factory=dict, compare=False) + ] = field(default_factory=dict) """Primary container object for Annotations on a Synapse object.""" submission_annotations: Optional[ @@ -374,10 +374,10 @@ class SubmissionStatus( List[datetime], ], ] - ] = field(default_factory=dict, compare=False) + ] = field(default_factory=dict) """Annotations are additional key-value pair metadata that are associated with an object.""" - private_status_annotations: Optional[bool] = field(default=True, compare=False) + private_status_annotations: Optional[bool] = field(default=True) """Indicates whether the submission status annotations (NOT to be confused with submission annotations) are private (True) or public (False). Default is True.""" entity_id: Optional[str] = None diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py new file mode 100644 index 000000000..76b00641f --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -0,0 +1,593 @@ +"""Unit tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" + +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import SubmissionStatus + +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" +SCORE = 85.5 +REPORT = "Test report" +VERSION_NUMBER = 1 +STATUS_VERSION = 1 +CAN_CANCEL = False +CANCEL_REQUESTED = False +PRIVATE_STATUS_ANNOTATIONS = True + + +class TestSubmissionStatusSync: + """Tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "score": SCORE, + "report": REPORT, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "statusVersion": STATUS_VERSION, + "canCancel": CAN_CANCEL, + "cancelRequested": CANCEL_REQUESTED, + "annotations": { + "objectId": SUBMISSION_STATUS_ID, + "scopeId": EVALUATION_ID, + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + } + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + def get_example_submission_dict(self) -> Dict[str, str]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "evaluationId": EVALUATION_ID, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "userId": "123456", + "submitterAlias": "test_user", + "createdOn": "2023-01-01T00:00:00.000Z", + } + + def test_init_submission_status(self) -> None: + """Test creating a SubmissionStatus with basic attributes.""" + # WHEN I create a SubmissionStatus object + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + + # THEN the SubmissionStatus should have the expected attributes + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + assert submission_status.entity_id == ENTITY_ID + assert submission_status.evaluation_id == EVALUATION_ID + assert submission_status.can_cancel is False # default value + assert submission_status.cancel_requested is False # default value + assert submission_status.private_status_annotations is True # default value + + def test_fill_from_dict(self) -> None: + """Test filling a SubmissionStatus from a REST API response.""" + # GIVEN an example submission status response + submission_status_data = self.get_example_submission_status_dict() + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(submission_status_data) + + # THEN all fields should be populated correctly + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.etag == ETAG + assert submission_status.modified_on == MODIFIED_ON + assert submission_status.status == STATUS + assert submission_status.score == SCORE + assert submission_status.report == REPORT + assert submission_status.entity_id == ENTITY_ID + assert submission_status.version_number == VERSION_NUMBER + assert submission_status.status_version == STATUS_VERSION + assert submission_status.can_cancel is CAN_CANCEL + assert submission_status.cancel_requested is CANCEL_REQUESTED + + # Check annotations + assert submission_status.annotations is not None + assert "objectId" in submission_status.annotations + assert "scopeId" in submission_status.annotations + assert "stringAnnos" in submission_status.annotations + assert "doubleAnnos" in submission_status.annotations + + # Check submission annotations + assert "feedback" in submission_status.submission_annotations + assert "score" in submission_status.submission_annotations + assert submission_status.submission_annotations["feedback"] == ["Great work!"] + assert submission_status.submission_annotations["score"] == [92.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionStatus from minimal REST API response.""" + # GIVEN a minimal submission status response + minimal_data = {"id": SUBMISSION_STATUS_ID, "status": STATUS} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(minimal_data) + + # THEN basic fields should be populated + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + # AND optional fields should have default values + assert submission_status.etag is None + assert submission_status.can_cancel is False + assert submission_status.cancel_requested is False + + def test_get(self) -> None: + """Test retrieving a SubmissionStatus by ID using sync method.""" + # GIVEN a SubmissionStatus with an ID + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call get (sync method) + with patch( + "synapseclient.api.evaluation_services.get_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_get_status, patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_dict(), + ) as mock_get_submission: + result = submission_status.get(synapse_client=self.syn) + + # THEN the submission status should be retrieved + mock_get_status.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + + # AND the result should have the expected data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS + assert result.evaluation_id == EVALUATION_ID + + def test_get_without_id(self) -> None: + """Test that getting a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call get + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + submission_status.get(synapse_client=self.syn) + + def test_store(self) -> None: + """Test storing a SubmissionStatus using sync method.""" + # GIVEN a SubmissionStatus with required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + ) + submission_status._set_last_persistent_instance() + + # AND I modify the status + submission_status.status = "VALIDATED" + + # WHEN I call store (sync method) + with patch( + "synapseclient.api.evaluation_services.update_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_update: + result = submission_status.store(synapse_client=self.syn) + + # THEN the submission status should be updated + mock_update.assert_called_once() + call_args = mock_update.call_args + assert call_args.kwargs["submission_id"] == SUBMISSION_STATUS_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # AND the result should have updated data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS # from mock response + + def test_store_without_id(self) -> None: + """Test that storing a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus(status="SCORED") + + # WHEN I call store + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + submission_status.store(synapse_client=self.syn) + + def test_store_without_changes(self) -> None: + """Test storing a SubmissionStatus without changes.""" + # GIVEN a SubmissionStatus that hasn't been modified + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status=STATUS, + ) + submission_status._set_last_persistent_instance() + + # WHEN I call store without making changes + result = submission_status.store(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is submission_status + + def test_to_synapse_request_missing_id(self) -> None: + """Test to_synapse_request with missing ID.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_etag(self) -> None: + """Test to_synapse_request with missing etag.""" + # GIVEN a SubmissionStatus with ID but no etag + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_status_version(self) -> None: + """Test to_synapse_request with missing status_version.""" + # GIVEN a SubmissionStatus with ID and etag but no status_version + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, etag=ETAG) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: + """Test to_synapse_request with annotations but missing evaluation_id.""" + # GIVEN a SubmissionStatus with annotations but no evaluation_id + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + annotations={"test": "value"}, + ) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_valid(self) -> None: + """Test to_synapse_request with valid attributes.""" + # GIVEN a SubmissionStatus with all required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + submission_annotations={"score": 85.5}, + annotations={"internal_note": "test"}, + ) + + # WHEN I call to_synapse_request + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the request should have the required fields + assert request_body["id"] == SUBMISSION_STATUS_ID + assert request_body["etag"] == ETAG + assert request_body["statusVersion"] == STATUS_VERSION + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + assert "annotations" in request_body + + def test_has_changed_property_new_instance(self) -> None: + """Test has_changed property for a new instance.""" + # GIVEN a new SubmissionStatus instance + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # THEN has_changed should be True (no persistent instance) + assert submission_status.has_changed is True + + def test_has_changed_property_after_get(self) -> None: + """Test has_changed property after retrieving from Synapse.""" + # GIVEN a SubmissionStatus that was retrieved (has persistent instance) + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False + assert submission_status.has_changed is False + + # WHEN I modify a field + submission_status.status = "VALIDATED" + + # THEN has_changed should be True + assert submission_status.has_changed is True + + def test_has_changed_property_annotations(self) -> None: + """Test has_changed property with annotation changes.""" + # GIVEN a SubmissionStatus with annotations + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + annotations={"original": "value"}, + submission_annotations={"score": 85.0}, + ) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False initially + assert submission_status.has_changed is False + + # WHEN I modify annotations + submission_status.annotations = {"modified": "value"} + + # THEN has_changed should be True + assert submission_status.has_changed is True + + # WHEN I reset annotations to original and modify submission_annotations + submission_status.annotations = {"original": "value"} + submission_status.submission_annotations = {"score": 90.0} + + # THEN has_changed should still be True + assert submission_status.has_changed is True + + def test_get_all_submission_statuses(self) -> None: + """Test getting all submission statuses using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + ] + } + + # WHEN I call get_all_submission_statuses (sync method) + with patch( + "synapseclient.api.evaluation_services.get_all_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_all: + result = SubmissionStatus.get_all_submission_statuses( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_all.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionStatus objects + assert len(result) == 2 + assert all(isinstance(status, SubmissionStatus) for status in result) + assert result[0].id == "123" + assert result[0].status == "RECEIVED" + assert result[1].id == "456" + assert result[1].status == "SCORED" + + def test_batch_update_submission_statuses(self) -> None: + """Test batch updating submission statuses using sync method.""" + # GIVEN a list of SubmissionStatus objects + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ), + SubmissionStatus( + id="456", + etag="etag2", + status_version=1, + status="SCORED", + evaluation_id=EVALUATION_ID, + ), + ] + + # AND mock response + mock_response = {"batchToken": "token123"} + + # WHEN I call batch_update_submission_statuses (sync method) + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_batch_update: + result = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=True, + is_last_batch=True, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_batch_update.assert_called_once() + call_args = mock_batch_update.call_args + assert call_args.kwargs["evaluation_id"] == EVALUATION_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # Check request body structure + request_body = call_args.kwargs["request_body"] + assert request_body["isFirstBatch"] is True + assert request_body["isLastBatch"] is True + assert "statuses" in request_body + assert len(request_body["statuses"]) == 2 + + # AND the result should be the mock response + assert result == mock_response + + def test_batch_update_with_batch_token(self) -> None: + """Test batch update with batch token for subsequent batches.""" + # GIVEN a list of SubmissionStatus objects and a batch token + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ) + ] + batch_token = "previous_batch_token" + + # WHEN I call batch_update_submission_statuses with a batch token + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value={}, + ) as mock_batch_update: + SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the batch token should be included in the request + call_args = mock_batch_update.call_args + request_body = call_args.kwargs["request_body"] + assert request_body["batchToken"] == batch_token + assert request_body["isFirstBatch"] is False + + def test_set_last_persistent_instance(self) -> None: + """Test setting the last persistent instance.""" + # GIVEN a SubmissionStatus + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + annotations={"test": "value"}, + ) + + # WHEN I set the last persistent instance + submission_status._set_last_persistent_instance() + + # THEN the persistent instance should be set + assert submission_status._last_persistent_instance is not None + assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID + assert submission_status._last_persistent_instance.status == STATUS + assert submission_status._last_persistent_instance.annotations == {"test": "value"} + + # AND modifying the current instance shouldn't affect the persistent one + submission_status.status = "MODIFIED" + assert submission_status._last_persistent_instance.status == STATUS + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionStatus objects with the same data + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # THEN they should be equal + assert status1 == status2 + + # WHEN I modify one of them + status2.status = "DIFFERENT" + + # THEN they should not be equal + assert status1 != status2 + + def test_dataclass_fields_excluded_from_comparison(self) -> None: + """Test that certain fields are excluded from comparison.""" + # GIVEN two SubmissionStatus objects that differ only in comparison-excluded fields + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag1", + modified_on="2023-01-01", + cancel_requested=False, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag2", # different etag + modified_on="2023-01-02", # different modified_on + cancel_requested=True, # different cancel_requested + ) + + # THEN they should still be equal (these fields are excluded from comparison) + assert status1 == status2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionStatus.""" + # GIVEN a SubmissionStatus with some data + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I get the string representation + repr_str = repr(submission_status) + str_str = str(submission_status) + + # THEN it should contain the relevant information + assert SUBMISSION_STATUS_ID in repr_str + assert STATUS in repr_str + assert ENTITY_ID in repr_str + assert "SubmissionStatus" in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str From 3f25df8242e1d70b4a08f0ac27099999a0215b60 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:29:24 -0500 Subject: [PATCH 32/60] add submissionBundle integration tests --- .../async/test_submission_bundle_async.py | 596 ++++++++++++++++++ .../synchronous/test_submission_bundle.py | 560 ++++++++++++++++ 2 files changed, 1156 insertions(+) create mode 100644 tests/integration/synapseclient/models/async/test_submission_bundle_async.py create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission_bundle.py diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py new file mode 100644 index 000000000..b9162b726 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -0,0 +1,596 @@ +"""Integration tests for the synapseclient.models.SubmissionBundle class async methods.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus + + +class TestSubmissionBundleRetrievalAsync: + """Tests for retrieving SubmissionBundle objects using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for SubmissionBundle async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for submission bundle async tests {uuid.uuid4()}" + with open("test_file_for_submission_bundle_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_submission_bundle_async.txt", + name=f"test_submission_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_bundle_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + @pytest.fixture(scope="function") + async def multiple_submissions( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple submissions for testing pagination and filtering.""" + submissions = [] + for i in range(3): + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}_{i}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias=f"test_user_async_{i}", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + submissions.append(submission) + return submissions + + async def test_get_evaluation_submission_bundles_basic_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles for an evaluation using async methods.""" + # WHEN I get submission bundles for an evaluation + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in bundles" + + async def test_get_evaluation_submission_bundles_with_status_filter_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles filtered by status using async methods.""" + # WHEN I get submission bundles filtered by "RECEIVED" status + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + + # AND all bundles should have RECEIVED status (if any exist) + for bundle in bundles: + if bundle.submission_status: + assert bundle.submission_status.status == "RECEIVED" + + # WHEN I attempt to get submission bundles with an invalid status + with pytest.raises(SynapseHTTPError) as exc_info: + await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + # THEN it should raise a SynapseHTTPError (400 for invalid enum) + assert exc_info.value.response.status_code == 400 + assert "No enum constant" in str(exc_info.value) + assert "NONEXISTENT_STATUS" in str(exc_info.value) + + async def test_get_evaluation_submission_bundles_with_pagination_async( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting submission bundles using async methods.""" + # WHEN I get submission bundles with a limit + bundles_page1 = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): + """Test getting submission bundles for invalid evaluation ID using async methods.""" + # WHEN I try to get submission bundles for a non-existent evaluation + with pytest.raises(SynapseHTTPError) as exc_info: + await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + + # THEN it should raise a SynapseHTTPError (likely 403 or 404) + assert exc_info.value.response.status_code in [403, 404] + + async def test_get_user_submission_bundles_basic_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting user submission bundles for an evaluation using async methods.""" + # WHEN I get user submission bundles for an evaluation + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in user bundles" + + async def test_get_user_submission_bundles_with_pagination_async( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting user submission bundles using async methods.""" + # WHEN I get user submission bundles with a limit + bundles_page1 = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + +class TestSubmissionBundleDataIntegrityAsync: + """Tests for data integrity and relationships in SubmissionBundle objects using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for data integrity async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for data integrity async tests {uuid.uuid4()}" + with open("test_file_for_data_integrity_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_data_integrity_async.txt", + name=f"test_integrity_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_integrity_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_submission_bundle_data_consistency_async( + self, test_evaluation: Evaluation, test_submission: Submission, test_file: File + ): + """Test that submission bundles maintain data consistency between submission and status using async methods.""" + # WHEN I get submission bundles for the evaluation + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should find our test submission + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None, "Test submission bundle should be found" + + # AND the submission data should be consistent + assert test_bundle.submission.id == test_submission.id + assert test_bundle.submission.entity_id == test_file.id + assert test_bundle.submission.evaluation_id == test_evaluation.id + assert test_bundle.submission.name == test_submission.name + + # AND if there's a submission status, it should reference the same entities + if test_bundle.submission_status: + assert test_bundle.submission_status.id == test_submission.id + assert test_bundle.submission_status.entity_id == test_file.id + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + async def test_submission_bundle_status_updates_reflected_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that submission status updates are reflected in bundles using async methods.""" + # GIVEN a submission status that I can update + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + original_status = submission_status.status + + # WHEN I update the submission status + submission_status.status = "VALIDATED" + submission_status.submission_annotations = { + "test_score": 95.5, + "test_feedback": "Excellent work!", + } + updated_status = await submission_status.store_async(synapse_client=self.syn) + + # AND I get submission bundles again + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundle should reflect the updated status + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + assert test_bundle.submission_status is not None + assert test_bundle.submission_status.status == "VALIDATED" + assert test_bundle.submission_status.submission_annotations is not None + assert "test_score" in test_bundle.submission_status.submission_annotations + assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + + # CLEANUP: Reset the status back to original + submission_status.status = original_status + submission_status.submission_annotations = {} + await submission_status.store_async(synapse_client=self.syn) + + async def test_submission_bundle_evaluation_id_propagation_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that evaluation_id is properly propagated from submission to status using async methods.""" + # WHEN I get submission bundles + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN find our test bundle + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + + # AND both submission and status should have the correct evaluation_id + assert test_bundle.submission.evaluation_id == test_evaluation.id + if test_bundle.submission_status: + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + +class TestSubmissionBundleEdgeCasesAsync: + """Tests for edge cases and error handling in SubmissionBundle operations using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for edge case async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for edge case async tests {uuid.uuid4()}" + with open("test_file_for_edge_case_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_edge_case_async.txt", + name=f"test_edge_case_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_edge_case_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_get_evaluation_submission_bundles_empty_evaluation_async( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles from an evaluation with no submissions using async methods.""" + # WHEN I get submission bundles from an evaluation with no submissions + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_user_submission_bundles_empty_evaluation_async( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles from an evaluation with no submissions using async methods.""" + # WHEN I get user submission bundles from an evaluation with no submissions + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_evaluation_submission_bundles_large_limit_async( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles with a very large limit using async methods.""" + # WHEN I request bundles with a large limit (within API bounds) + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=100, # Maximum allowed by API + synapse_client=self.syn, + ) + + # THEN it should work without error + assert bundles is not None + assert isinstance(bundles, list) + # The actual count doesn't matter since the evaluation is empty + + async def test_get_user_submission_bundles_large_offset_async( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles with a large offset using async methods.""" + # WHEN I request bundles with a large offset (beyond available data) + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + offset=1000, # Large offset beyond any real data + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_submission_bundles_with_default_parameters_async( + self, test_evaluation: Evaluation + ): + """Test that default parameters work correctly using async methods.""" + # WHEN I call methods without optional parameters + eval_bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + user_bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN both should work with default values + assert eval_bundles is not None + assert user_bundles is not None + assert isinstance(eval_bundles, list) + assert isinstance(user_bundles, list) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py new file mode 100644 index 000000000..6e32a3cad --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -0,0 +1,560 @@ +"""Integration tests for the synapseclient.models.SubmissionBundle class.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus + + +class TestSubmissionBundleRetrieval: + """Tests for retrieving SubmissionBundle objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for SubmissionBundle testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for submission bundle tests {uuid.uuid4()}" + with open("test_file_for_submission_bundle.txt", "w") as f: + f.write(file_content) + + file_entity = File( + path="test_file_for_submission_bundle.txt", + name=f"test_submission_file_{uuid.uuid4()}", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_bundle", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + @pytest.fixture(scope="function") + async def multiple_submissions( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple submissions for testing pagination and filtering.""" + submissions = [] + for i in range(3): + submission = Submission( + name=f"test_submission_{uuid.uuid4()}_{i}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias=f"test_user_{i}", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + submissions.append(submission) + return submissions + + async def test_get_evaluation_submission_bundles_basic( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles for an evaluation.""" + # WHEN I get submission bundles for an evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in bundles" + + async def test_get_evaluation_submission_bundles_with_status_filter( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles filtered by status.""" + # WHEN I get submission bundles filtered by "RECEIVED" status + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + + # AND all bundles should have RECEIVED status (if any exist) + for bundle in bundles: + if bundle.submission_status: + assert bundle.submission_status.status == "RECEIVED" + + # WHEN I attempt to get submission bundles with an invalid status + with pytest.raises(SynapseHTTPError) as exc_info: + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + # THEN it should raise a SynapseHTTPError (400 for invalid enum) + assert exc_info.value.response.status_code == 400 + assert "No enum constant" in str(exc_info.value) + assert "NONEXISTENT_STATUS" in str(exc_info.value) + + async def test_get_evaluation_submission_bundles_with_pagination( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting submission bundles.""" + # WHEN I get submission bundles with a limit + bundles_page1 = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + async def test_get_evaluation_submission_bundles_invalid_evaluation(self): + """Test getting submission bundles for invalid evaluation ID.""" + # WHEN I try to get submission bundles for a non-existent evaluation + with pytest.raises(SynapseHTTPError) as exc_info: + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + + # THEN it should raise a SynapseHTTPError (likely 403 or 404) + assert exc_info.value.response.status_code in [403, 404] + + async def test_get_user_submission_bundles_basic( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting user submission bundles for an evaluation.""" + # WHEN I get user submission bundles for an evaluation + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in user bundles" + + async def test_get_user_submission_bundles_with_pagination( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting user submission bundles.""" + # WHEN I get user submission bundles with a limit + bundles_page1 = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + +class TestSubmissionBundleDataIntegrity: + """Tests for data integrity and relationships in SubmissionBundle objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for data integrity testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for data integrity tests {uuid.uuid4()}" + with open("test_file_for_data_integrity.txt", "w") as f: + f.write(file_content) + + file_entity = File( + path="test_file_for_data_integrity.txt", + name=f"test_integrity_file_{uuid.uuid4()}", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_integrity", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_submission_bundle_data_consistency( + self, test_evaluation: Evaluation, test_submission: Submission, test_file: File + ): + """Test that submission bundles maintain data consistency between submission and status.""" + # WHEN I get submission bundles for the evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should find our test submission + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None, "Test submission bundle should be found" + + # AND the submission data should be consistent + assert test_bundle.submission.id == test_submission.id + assert test_bundle.submission.entity_id == test_file.id + assert test_bundle.submission.evaluation_id == test_evaluation.id + assert test_bundle.submission.name == test_submission.name + + # AND if there's a submission status, it should reference the same entities + if test_bundle.submission_status: + assert test_bundle.submission_status.id == test_submission.id + assert test_bundle.submission_status.entity_id == test_file.id + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + async def test_submission_bundle_status_updates_reflected( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that submission status updates are reflected in bundles.""" + # GIVEN a submission status that I can update + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + original_status = submission_status.status + + # WHEN I update the submission status + submission_status.status = "VALIDATED" + submission_status.submission_annotations = { + "test_score": 95.5, + "test_feedback": "Excellent work!", + } + updated_status = submission_status.store(synapse_client=self.syn) + + # AND I get submission bundles again + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundle should reflect the updated status + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + assert test_bundle.submission_status is not None + assert test_bundle.submission_status.status == "VALIDATED" + assert test_bundle.submission_status.submission_annotations is not None + assert "test_score" in test_bundle.submission_status.submission_annotations + assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + + # CLEANUP: Reset the status back to original + submission_status.status = original_status + submission_status.submission_annotations = {} + submission_status.store(synapse_client=self.syn) + + async def test_submission_bundle_evaluation_id_propagation( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that evaluation_id is properly propagated from submission to status.""" + # WHEN I get submission bundles + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN find our test bundle + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + + # AND both submission and status should have the correct evaluation_id + assert test_bundle.submission.evaluation_id == test_evaluation.id + if test_bundle.submission_status: + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + +class TestSubmissionBundleEdgeCases: + """Tests for edge cases and error handling in SubmissionBundle operations.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for edge case testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + async def test_get_evaluation_submission_bundles_empty_evaluation( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles from an evaluation with no submissions.""" + # WHEN I get submission bundles from an evaluation with no submissions + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_user_submission_bundles_empty_evaluation( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles from an evaluation with no submissions.""" + # WHEN I get user submission bundles from an evaluation with no submissions + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_evaluation_submission_bundles_large_limit( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles with a very large limit.""" + # WHEN I request bundles with a large limit (within API bounds) + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=100, # Maximum allowed by API + synapse_client=self.syn, + ) + + # THEN it should work without error + assert bundles is not None + assert isinstance(bundles, list) + # The actual count doesn't matter since the evaluation is empty + + async def test_get_user_submission_bundles_large_offset( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles with a large offset.""" + # WHEN I request bundles with a large offset (beyond available data) + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + offset=1000, # Large offset beyond any real data + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_submission_bundles_with_default_parameters( + self, test_evaluation: Evaluation + ): + """Test that default parameters work correctly.""" + # WHEN I call methods without optional parameters + eval_bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + user_bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN both should work with default values + assert eval_bundles is not None + assert user_bundles is not None + assert isinstance(eval_bundles, list) + assert isinstance(user_bundles, list) From b5839c37f09d5091956c83e36575fa4e8ac0c73d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:29:53 -0500 Subject: [PATCH 33/60] add submissionBundle unit tests --- .../unit_test_submission_bundle_async.py | 452 +++++++++++++++++ .../unit_test_submission_bundle.py | 462 ++++++++++++++++++ 2 files changed, 914 insertions(+) create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py new file mode 100644 index 000000000..1e2df93d7 --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -0,0 +1,452 @@ +"""Unit tests for the synapseclient.models.SubmissionBundle class async methods.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus + +SUBMISSION_ID = "9999999" +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +USER_ID = "123456" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +CREATED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" + + +class TestSubmissionBundle: + """Tests for the synapseclient.models.SubmissionBundle class async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": "test_user", + "entityId": ENTITY_ID, + "versionNumber": 1, + "name": "Test Submission", + "createdOn": CREATED_ON, + "evaluationId": EVALUATION_ID, + "entityBundle": { + "entity": { + "id": ENTITY_ID, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + }, + "entityType": "org.sagebionetworks.repo.model.FileEntity", + }, + } + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "entityId": ENTITY_ID, + "versionNumber": 1, + "statusVersion": 1, + "canCancel": False, + "cancelRequested": False, + "submissionAnnotations": {"score": [85.5], "feedback": ["Good work!"]}, + } + + def get_example_submission_bundle_dict(self) -> Dict[str, Dict]: + """Return example submission bundle data from REST API.""" + return { + "submission": self.get_example_submission_dict(), + "submissionStatus": self.get_example_submission_status_dict(), + } + + def get_example_submission_bundle_minimal_dict(self) -> Dict[str, Dict]: + """Return example minimal submission bundle data from REST API.""" + return { + "submission": { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + "submissionStatus": None, + } + + def test_init_submission_bundle(self) -> None: + """Test creating a SubmissionBundle with basic attributes.""" + # GIVEN submission and submission status objects + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I create a SubmissionBundle object + bundle = SubmissionBundle( + submission=submission, + submission_status=submission_status, + ) + + # THEN the SubmissionBundle should have the expected attributes + assert bundle.submission == submission + assert bundle.submission_status == submission_status + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + + def test_init_submission_bundle_empty(self) -> None: + """Test creating an empty SubmissionBundle.""" + # WHEN I create an empty SubmissionBundle object + bundle = SubmissionBundle() + + # THEN the SubmissionBundle should have None attributes + assert bundle.submission is None + assert bundle.submission_status is None + + def test_fill_from_dict_complete(self) -> None: + """Test filling a SubmissionBundle from complete REST API response.""" + # GIVEN a complete submission bundle response + bundle_data = self.get_example_submission_bundle_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN all fields should be populated correctly + assert bundle.submission is not None + assert bundle.submission_status is not None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission.user_id == USER_ID + + # Check submission status fields + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + assert bundle.submission_status.entity_id == ENTITY_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check submission annotations + assert "score" in bundle.submission_status.submission_annotations + assert bundle.submission_status.submission_annotations["score"] == [85.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionBundle from minimal REST API response.""" + # GIVEN a minimal submission bundle response + bundle_data = self.get_example_submission_bundle_minimal_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be populated but submission_status should be None + assert bundle.submission is not None + assert bundle.submission_status is None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + + def test_fill_from_dict_no_submission(self) -> None: + """Test filling a SubmissionBundle with no submission data.""" + # GIVEN a bundle response with no submission + bundle_data = { + "submission": None, + "submissionStatus": self.get_example_submission_status_dict(), + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be None but submission_status should be populated + assert bundle.submission is None + assert bundle.submission_status is not None + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + + def test_fill_from_dict_evaluation_id_setting(self) -> None: + """Test that evaluation_id is properly set from submission to submission_status.""" + # GIVEN a bundle response where submission_status doesn't have evaluation_id + submission_dict = self.get_example_submission_dict() + status_dict = self.get_example_submission_status_dict() + # Remove evaluation_id from status_dict to simulate API response + status_dict.pop("evaluationId", None) + + bundle_data = { + "submission": submission_dict, + "submissionStatus": status_dict, + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission_status should get evaluation_id from submission + assert bundle.submission is not None + assert bundle.submission_status is not None + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID + + async def test_get_evaluation_submission_bundles_async(self) -> None: + """Test getting submission bundles for an evaluation.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "123", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + }, + }, + { + "submission": { + "id": "456", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_evaluation_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 2 + assert all(isinstance(bundle, SubmissionBundle) for bundle in result) + + # Check first bundle + assert result[0].submission is not None + assert result[0].submission.id == "123" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "123" + assert result[0].submission_status.status == "RECEIVED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check second bundle + assert result[1].submission is not None + assert result[1].submission.id == "456" + assert result[1].submission_status is not None + assert result[1].submission_status.id == "456" + assert result[1].submission_status.status == "SCORED" + + async def test_get_evaluation_submission_bundles_async_empty_response(self) -> None: + """Test getting submission bundles with empty response.""" + # GIVEN empty mock response + mock_response = {"results": []} + + # WHEN I call get_evaluation_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status=None, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be an empty list + assert len(result) == 0 + + async def test_get_user_submission_bundles_async(self) -> None: + """Test getting user submission bundles.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "789", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + "name": "User Submission 1", + }, + "submissionStatus": { + "id": "789", + "status": "VALIDATED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_user_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 1 + assert isinstance(result[0], SubmissionBundle) + + # Check bundle contents + assert result[0].submission is not None + assert result[0].submission.id == "789" + assert result[0].submission.name == "User Submission 1" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "789" + assert result[0].submission_status.status == "VALIDATED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID + + async def test_get_user_submission_bundles_async_default_params(self) -> None: + """Test getting user submission bundles with default parameters.""" + # GIVEN mock response + mock_response = {"results": []} + + # WHEN I call get_user_submission_bundles_async with defaults + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called with default parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be empty + assert len(result) == 0 + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionBundle objects with the same data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + + bundle1 = SubmissionBundle(submission=submission, submission_status=status) + bundle2 = SubmissionBundle(submission=submission, submission_status=status) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I modify one of them + bundle2.submission_status = SubmissionStatus(id="different", status="DIFFERENT") + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_dataclass_equality_with_none(self) -> None: + """Test dataclass equality with None values.""" + # GIVEN two SubmissionBundle objects with None values + bundle1 = SubmissionBundle(submission=None, submission_status=None) + bundle2 = SubmissionBundle(submission=None, submission_status=None) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I add a submission to one + bundle2.submission = Submission(id=SUBMISSION_ID) + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionBundle.""" + # GIVEN a SubmissionBundle with some data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + bundle = SubmissionBundle(submission=submission, submission_status=status) + + # WHEN I get the string representation + repr_str = repr(bundle) + str_str = str(bundle) + + # THEN it should contain relevant information + assert "SubmissionBundle" in repr_str + assert SUBMISSION_ID in repr_str + assert SUBMISSION_STATUS_ID in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str + + def test_repr_with_none_values(self) -> None: + """Test string representation with None values.""" + # GIVEN a SubmissionBundle with None values + bundle = SubmissionBundle(submission=None, submission_status=None) + + # WHEN I get the string representation + repr_str = repr(bundle) + + # THEN it should show None values + assert "SubmissionBundle" in repr_str + assert "submission=None" in repr_str + assert "submission_status=None" in repr_str \ No newline at end of file diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py new file mode 100644 index 000000000..22287c528 --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -0,0 +1,462 @@ +"""Unit tests for the synapseclient.models.SubmissionBundle class synchronous methods.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus + +SUBMISSION_ID = "9999999" +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +USER_ID = "123456" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +CREATED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" + + +class TestSubmissionBundleSync: + """Tests for the synapseclient.models.SubmissionBundle class synchronous methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": "test_user", + "entityId": ENTITY_ID, + "versionNumber": 1, + "name": "Test Submission", + "createdOn": CREATED_ON, + "evaluationId": EVALUATION_ID, + "entityBundle": { + "entity": { + "id": ENTITY_ID, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + }, + "entityType": "org.sagebionetworks.repo.model.FileEntity", + }, + } + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "entityId": ENTITY_ID, + "versionNumber": 1, + "statusVersion": 1, + "canCancel": False, + "cancelRequested": False, + "submissionAnnotations": {"score": [85.5], "feedback": ["Good work!"]}, + } + + def get_example_submission_bundle_dict(self) -> Dict[str, Dict]: + """Return example submission bundle data from REST API.""" + return { + "submission": self.get_example_submission_dict(), + "submissionStatus": self.get_example_submission_status_dict(), + } + + def get_example_submission_bundle_minimal_dict(self) -> Dict[str, Dict]: + """Return example minimal submission bundle data from REST API.""" + return { + "submission": { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + "submissionStatus": None, + } + + def test_init_submission_bundle(self) -> None: + """Test creating a SubmissionBundle with basic attributes.""" + # GIVEN submission and submission status objects + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I create a SubmissionBundle object + bundle = SubmissionBundle( + submission=submission, + submission_status=submission_status, + ) + + # THEN the SubmissionBundle should have the expected attributes + assert bundle.submission == submission + assert bundle.submission_status == submission_status + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + + def test_init_submission_bundle_empty(self) -> None: + """Test creating an empty SubmissionBundle.""" + # WHEN I create an empty SubmissionBundle object + bundle = SubmissionBundle() + + # THEN the SubmissionBundle should have None attributes + assert bundle.submission is None + assert bundle.submission_status is None + + def test_fill_from_dict_complete(self) -> None: + """Test filling a SubmissionBundle from complete REST API response.""" + # GIVEN a complete submission bundle response + bundle_data = self.get_example_submission_bundle_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN all fields should be populated correctly + assert bundle.submission is not None + assert bundle.submission_status is not None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission.user_id == USER_ID + + # Check submission status fields + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + assert bundle.submission_status.entity_id == ENTITY_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check submission annotations + assert "score" in bundle.submission_status.submission_annotations + assert bundle.submission_status.submission_annotations["score"] == [85.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionBundle from minimal REST API response.""" + # GIVEN a minimal submission bundle response + bundle_data = self.get_example_submission_bundle_minimal_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be populated but submission_status should be None + assert bundle.submission is not None + assert bundle.submission_status is None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + + def test_fill_from_dict_no_submission(self) -> None: + """Test filling a SubmissionBundle with no submission data.""" + # GIVEN a bundle response with no submission + bundle_data = { + "submission": None, + "submissionStatus": self.get_example_submission_status_dict(), + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be None but submission_status should be populated + assert bundle.submission is None + assert bundle.submission_status is not None + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + + def test_fill_from_dict_evaluation_id_setting(self) -> None: + """Test that evaluation_id is properly set from submission to submission_status.""" + # GIVEN a bundle response where submission_status doesn't have evaluation_id + submission_dict = self.get_example_submission_dict() + status_dict = self.get_example_submission_status_dict() + # Remove evaluation_id from status_dict to simulate API response + status_dict.pop("evaluationId", None) + + bundle_data = { + "submission": submission_dict, + "submissionStatus": status_dict, + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission_status should get evaluation_id from submission + assert bundle.submission is not None + assert bundle.submission_status is not None + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID + + def test_get_evaluation_submission_bundles(self) -> None: + """Test getting submission bundles for an evaluation using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "123", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + }, + }, + { + "submission": { + "id": "456", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_evaluation_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 2 + assert all(isinstance(bundle, SubmissionBundle) for bundle in result) + + # Check first bundle + assert result[0].submission is not None + assert result[0].submission.id == "123" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "123" + assert result[0].submission_status.status == "RECEIVED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check second bundle + assert result[1].submission is not None + assert result[1].submission.id == "456" + assert result[1].submission_status is not None + assert result[1].submission_status.id == "456" + assert result[1].submission_status.status == "SCORED" + + def test_get_evaluation_submission_bundles_empty_response(self) -> None: + """Test getting submission bundles with empty response using sync method.""" + # GIVEN empty mock response + mock_response = {"results": []} + + # WHEN I call get_evaluation_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status=None, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be an empty list + assert len(result) == 0 + + def test_get_user_submission_bundles(self) -> None: + """Test getting user submission bundles using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "789", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + "name": "User Submission 1", + }, + "submissionStatus": { + "id": "789", + "status": "VALIDATED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_user_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 1 + assert isinstance(result[0], SubmissionBundle) + + # Check bundle contents + assert result[0].submission is not None + assert result[0].submission.id == "789" + assert result[0].submission.name == "User Submission 1" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "789" + assert result[0].submission_status.status == "VALIDATED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID + + def test_get_user_submission_bundles_default_params(self) -> None: + """Test getting user submission bundles with default parameters using sync method.""" + # GIVEN mock response + mock_response = {"results": []} + + # WHEN I call get_user_submission_bundles with defaults (sync method) + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called with default parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be empty + assert len(result) == 0 + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionBundle objects with the same data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + + bundle1 = SubmissionBundle(submission=submission, submission_status=status) + bundle2 = SubmissionBundle(submission=submission, submission_status=status) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I modify one of them + bundle2.submission_status = SubmissionStatus(id="different", status="DIFFERENT") + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_dataclass_equality_with_none(self) -> None: + """Test dataclass equality with None values.""" + # GIVEN two SubmissionBundle objects with None values + bundle1 = SubmissionBundle(submission=None, submission_status=None) + bundle2 = SubmissionBundle(submission=None, submission_status=None) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I add a submission to one + bundle2.submission = Submission(id=SUBMISSION_ID) + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionBundle.""" + # GIVEN a SubmissionBundle with some data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + bundle = SubmissionBundle(submission=submission, submission_status=status) + + # WHEN I get the string representation + repr_str = repr(bundle) + str_str = str(bundle) + + # THEN it should contain relevant information + assert "SubmissionBundle" in repr_str + assert SUBMISSION_ID in repr_str + assert SUBMISSION_STATUS_ID in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str + + def test_repr_with_none_values(self) -> None: + """Test string representation with None values.""" + # GIVEN a SubmissionBundle with None values + bundle = SubmissionBundle(submission=None, submission_status=None) + + # WHEN I get the string representation + repr_str = repr(bundle) + + # THEN it should show None values + assert "SubmissionBundle" in repr_str + assert "submission=None" in repr_str + assert "submission_status=None" in repr_str + + def test_protocol_implementation(self) -> None: + """Test that SubmissionBundle implements the synchronous protocol correctly.""" + # THEN it should have all the required synchronous methods + assert hasattr(SubmissionBundle, "get_evaluation_submission_bundles") + assert hasattr(SubmissionBundle, "get_user_submission_bundles") + + # AND the methods should be callable + assert callable(SubmissionBundle.get_evaluation_submission_bundles) + assert callable(SubmissionBundle.get_user_submission_bundles) \ No newline at end of file From 5526b8d9e979e51edab049a42392cff65f3f7cf3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:30:40 -0500 Subject: [PATCH 34/60] remove unnecessary imports and add style --- .../async/test_submission_bundle_async.py | 45 +++++++++++--- .../async/test_submission_status_async.py | 62 ++++++++++++------- .../synchronous/test_submission_bundle.py | 49 ++++++++++----- .../synchronous/test_submission_status.py | 16 +++-- .../unit_test_submission_bundle_async.py | 28 +++++---- .../unit_test_submission_status_async.py | 12 +++- .../unit_test_submission_bundle.py | 28 +++++---- .../unit_test_submission_status.py | 16 +++-- 8 files changed, 168 insertions(+), 88 deletions(-) diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index b9162b726..eb30af938 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -7,7 +7,14 @@ from synapseclient import Synapse from synapseclient.core.exceptions import SynapseHTTPError -from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus +from synapseclient.models import ( + Evaluation, + File, + Project, + Submission, + SubmissionBundle, + SubmissionStatus, +) class TestSubmissionBundleRetrievalAsync: @@ -52,7 +59,9 @@ async def test_file( syn: Synapse, schedule_for_cleanup: Callable[..., None], ) -> File: - file_content = f"Test file content for submission bundle async tests {uuid.uuid4()}" + file_content = ( + f"Test file content for submission bundle async tests {uuid.uuid4()}" + ) with open("test_file_for_submission_bundle_async.txt", "w") as f: f.write(file_content) @@ -192,9 +201,15 @@ async def test_get_evaluation_submission_bundles_with_pagination_async( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): """Test getting submission bundles for invalid evaluation ID using async methods.""" @@ -267,9 +282,15 @@ async def test_get_user_submission_bundles_with_pagination_async( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" class TestSubmissionBundleDataIntegrityAsync: @@ -314,7 +335,9 @@ async def test_file( syn: Synapse, schedule_for_cleanup: Callable[..., None], ) -> File: - file_content = f"Test file content for data integrity async tests {uuid.uuid4()}" + file_content = ( + f"Test file content for data integrity async tests {uuid.uuid4()}" + ) with open("test_file_for_data_integrity_async.txt", "w") as f: f.write(file_content) @@ -410,7 +433,9 @@ async def test_submission_bundle_status_updates_reflected_async( assert test_bundle.submission_status.status == "VALIDATED" assert test_bundle.submission_status.submission_annotations is not None assert "test_score" in test_bundle.submission_status.submission_annotations - assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + assert test_bundle.submission_status.submission_annotations["test_score"] == [ + 95.5 + ] # CLEANUP: Reset the status back to original submission_status.status = original_status diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index b558d7f08..bf72a8158 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -221,7 +221,9 @@ async def test_store_submission_status_with_status_change( # WHEN I update the status test_submission_status.status = "VALIDATED" - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN the submission status should be updated assert updated_status.id == test_submission_status.id @@ -239,7 +241,9 @@ async def test_store_submission_status_with_submission_annotations( "score": 85.5, "feedback": "Good work!", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None @@ -256,7 +260,9 @@ async def test_store_submission_status_with_legacy_annotations( "internal_score": 92.3, "reviewer_notes": "Excellent submission", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) assert updated_status.annotations is not None converted_annotations = from_submission_status_annotations( @@ -281,7 +287,9 @@ async def test_store_submission_status_with_combined_annotations( "internal_review": True, "notes": "Needs minor improvements", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None @@ -307,7 +315,9 @@ async def test_store_submission_status_with_private_annotations_false( test_submission_status.private_status_annotations = False # WHEN I store the submission status - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN they should be properly stored assert updated_status.annotations is not None @@ -340,7 +350,9 @@ async def test_store_submission_status_with_private_annotations_true( ) # WHEN I store the submission status - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN they should be properly stored assert updated_status.annotations is not None @@ -395,7 +407,9 @@ async def test_store_submission_status_change_tracking( assert test_submission_status.has_changed # WHEN I store the changes - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN has_changed should be False again assert not updated_status.has_changed @@ -431,7 +445,9 @@ async def test_has_changed_property_edge_cases( assert test_submission_status.has_changed # WHEN I store and get a fresh copy - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) fresh_status = await SubmissionStatus(id=updated_status.id).get_async( synapse_client=self.syn ) @@ -510,7 +526,7 @@ async def test_files( finally: # Clean up the temporary file os.unlink(temp_file_path) - + return files @pytest.fixture(scope="function") @@ -611,7 +627,9 @@ async def test_batch_update_submission_statuses( # GIVEN multiple submission statuses statuses = [] for submission in test_submissions: - status = await SubmissionStatus(id=submission.id).get_async(synapse_client=self.syn) + status = await SubmissionStatus(id=submission.id).get_async( + synapse_client=self.syn + ) # Update each status status.status = "VALIDATED" status.submission_annotations = { @@ -717,33 +735,35 @@ async def test_submission( schedule_for_cleanup(created_submission.id) return created_submission - async def test_submission_cancellation_workflow( - self, test_submission: Submission - ): + async def test_submission_cancellation_workflow(self, test_submission: Submission): """Test the complete submission cancellation workflow async.""" # GIVEN a submission that exists submission_id = test_submission.id - + # WHEN I get the initial submission status - initial_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) - + initial_status = await SubmissionStatus(id=submission_id).get_async( + synapse_client=self.syn + ) + # THEN initially it should not be cancellable or cancelled assert initial_status.can_cancel is False assert initial_status.cancel_requested is False - + # WHEN I update the submission status to allow cancellation initial_status.can_cancel = True updated_status = await initial_status.store_async(synapse_client=self.syn) - + # THEN the submission should be marked as cancellable assert updated_status.can_cancel is True assert updated_status.cancel_requested is False - + # WHEN I cancel the submission await test_submission.cancel_async() - + # THEN I should be able to retrieve the updated status showing cancellation was requested - final_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + final_status = await SubmissionStatus(id=submission_id).get_async( + synapse_client=self.syn + ) assert final_status.can_cancel is True assert final_status.cancel_requested is True diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index 6e32a3cad..a9816765b 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -7,7 +7,14 @@ from synapseclient import Synapse from synapseclient.core.exceptions import SynapseHTTPError -from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus +from synapseclient.models import ( + Evaluation, + File, + Project, + Submission, + SubmissionBundle, + SubmissionStatus, +) class TestSubmissionBundleRetrieval: @@ -22,9 +29,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -192,9 +197,15 @@ async def test_get_evaluation_submission_bundles_with_pagination( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" @@ -267,9 +278,15 @@ async def test_get_user_submission_bundles_with_pagination( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" class TestSubmissionBundleDataIntegrity: @@ -284,9 +301,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -410,7 +425,9 @@ async def test_submission_bundle_status_updates_reflected( assert test_bundle.submission_status.status == "VALIDATED" assert test_bundle.submission_status.submission_annotations is not None assert "test_score" in test_bundle.submission_status.submission_annotations - assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + assert test_bundle.submission_status.submission_annotations["test_score"] == [ + 95.5 + ] # CLEANUP: Reset the status back to original submission_status.status = original_status @@ -454,9 +471,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index b720a6ba3..b4ae45372 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -713,31 +713,29 @@ async def test_submission( schedule_for_cleanup(created_submission.id) return created_submission - async def test_submission_cancellation_workflow( - self, test_submission: Submission - ): + async def test_submission_cancellation_workflow(self, test_submission: Submission): """Test the complete submission cancellation workflow.""" # GIVEN a submission that exists submission_id = test_submission.id - + # WHEN I get the initial submission status initial_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) - + # THEN initially it should not be cancellable or cancelled assert initial_status.can_cancel is False assert initial_status.cancel_requested is False - + # WHEN I update the submission status to allow cancellation initial_status.can_cancel = True updated_status = initial_status.store(synapse_client=self.syn) - + # THEN the submission should be marked as cancellable assert updated_status.can_cancel is True assert updated_status.cancel_requested is False - + # WHEN I cancel the submission test_submission.cancel() - + # THEN I should be able to retrieve the updated status showing cancellation was requested final_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) assert final_status.can_cancel is True diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py index 1e2df93d7..1e2efca02 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -47,7 +47,9 @@ def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: }, } - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -126,7 +128,7 @@ def test_fill_from_dict_complete(self) -> None: # THEN all fields should be populated correctly assert bundle.submission is not None assert bundle.submission_status is not None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -137,7 +139,9 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + assert ( + bundle.submission_status.evaluation_id == EVALUATION_ID + ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -154,7 +158,7 @@ def test_fill_from_dict_minimal(self) -> None: # THEN submission should be populated but submission_status should be None assert bundle.submission is not None assert bundle.submission_status is None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -184,7 +188,7 @@ def test_fill_from_dict_evaluation_id_setting(self) -> None: status_dict = self.get_example_submission_status_dict() # Remove evaluation_id from status_dict to simulate API response status_dict.pop("evaluationId", None) - + bundle_data = { "submission": submission_dict, "submissionStatus": status_dict, @@ -259,15 +263,17 @@ async def test_get_evaluation_submission_bundles_async(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 2 assert all(isinstance(bundle, SubmissionBundle) for bundle in result) - + # Check first bundle assert result[0].submission is not None assert result[0].submission.id == "123" assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission - + assert ( + result[0].submission_status.evaluation_id == EVALUATION_ID + ) # set from submission + # Check second bundle assert result[1].submission is not None assert result[1].submission.id == "456" @@ -349,7 +355,7 @@ async def test_get_user_submission_bundles_async(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 1 assert isinstance(result[0], SubmissionBundle) - + # Check bundle contents assert result[0].submission is not None assert result[0].submission.id == "789" @@ -391,7 +397,7 @@ def test_dataclass_equality(self) -> None: # GIVEN two SubmissionBundle objects with the same data submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) - + bundle1 = SubmissionBundle(submission=submission, submission_status=status) bundle2 = SubmissionBundle(submission=submission, submission_status=status) @@ -449,4 +455,4 @@ def test_repr_with_none_values(self) -> None: # THEN it should show None values assert "SubmissionBundle" in repr_str assert "submission=None" in repr_str - assert "submission_status=None" in repr_str \ No newline at end of file + assert "submission_status=None" in repr_str diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py index 3ca7d906d..c48ad2e1e 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py @@ -30,7 +30,9 @@ class TestSubmissionStatus: def init_syn(self, syn: Synapse) -> None: self.syn = syn - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -181,7 +183,9 @@ async def test_get_async_without_id(self) -> None: # WHEN I call get_async # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): await submission_status.get_async(synapse_client=self.syn) async def test_store_async(self) -> None: @@ -520,7 +524,9 @@ def test_set_last_persistent_instance(self) -> None: assert submission_status._last_persistent_instance is not None assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID assert submission_status._last_persistent_instance.status == STATUS - assert submission_status._last_persistent_instance.annotations == {"test": "value"} + assert submission_status._last_persistent_instance.annotations == { + "test": "value" + } # AND modifying the current instance shouldn't affect the persistent one submission_status.status = "MODIFIED" diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index 22287c528..9dc252286 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -47,7 +47,9 @@ def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: }, } - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -126,7 +128,7 @@ def test_fill_from_dict_complete(self) -> None: # THEN all fields should be populated correctly assert bundle.submission is not None assert bundle.submission_status is not None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -137,7 +139,9 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + assert ( + bundle.submission_status.evaluation_id == EVALUATION_ID + ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -154,7 +158,7 @@ def test_fill_from_dict_minimal(self) -> None: # THEN submission should be populated but submission_status should be None assert bundle.submission is not None assert bundle.submission_status is None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -184,7 +188,7 @@ def test_fill_from_dict_evaluation_id_setting(self) -> None: status_dict = self.get_example_submission_status_dict() # Remove evaluation_id from status_dict to simulate API response status_dict.pop("evaluationId", None) - + bundle_data = { "submission": submission_dict, "submissionStatus": status_dict, @@ -259,15 +263,17 @@ def test_get_evaluation_submission_bundles(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 2 assert all(isinstance(bundle, SubmissionBundle) for bundle in result) - + # Check first bundle assert result[0].submission is not None assert result[0].submission.id == "123" assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission - + assert ( + result[0].submission_status.evaluation_id == EVALUATION_ID + ) # set from submission + # Check second bundle assert result[1].submission is not None assert result[1].submission.id == "456" @@ -349,7 +355,7 @@ def test_get_user_submission_bundles(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 1 assert isinstance(result[0], SubmissionBundle) - + # Check bundle contents assert result[0].submission is not None assert result[0].submission.id == "789" @@ -391,7 +397,7 @@ def test_dataclass_equality(self) -> None: # GIVEN two SubmissionBundle objects with the same data submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) - + bundle1 = SubmissionBundle(submission=submission, submission_status=status) bundle2 = SubmissionBundle(submission=submission, submission_status=status) @@ -459,4 +465,4 @@ def test_protocol_implementation(self) -> None: # AND the methods should be callable assert callable(SubmissionBundle.get_evaluation_submission_bundles) - assert callable(SubmissionBundle.get_user_submission_bundles) \ No newline at end of file + assert callable(SubmissionBundle.get_user_submission_bundles) diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py index 76b00641f..e724e64f0 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -1,13 +1,11 @@ """Unit tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" -import uuid -from typing import Dict, List, Union +from typing import Dict, Union from unittest.mock import AsyncMock, patch import pytest from synapseclient import Synapse -from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.models import SubmissionStatus SUBMISSION_STATUS_ID = "9999999" @@ -32,7 +30,9 @@ class TestSubmissionStatusSync: def init_syn(self, syn: Synapse) -> None: self.syn = syn - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -182,7 +182,9 @@ def test_get_without_id(self) -> None: # WHEN I call get # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): submission_status.get(synapse_client=self.syn) def test_store(self) -> None: @@ -520,7 +522,9 @@ def test_set_last_persistent_instance(self) -> None: assert submission_status._last_persistent_instance is not None assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID assert submission_status._last_persistent_instance.status == STATUS - assert submission_status._last_persistent_instance.annotations == {"test": "value"} + assert submission_status._last_persistent_instance.annotations == { + "test": "value" + } # AND modifying the current instance shouldn't affect the persistent one submission_status.status = "MODIFIED" From bdeaaa70636b786eb06143247d71b83d1a41da9e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 11:15:02 -0500 Subject: [PATCH 35/60] get_evaluation_submissions returns generator object --- synapseclient/api/evaluation_services.py | 28 +- synapseclient/models/submission.py | 255 ++++++++++++++++-- .../models/async/test_submission_async.py | 58 ++-- .../models/synchronous/test_submission.py | 50 ++-- .../async/unit_test_submission_async.py | 40 ++- .../synchronous/unit_test_submission.py | 42 ++- 6 files changed, 333 insertions(+), 140 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 478976137..19184e562 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -4,7 +4,9 @@ """ import json -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional + +from synapseclient.api.api_client import rest_get_paginated_async if TYPE_CHECKING: from synapseclient import Synapse @@ -448,12 +450,11 @@ async def get_submission( async def get_evaluation_submissions( evaluation_id: str, status: Optional[str] = None, - limit: int = 20, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Retrieves all Submissions for a specified Evaluation queue. + Generator to get all Submissions for a specified Evaluation queue. @@ -461,29 +462,26 @@ async def get_evaluation_submissions( evaluation_id: The ID of the evaluation queue. status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - # TODO: Support pagination in the return type. - A response JSON containing a paginated list of submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = {"limit": limit, "offset": offset} + query_params = {} if status: query_params["status"] = status - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_user_submissions( diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 7096201c9..2e0860cb6 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -1,17 +1,55 @@ from dataclasses import dataclass, field -from typing import Dict, List, Optional, Protocol, Union +from typing import AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, otel_trace_method, wrap_async_generator_to_sync_generator from synapseclient.models.mixins.access_control import AccessControllable class SubmissionSynchronousProtocol(Protocol): """Protocol defining the synchronous interface for Submission operations.""" + def store( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Store the submission in Synapse. This creates a new submission in an evaluation queue. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission object with the ID set. + + Raises: + ValueError: If the submission is missing required fields, or if unable to fetch entity etag. + + Example: Creating a submission +   + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ).store() + print(submission.id) + ``` + """ + return self + def get( self, *, @@ -21,9 +59,6 @@ def get( Retrieve a Submission from Synapse. Arguments: - include_activity: Whether to include the activity in the returned submission. - Defaults to False. Setting this to True will include the activity - record associated with this submission. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -31,9 +66,11 @@ def get( Returns: The Submission instance retrieved from Synapse. - Example: Retrieving a submission by ID. -   + Raises: + ValueError: If the submission does not have an ID to get. + Example: Retrieving a submission by ID +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -56,9 +93,11 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Example: Delete a submission. -   + Raises: + ValueError: If the submission does not have an ID to delete. + Example: Delete a submission +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -73,6 +112,171 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: """ pass + def cancel( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Cancel a Submission. Only the user who created the Submission may cancel it. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated Submission object. + + Raises: + ValueError: If the submission does not have an ID to cancel. + + Example: Cancel a submission +   + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + canceled_submission = submission.cancel() + ``` + """ + return self + + @classmethod + def get_evaluation_submissions( + cls, + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Generator["Submission", None, None]: + """ + Retrieves all Submissions for a specified Evaluation queue. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status. + Submission status can be one of + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Submission objects as they are retrieved from the API. + + Example: Getting submissions for an evaluation +   + Get SCORED submissions from a specific evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submissions = list(Submission.get_evaluation_submissions( + evaluation_id="9999999", + status="SCORED" + )) + print(f"Found {len(submissions)} submissions") + ``` + """ + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_evaluation_submissions_async, + evaluation_id=evaluation_id, + status=status, + synapse_client=synapse_client, + ) + + @staticmethod + def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + + Example: Getting user submissions + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = Submission.get_user_submissions( + evaluation_id="9999999", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + ``` + """ + return {} + + @staticmethod + def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + + Example: Getting submission count +   + Get the total number of SCORED submissions from a specific evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = Submission.get_submission_count( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {response} submissions") + ``` + """ + return {} + @dataclass @async_to_sync @@ -448,31 +652,28 @@ async def get_submission_example(): return self # TODO: Have all staticmethods return generators for pagination - @staticmethod + @skip_async_to_sync + @classmethod async def get_evaluation_submissions_async( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> AsyncGenerator["Submission", None]: """ - Retrieves all Submissions for a specified Evaluation queue. + Generator to get all Submissions for a specified Evaluation queue. Arguments: evaluation_id: The ID of the evaluation queue. status: Optionally filter submissions by a submission status. Submission status can be one of - limit: Limits the number of submissions in a single response. Defaults to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Defaults to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. Example: Getting submissions for an evaluation   @@ -486,22 +687,24 @@ async def get_evaluation_submissions_async( syn.login() async def get_evaluation_submissions_example(): - response = await Submission.get_evaluation_submissions_async( + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id="9999999", status="SCORED" - ) - print(f"Found {len(response['results'])} submissions") + ): + submissions.append(submission) + print(f"Found {len(submissions)} submissions") asyncio.run(get_evaluation_submissions_example()) ``` """ - return await evaluation_services.get_evaluation_submissions( + async for submission_data in evaluation_services.get_evaluation_submissions( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) + ): + submission_object = cls().fill_from_dict(synapse_submission=submission_data) + yield submission_object @staticmethod async def get_user_submissions_async( diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 745a60960..622e4dce3 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -245,52 +245,54 @@ async def test_get_submission_by_id_async( async def test_get_evaluation_submissions_async( self, test_evaluation: Evaluation, test_submission: Submission ): - # WHEN I get all submissions for an evaluation using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get all submissions for an evaluation using async generator + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + ): + submissions.append(submission) - # THEN I should get a response with submissions - assert "results" in response - assert len(response["results"]) > 0 + # THEN I should get a list of submission objects + assert len(submissions) > 0 + assert all(isinstance(sub, Submission) for sub in submissions) - # AND the submission should be in the results - submission_ids = [sub.get("id") for sub in response["results"]] + # AND the test submission should be in the results + submission_ids = [sub.id for sub in submissions] assert test_submission.id in submission_ids async def test_get_evaluation_submissions_with_status_filter_async( self, test_evaluation: Evaluation, test_submission: Submission ): - # WHEN I get submissions filtered by status using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get submissions filtered by status using async generator + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN I should get submissions with the specified status - assert "results" in response - for submission in response["results"]: - if submission.get("id") == test_submission.id: - # The submission should be in RECEIVED status initially - break - else: - pytest.fail("Test submission not found in filtered results") - - async def test_get_evaluation_submissions_with_pagination_async( + # The test submission should be in the results (initially in RECEIVED status) + submission_ids = [sub.id for sub in submissions] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_async_generator_behavior( self, test_evaluation: Evaluation ): - # WHEN I get submissions with pagination parameters using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get submissions using the async generator + submissions_generator = Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, - limit=5, - offset=0, synapse_client=self.syn, ) - # THEN the response should respect pagination - assert "results" in response - assert len(response["results"]) <= 5 + # THEN I should be able to iterate through the results + submissions = [] + async for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions_async(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user using async method diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index e6943a006..a16109b07 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -239,51 +239,49 @@ async def test_get_evaluation_submissions( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get all submissions for an evaluation - response = Submission.get_evaluation_submissions( + submissions = list(Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + )) - # THEN I should get a response with submissions - assert "results" in response - assert len(response["results"]) > 0 + # THEN I should get a list of submission objects + assert len(submissions) > 0 + assert all(isinstance(sub, Submission) for sub in submissions) - # AND the submission should be in the results - submission_ids = [sub.get("id") for sub in response["results"]] + # AND the test submission should be in the results + submission_ids = [sub.id for sub in submissions] assert test_submission.id in submission_ids async def test_get_evaluation_submissions_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get submissions filtered by status - response = Submission.get_evaluation_submissions( + submissions = list(Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + )) - # THEN I should get submissions with the specified status - assert "results" in response - for submission in response["results"]: - if submission.get("id") == test_submission.id: - # The submission should be in RECEIVED status initially - break - else: - pytest.fail("Test submission not found in filtered results") - - async def test_get_evaluation_submissions_with_pagination( + # The test submission should be in the results (initially in RECEIVED status) + submission_ids = [sub.id for sub in submissions] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_generator_behavior( self, test_evaluation: Evaluation ): - # WHEN I get submissions with pagination parameters - response = Submission.get_evaluation_submissions( + # WHEN I get submissions using the generator + submissions_generator = Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, - limit=5, - offset=0, synapse_client=self.syn, ) - # THEN the response should respect pagination - assert "results" in response - assert len(response["results"]) <= 5 + # THEN I should be able to iterate through the results + submissions = [] + for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index 8a0ca25ac..0d33095f2 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -471,48 +471,44 @@ async def test_cancel_async_success(self) -> None: synapse_client=self.syn, ) self.syn.logger.info.assert_called_once_with( - f"Submission {SUBMISSION_ID} has successfully been cancelled." + f"A request to cancel Submission {SUBMISSION_ID} has been submitted." ) - assert cancelled_submission.id == SUBMISSION_ID - assert cancelled_submission.entity_id == ENTITY_ID - assert cancelled_submission.evaluation_id == EVALUATION_ID @pytest.mark.asyncio async def test_get_evaluation_submissions_async(self) -> None: # GIVEN evaluation parameters evaluation_id = EVALUATION_ID status = "SCORED" - limit = 10 - offset = 5 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_evaluation_submissions_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_evaluation_submissions" ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_submissions.assert_called_once_with( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_user_submissions_async(self) -> None: diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 038045f11..31f00eb0f 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -614,7 +614,7 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async( + await submission.cancel_async( synapse_client=self.syn ) @@ -624,11 +624,8 @@ async def test_cancel_async_success(self) -> None: synapse_client=self.syn, ) self.syn.logger.info.assert_called_once_with( - f"Submission {SUBMISSION_ID} has successfully been cancelled." + f"A request to cancel Submission {SUBMISSION_ID} has been submitted." ) - assert cancelled_submission.id == SUBMISSION_ID - assert cancelled_submission.entity_id == ENTITY_ID - assert cancelled_submission.evaluation_id == EVALUATION_ID @pytest.mark.asyncio async def test_cancel_async_without_id(self) -> None: @@ -645,37 +642,36 @@ async def test_get_evaluation_submissions_async(self) -> None: # GIVEN evaluation parameters evaluation_id = EVALUATION_ID status = "SCORED" - limit = 10 - offset = 5 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_evaluation_submissions_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_evaluation_submissions" ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_submissions.assert_called_once_with( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_user_submissions_async(self) -> None: From 7add1574f078b4fe601ad14bb5c5a1f7b948024e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 12:01:55 -0500 Subject: [PATCH 36/60] get_user_submissions returns generator object --- synapseclient/api/evaluation_services.py | 23 +++---- synapseclient/models/submission.py | 66 +++++++++---------- .../models/async/test_submission_async.py | 30 +++++++-- .../models/synchronous/test_submission.py | 25 ++++++- .../async/unit_test_submission_async.py | 35 +++++----- .../synchronous/unit_test_submission.py | 35 +++++----- 6 files changed, 123 insertions(+), 91 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 19184e562..cb5784808 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -487,12 +487,11 @@ async def get_evaluation_submissions( async def get_user_submissions( evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Generator to get all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. @@ -501,28 +500,26 @@ async def get_user_submissions( evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission" - query_params = {"limit": limit, "offset": offset} + query_params = {} if user_id: query_params["userId"] = user_id - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_submission_count( diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 2e0860cb6..5ab8f3c64 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -192,32 +192,28 @@ def get_evaluation_submissions( synapse_client=synapse_client, ) - @staticmethod + @classmethod def get_user_submissions( + cls, evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> Generator["Submission", None, None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Retrieves all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. Arguments: evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Submission objects as they are retrieved from the API. Example: Getting user submissions ```python @@ -227,15 +223,19 @@ def get_user_submissions( syn = Synapse() syn.login() - response = Submission.get_user_submissions( + submissions = list(Submission.get_user_submissions( evaluation_id="9999999", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + user_id="123456" + )) + print(f"Found {len(submissions)} user submissions") ``` """ - return {} + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_user_submissions_async, + evaluation_id=evaluation_id, + user_id=user_id, + synapse_client=synapse_client, + ) @staticmethod def get_submission_count( @@ -706,32 +706,29 @@ async def get_evaluation_submissions_example(): submission_object = cls().fill_from_dict(synapse_submission=submission_data) yield submission_object - @staticmethod + @skip_async_to_sync + @classmethod async def get_user_submissions_async( + cls, evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> AsyncGenerator["Submission", None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Generator to get all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. Arguments: evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. Example: Getting user submissions ```python @@ -743,23 +740,24 @@ async def get_user_submissions_async( syn.login() async def get_user_submissions_example(): - response = await Submission.get_user_submissions_async( + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id="9999999", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + user_id="123456" + ): + submissions.append(submission) + print(f"Found {len(submissions)} user submissions") asyncio.run(get_user_submissions_example()) ``` """ - return await evaluation_services.get_user_submissions( + async for submission_data in evaluation_services.get_user_submissions( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) + ): + submission_object = cls().fill_from_dict(synapse_submission=submission_data) + yield submission_object @staticmethod async def get_submission_count_async( diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 622e4dce3..5ef52ed4e 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -295,14 +295,34 @@ async def test_get_evaluation_submissions_async_generator_behavior( assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions_async(self, test_evaluation: Evaluation): - # WHEN I get submissions for the current user using async method - response = await Submission.get_user_submissions_async( + # WHEN I get submissions for the current user using async generator + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + ): + submissions.append(submission) - # THEN I should get a response with user submissions - assert "results" in response + # THEN all submissions should be valid Submission objects # Note: Could be empty if user hasn't made submissions to this evaluation + assert all(isinstance(sub, Submission) for sub in submissions) + + async def test_get_user_submissions_async_generator_behavior( + self, test_evaluation: Evaluation + ): + # WHEN I get user submissions using the async generator + submissions_generator = Submission.get_user_submissions_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + submissions = [] + async for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_submission_count_async(self, test_evaluation: Evaluation): # WHEN I get the submission count for an evaluation using async method diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index a16109b07..16fa52b1a 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -285,13 +285,32 @@ async def test_get_evaluation_submissions_generator_behavior( async def test_get_user_submissions(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user - response = Submission.get_user_submissions( + submissions_generator = Submission.get_user_submissions( evaluation_id=test_evaluation.id, synapse_client=self.syn ) - # THEN I should get a response with user submissions - assert "results" in response + # THEN I should get a generator that yields Submission objects + submissions = list(submissions_generator) # Note: Could be empty if user hasn't made submissions to this evaluation + assert all(isinstance(sub, Submission) for sub in submissions) + + async def test_get_user_submissions_generator_behavior( + self, test_evaluation: Evaluation + ): + # WHEN I get user submissions using the generator + submissions_generator = Submission.get_user_submissions( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + submissions = [] + for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_submission_count(self, test_evaluation: Evaluation): # WHEN I get the submission count for an evaluation diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index 0d33095f2..a18f22fe8 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -515,37 +515,36 @@ async def test_get_user_submissions_async(self) -> None: # GIVEN user submission parameters evaluation_id = EVALUATION_ID user_id = USER_ID - limit = 15 - offset = 0 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_user_submissions_async with patch( - "synapseclient.api.evaluation_services.get_user_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_user_submissions" ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_user_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_user_submissions.assert_called_once_with( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_submission_count_async(self) -> None: diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 31f00eb0f..c1d2eb2c0 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -678,37 +678,36 @@ async def test_get_user_submissions_async(self) -> None: # GIVEN user submission parameters evaluation_id = EVALUATION_ID user_id = USER_ID - limit = 15 - offset = 0 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_user_submissions_async with patch( - "synapseclient.api.evaluation_services.get_user_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_user_submissions" ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_user_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_user_submissions.assert_called_once_with( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_submission_count_async(self) -> None: From 35696eba60879dfa725fc621be4527916fd7d6bb Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 13:08:35 -0500 Subject: [PATCH 37/60] submissionBundle methods return generators --- synapseclient/api/evaluation_services.py | 50 ++-- synapseclient/models/submission.py | 1 - synapseclient/models/submission_bundle.py | 176 ++++++------ .../async/test_submission_bundle_async.py | 250 ++++++++++-------- .../synchronous/test_submission_bundle.py | 198 +++++++------- .../unit_test_submission_bundle_async.py | 84 +++--- .../unit_test_submission_bundle.py | 76 +++--- 7 files changed, 434 insertions(+), 401 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index cb5784808..80fee69a8 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -783,29 +783,23 @@ async def batch_update_submission_statuses( async def get_evaluation_submission_bundles( evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Generator to get all bundled Submissions and SubmissionStatuses to a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -815,48 +809,44 @@ async def get_evaluation_submission_bundles( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/bundle/all" - query_params = {"limit": limit, "offset": offset} + query_params = {} if status: query_params["status"] = status - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_user_submission_bundles( evaluation_id: str, - limit: int = 10, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Generator to get all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of the requesting user's submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/bundle" - query_params = {"limit": limit, "offset": offset} - - response = await client.rest_get_async(uri, params=query_params) + query_params = {} - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 5ab8f3c64..08dc642ee 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -651,7 +651,6 @@ async def get_submission_example(): return self - # TODO: Have all staticmethods return generators for pagination @skip_async_to_sync @classmethod async def get_evaluation_submissions_async( diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 6c7ae42e4..75dde8610 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,9 +1,9 @@ from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union +from typing import TYPE_CHECKING, AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, wrap_async_generator_to_sync_generator if TYPE_CHECKING: from synapseclient.models.submission import Submission @@ -13,32 +13,26 @@ class SubmissionBundleSynchronousProtocol(Protocol): """Protocol defining the synchronous interface for SubmissionBundle operations.""" - @staticmethod + @classmethod def get_evaluation_submission_bundles( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> Generator["SubmissionBundle", None, None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Retrieves bundled Submissions and SubmissionStatuses for a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the submission bundles - for the evaluation queue. + SubmissionBundle objects as they are retrieved from the API. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -52,42 +46,40 @@ def get_evaluation_submission_bundles( syn = Synapse() syn.login() - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id="9614543", - status="SCORED", - limit=50 - ) + status="SCORED" + )) print(f"Found {len(bundles)} submission bundles") for bundle in bundles: print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - return [] + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_evaluation_submission_bundles_async, + evaluation_id=evaluation_id, + status=status, + synapse_client=synapse_client, + ) - @staticmethod + @classmethod def get_user_submission_bundles( + cls, evaluation_id: str, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> Generator["SubmissionBundle", None, None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Retrieves all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's - submission bundles for the evaluation queue. + SubmissionBundle objects as they are retrieved from the API. Example: Getting user submission bundles   @@ -98,16 +90,19 @@ def get_user_submission_bundles( syn = Synapse() syn.login() - bundles = SubmissionBundle.get_user_submission_bundles( - evaluation_id="9999999", - limit=25 - ) + bundles = list(SubmissionBundle.get_user_submission_bundles( + evaluation_id="9999999" + )) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: print(f"Submission ID: {bundle.submission.id}") ``` """ - return [] + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_user_submission_bundles_async, + evaluation_id=evaluation_id, + synapse_client=synapse_client, + ) @dataclass @@ -217,32 +212,27 @@ def fill_from_dict( return self - @staticmethod + @skip_async_to_sync + @classmethod async def get_evaluation_submission_bundles_async( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> AsyncGenerator["SubmissionBundle", None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Generator to get all bundled Submissions and SubmissionStatuses for a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A list of SubmissionBundle objects containing the submission bundles - for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -250,94 +240,84 @@ async def get_evaluation_submission_bundles_async( Example: Getting submission bundles for an evaluation   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import SubmissionBundle syn = Synapse() syn.login() - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id="9999999", - status="SCORED", - limit=50 - ) - print(f"Found {len(bundles)} submission bundles") - for bundle in bundles: - print(f"Submission ID: {bundle.submission.id}") + async def get_submission_bundles_example(): + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="9999999", + status="SCORED" + ): + bundles.append(bundle) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id}") + + asyncio.run(get_submission_bundles_example()) ``` """ - response = await evaluation_services.get_evaluation_submission_bundles( + async for bundle_data in evaluation_services.get_evaluation_submission_bundles( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) - - bundles = [] - for bundle_dict in response.get("results", []): - bundle = SubmissionBundle() - bundle.fill_from_dict(bundle_dict) - bundles.append(bundle) + ): + bundle = cls() + bundle.fill_from_dict(bundle_data) + yield bundle - return bundles - - @staticmethod + @skip_async_to_sync + @classmethod async def get_user_submission_bundles_async( + cls, evaluation_id: str, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> AsyncGenerator["SubmissionBundle", None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Generator to get all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A list of SubmissionBundle objects containing the requesting user's - submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Example: Getting user submission bundles   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import SubmissionBundle syn = Synapse() syn.login() - bundles = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id="9999999", - limit=25 - ) - print(f"Found {len(bundles)} user submission bundles") - for bundle in bundles: - print(f"Submission ID: {bundle.submission.id}") + async def get_user_submission_bundles_example(): + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( + evaluation_id="9999999" + ): + bundles.append(bundle) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id}") + + asyncio.run(get_user_submission_bundles_example()) ``` """ - response = await evaluation_services.get_user_submission_bundles( + async for bundle_data in evaluation_services.get_user_submission_bundles( evaluation_id=evaluation_id, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) - - # Convert response to list of SubmissionBundle objects - bundles = [] - for bundle_dict in response.get("results", []): - bundle = SubmissionBundle() - bundle.fill_from_dict(bundle_dict) - bundles.append(bundle) - - return bundles + ): + bundle = cls() + bundle.fill_from_dict(bundle_data) + yield bundle diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index eb30af938..5f4273826 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -115,14 +115,15 @@ async def test_get_evaluation_submission_bundles_basic_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles for an evaluation using async methods.""" - # WHEN I get submission bundles for an evaluation - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles for an evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved - assert bundles is not None assert len(bundles) >= 1 # At least our test submission # AND each bundle should have proper structure @@ -133,10 +134,28 @@ async def test_get_evaluation_submission_bundles_basic_async( assert bundle.submission.id is not None assert bundle.submission.evaluation_id == test_evaluation.id - if bundle.submission.id == test_submission.id: - found_test_bundle = True - assert bundle.submission.entity_id == test_submission.entity_id - assert bundle.submission.name == test_submission.name + async def test_get_evaluation_submission_bundles_async_generator_behavior( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submission bundles using the async generator + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + bundles = [] + async for bundle in bundles_generator: + assert isinstance(bundle, SubmissionBundle) + bundles.append(bundle) + + # AND all bundles should be valid SubmissionBundle objects + assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name # AND our test submission should be found assert found_test_bundle, "Test submission should be found in bundles" @@ -146,11 +165,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter_async( ): """Test getting submission bundles filtered by status using async methods.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved assert bundles is not None @@ -162,63 +183,59 @@ async def test_get_evaluation_submission_bundles_with_status_filter_async( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, status="NONEXISTENT_STATUS", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) assert "NONEXISTENT_STATUS" in str(exc_info.value) - async def test_get_evaluation_submission_bundles_with_pagination_async( + async def test_get_evaluation_submission_bundles_with_automatic_pagination_async( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting submission bundles using async methods.""" - # WHEN I get submission bundles with a limit - bundles_page1 = await SubmissionBundle.get_evaluation_submission_bundles_async( + """Test automatic pagination when getting submission bundles using async generator methods.""" + # WHEN I get submission bundles using async generator (handles pagination automatically) + all_bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + ): + all_bundles.append(bundle) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all bundles for the evaluation + assert all_bundles is not None + assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions - # WHEN I get the next page - bundles_page2 = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id=test_evaluation.id, - limit=2, - offset=2, - synapse_client=self.syn, - ) + # AND each bundle should be valid + for bundle in all_bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.evaluation_id == test_evaluation.id - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + # AND all our test submissions should be found + found_submission_ids = { + bundle.submission.id for bundle in all_bundles if bundle.submission + } + test_submission_ids = {submission.id for submission in multiple_submissions} + assert test_submission_ids.issubset( + found_submission_ids + ), "All test submissions should be found in the results" async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): """Test getting submission bundles for invalid evaluation ID using async methods.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id="syn999999999999", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -227,11 +244,13 @@ async def test_get_user_submission_bundles_basic_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting user submission bundles for an evaluation using async methods.""" - # WHEN I get user submission bundles for an evaluation - bundles = await SubmissionBundle.get_user_submission_bundles_async( + # WHEN I get user submission bundles for an evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved assert bundles is not None @@ -253,44 +272,36 @@ async def test_get_user_submission_bundles_basic_async( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in user bundles" - async def test_get_user_submission_bundles_with_pagination_async( + async def test_get_user_submission_bundles_with_automatic_pagination_async( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting user submission bundles using async methods.""" - # WHEN I get user submission bundles with a limit - bundles_page1 = await SubmissionBundle.get_user_submission_bundles_async( + """Test automatic pagination when getting user submission bundles using async generator methods.""" + # WHEN I get user submission bundles using async generator (handles pagination automatically) + all_bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + ): + all_bundles.append(bundle) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all bundles for the user in this evaluation + assert all_bundles is not None + assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions - # WHEN I get the next page - bundles_page2 = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id=test_evaluation.id, - limit=2, - offset=2, - synapse_client=self.syn, - ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None + # AND each bundle should be valid + for bundle in all_bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.evaluation_id == test_evaluation.id - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + # AND all our test submissions should be found + found_submission_ids = { + bundle.submission.id for bundle in all_bundles if bundle.submission + } + test_submission_ids = {submission.id for submission in multiple_submissions} + assert test_submission_ids.issubset( + found_submission_ids + ), "All test submissions should be found in the user results" class TestSubmissionBundleDataIntegrityAsync: @@ -370,11 +381,13 @@ async def test_submission_bundle_data_consistency_async( self, test_evaluation: Evaluation, test_submission: Submission, test_file: File ): """Test that submission bundles maintain data consistency between submission and status using async methods.""" - # WHEN I get submission bundles for the evaluation - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles for the evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN I should find our test submission test_bundle = None @@ -415,11 +428,13 @@ async def test_submission_bundle_status_updates_reflected_async( } updated_status = await submission_status.store_async(synapse_client=self.syn) - # AND I get submission bundles again - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # AND I get submission bundles again using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundle should reflect the updated status test_bundle = None @@ -446,11 +461,13 @@ async def test_submission_bundle_evaluation_id_propagation_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test that evaluation_id is properly propagated from submission to status using async methods.""" - # WHEN I get submission bundles - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN find our test bundle test_bundle = None @@ -542,11 +559,13 @@ async def test_get_evaluation_submission_bundles_empty_evaluation_async( self, test_evaluation: Evaluation ): """Test getting submission bundles from an evaluation with no submissions using async methods.""" - # WHEN I get submission bundles from an evaluation with no submissions - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles from an evaluation with no submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -557,43 +576,47 @@ async def test_get_user_submission_bundles_empty_evaluation_async( self, test_evaluation: Evaluation ): """Test getting user submission bundles from an evaluation with no submissions using async methods.""" - # WHEN I get user submission bundles from an evaluation with no submissions - bundles = await SubmissionBundle.get_user_submission_bundles_async( + # WHEN I get user submission bundles from an evaluation with no submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not None or error) assert bundles is not None assert isinstance(bundles, list) assert len(bundles) == 0 - async def test_get_evaluation_submission_bundles_large_limit_async( + async def test_get_evaluation_submission_bundles_all_results_async( self, test_evaluation: Evaluation ): - """Test getting submission bundles with a very large limit using async methods.""" - # WHEN I request bundles with a large limit (within API bounds) - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + """Test getting all submission bundles using async generator methods.""" + # WHEN I request all bundles using async generator (no limit needed) + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=100, # Maximum allowed by API synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should work without error assert bundles is not None assert isinstance(bundles, list) # The actual count doesn't matter since the evaluation is empty - async def test_get_user_submission_bundles_large_offset_async( + async def test_get_user_submission_bundles_empty_results_async( self, test_evaluation: Evaluation ): - """Test getting user submission bundles with a large offset using async methods.""" - # WHEN I request bundles with a large offset (beyond available data) - bundles = await SubmissionBundle.get_user_submission_bundles_async( + """Test getting user submission bundles when no results exist using async generator methods.""" + # WHEN I request bundles from an evaluation with no user submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, - offset=1000, # Large offset beyond any real data synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not error) assert bundles is not None @@ -604,15 +627,20 @@ async def test_get_submission_bundles_with_default_parameters_async( self, test_evaluation: Evaluation ): """Test that default parameters work correctly using async methods.""" - # WHEN I call methods without optional parameters - eval_bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I call methods without optional parameters using async generators + eval_bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) - user_bundles = await SubmissionBundle.get_user_submission_bundles_async( + ): + eval_bundles.append(bundle) + + user_bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + user_bundles.append(bundle) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index a9816765b..cf62e4e22 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -111,14 +111,13 @@ async def test_get_evaluation_submission_bundles_basic( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles for an evaluation.""" - # WHEN I get submission bundles for an evaluation - bundles = SubmissionBundle.get_evaluation_submission_bundles( + # WHEN I get submission bundles for an evaluation using generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) - # THEN the bundles should be retrieved - assert bundles is not None + # THEN I should get at least our test submission assert len(bundles) >= 1 # At least our test submission # AND each bundle should have proper structure @@ -137,16 +136,35 @@ async def test_get_evaluation_submission_bundles_basic( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in bundles" + async def test_get_evaluation_submission_bundles_generator_behavior( + self, test_evaluation: Evaluation + ): + """Test that the generator returns SubmissionBundle objects correctly.""" + # WHEN I get submission bundles using the generator + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + bundles = [] + for bundle in bundles_generator: + assert isinstance(bundle, SubmissionBundle) + bundles.append(bundle) + + # AND all bundles should be valid SubmissionBundle objects + assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) + async def test_get_evaluation_submission_bundles_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles filtered by status.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + )) # THEN the bundles should be retrieved assert bundles is not None @@ -158,63 +176,57 @@ async def test_get_evaluation_submission_bundles_with_status_filter( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - SubmissionBundle.get_evaluation_submission_bundles( + list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, status="NONEXISTENT_STATUS", synapse_client=self.syn, - ) + )) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) assert "NONEXISTENT_STATUS" in str(exc_info.value) - async def test_get_evaluation_submission_bundles_with_pagination( + async def test_get_evaluation_submission_bundles_generator_behavior_with_multiple( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting submission bundles.""" - # WHEN I get submission bundles with a limit - bundles_page1 = SubmissionBundle.get_evaluation_submission_bundles( + """Test generator behavior when getting submission bundles with multiple submissions.""" + # WHEN I get submission bundles using the generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + )) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all available bundles (at least the ones we created) + assert bundles is not None + assert len(bundles) >= len(multiple_submissions) - # WHEN I get the next page - bundles_page2 = SubmissionBundle.get_evaluation_submission_bundles( + # AND I should be able to iterate through the generator multiple times + # by creating a new generator each time + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=2, synapse_client=self.syn, ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + + # THEN I should get the same bundles when iterating again + bundles_second_iteration = list(bundles_generator) + assert len(bundles_second_iteration) == len(bundles) + + # AND all created submissions should be found + bundle_submission_ids = { + bundle.submission.id for bundle in bundles if bundle.submission + } + created_submission_ids = {sub.id for sub in multiple_submissions} + assert created_submission_ids.issubset(bundle_submission_ids), \ + "All created submissions should be found in bundles" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - SubmissionBundle.get_evaluation_submission_bundles( + list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id="syn999999999999", synapse_client=self.syn, - ) + )) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -224,10 +236,10 @@ async def test_get_user_submission_bundles_basic( ): """Test getting user submission bundles for an evaluation.""" # WHEN I get user submission bundles for an evaluation - bundles = SubmissionBundle.get_user_submission_bundles( + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN the bundles should be retrieved assert bundles is not None @@ -249,44 +261,38 @@ async def test_get_user_submission_bundles_basic( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in user bundles" - async def test_get_user_submission_bundles_with_pagination( + async def test_get_user_submission_bundles_generator_behavior_with_multiple( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting user submission bundles.""" - # WHEN I get user submission bundles with a limit - bundles_page1 = SubmissionBundle.get_user_submission_bundles( + """Test generator behavior when getting user submission bundles with multiple submissions.""" + # WHEN I get user submission bundles using the generator + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + )) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all available bundles (at least the ones we created) + assert bundles is not None + assert len(bundles) >= len(multiple_submissions) - # WHEN I get the next page - bundles_page2 = SubmissionBundle.get_user_submission_bundles( + # AND I should be able to iterate through the generator multiple times + # by creating a new generator each time + bundles_generator = SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=2, synapse_client=self.syn, ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + + # THEN I should get the same bundles when iterating again + bundles_second_iteration = list(bundles_generator) + assert len(bundles_second_iteration) == len(bundles) + + # AND all created submissions should be found + bundle_submission_ids = { + bundle.submission.id for bundle in bundles if bundle.submission + } + created_submission_ids = {sub.id for sub in multiple_submissions} + assert created_submission_ids.issubset(bundle_submission_ids), \ + "All created submissions should be found in user bundles" class TestSubmissionBundleDataIntegrity: @@ -363,10 +369,10 @@ async def test_submission_bundle_data_consistency( ): """Test that submission bundles maintain data consistency between submission and status.""" # WHEN I get submission bundles for the evaluation - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN I should find our test submission test_bundle = None @@ -408,10 +414,10 @@ async def test_submission_bundle_status_updates_reflected( updated_status = submission_status.store(synapse_client=self.syn) # AND I get submission bundles again - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN the bundle should reflect the updated status test_bundle = None @@ -439,10 +445,10 @@ async def test_submission_bundle_evaluation_id_propagation( ): """Test that evaluation_id is properly propagated from submission to status.""" # WHEN I get submission bundles - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN find our test bundle test_bundle = None @@ -497,10 +503,10 @@ async def test_get_evaluation_submission_bundles_empty_evaluation( ): """Test getting submission bundles from an evaluation with no submissions.""" # WHEN I get submission bundles from an evaluation with no submissions - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -512,42 +518,40 @@ async def test_get_user_submission_bundles_empty_evaluation( ): """Test getting user submission bundles from an evaluation with no submissions.""" # WHEN I get user submission bundles from an evaluation with no submissions - bundles = SubmissionBundle.get_user_submission_bundles( + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not None or error) assert bundles is not None assert isinstance(bundles, list) assert len(bundles) == 0 - async def test_get_evaluation_submission_bundles_large_limit( + async def test_get_evaluation_submission_bundles_generator_consistency( self, test_evaluation: Evaluation ): - """Test getting submission bundles with a very large limit.""" - # WHEN I request bundles with a large limit (within API bounds) - bundles = SubmissionBundle.get_evaluation_submission_bundles( + """Test that the generator produces consistent results across multiple iterations.""" + # WHEN I request bundles using the generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=100, # Maximum allowed by API synapse_client=self.syn, - ) + )) # THEN it should work without error assert bundles is not None assert isinstance(bundles, list) # The actual count doesn't matter since the evaluation is empty - async def test_get_user_submission_bundles_large_offset( + async def test_get_user_submission_bundles_generator_empty_results( self, test_evaluation: Evaluation ): - """Test getting user submission bundles with a large offset.""" - # WHEN I request bundles with a large offset (beyond available data) - bundles = SubmissionBundle.get_user_submission_bundles( + """Test that user submission bundles generator handles empty results correctly.""" + # WHEN I request bundles from an empty evaluation + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - offset=1000, # Large offset beyond any real data synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not error) assert bundles is not None @@ -559,14 +563,14 @@ async def test_get_submission_bundles_with_default_parameters( ): """Test that default parameters work correctly.""" # WHEN I call methods without optional parameters - eval_bundles = SubmissionBundle.get_evaluation_submission_bundles( + eval_bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) - user_bundles = SubmissionBundle.get_user_submission_bundles( + )) + user_bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py index 1e2efca02..5253c2f27 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -239,24 +239,28 @@ async def test_get_evaluation_submission_bundles_async(self) -> None: # WHEN I call get_evaluation_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = await SubmissionBundle.get_evaluation_submission_bundles_async( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, ) @@ -288,21 +292,27 @@ async def test_get_evaluation_submission_bundles_async_empty_response(self) -> N # WHEN I call get_evaluation_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = await SubmissionBundle.get_evaluation_submission_bundles_async( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield # This will never execute + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status=None, - limit=10, - offset=0, synapse_client=self.syn, ) @@ -333,22 +343,26 @@ async def test_get_user_submission_bundles_async(self) -> None: # WHEN I call get_user_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = await SubmissionBundle.get_user_submission_bundles_async( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, ) @@ -372,20 +386,26 @@ async def test_get_user_submission_bundles_async_default_params(self) -> None: # WHEN I call get_user_submission_bundles_async with defaults with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = await SubmissionBundle.get_user_submission_bundles_async( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=10, - offset=0, synapse_client=self.syn, ) diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index 9dc252286..f06f52c4b 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -239,24 +239,26 @@ def test_get_evaluation_submission_bundles(self) -> None: # WHEN I call get_evaluation_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = SubmissionBundle.get_evaluation_submission_bundles( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, - ) + )) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, ) @@ -288,21 +290,25 @@ def test_get_evaluation_submission_bundles_empty_response(self) -> None: # WHEN I call get_evaluation_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = SubmissionBundle.get_evaluation_submission_bundles( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + )) # THEN the service should be called mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status=None, - limit=10, - offset=0, synapse_client=self.syn, ) @@ -333,22 +339,24 @@ def test_get_user_submission_bundles(self) -> None: # WHEN I call get_user_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = SubmissionBundle.get_user_submission_bundles( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, - ) + )) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, ) @@ -372,20 +380,24 @@ def test_get_user_submission_bundles_default_params(self) -> None: # WHEN I call get_user_submission_bundles with defaults (sync method) with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = SubmissionBundle.get_user_submission_bundles( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield # This will never execute + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + )) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=10, - offset=0, synapse_client=self.syn, ) From b8c0ee5e546104ad3f91af28eca27898d3d1a78b Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 14:16:51 -0500 Subject: [PATCH 38/60] address final todo: implement docker_tag, and note in docs that version_number can be ignored for docker submissions --- synapseclient/models/submission.py | 20 +- synapseclient/models/submission_bundle.py | 16 +- .../models/async/test_submission_async.py | 4 +- .../async/test_submission_bundle_async.py | 12 +- .../models/synchronous/test_submission.py | 24 ++- .../synchronous/test_submission_bundle.py | 184 +++++++++++------- .../async/unit_test_submission_async.py | 2 +- .../synchronous/unit_test_submission.py | 4 +- .../unit_test_submission_bundle.py | 42 ++-- 9 files changed, 189 insertions(+), 119 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 08dc642ee..f90660d26 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -5,7 +5,12 @@ from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, otel_trace_method, wrap_async_generator_to_sync_generator +from synapseclient.core.async_utils import ( + async_to_sync, + otel_trace_method, + skip_async_to_sync, + wrap_async_generator_to_sync_generator, +) from synapseclient.models.mixins.access_control import AccessControllable @@ -340,7 +345,7 @@ class Submission( version_number: Optional[int] = field(default=None, compare=False) """ - The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. + The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. If entity is a Docker repository, this attribute should be ignored in favor of `docker_digest` or `docker_tag`. """ evaluation_id: Optional[str] = None @@ -389,6 +394,11 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ + docker_tag: Optional[str] = None + """ + For Docker repositories, the tag of the submitted Docker image. + """ + etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" @@ -580,11 +590,11 @@ async def create_submission_example(): entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository" ): - self.version_number = ( - 1 # TODO: Docker repositories do not have version numbers - ) self.docker_repository_name = entity_info.get("repositoryName") self.docker_digest = entity_info.get("digest") + self.docker_tag = entity_info.get("tag") + # All docker repositories are assigned version number 1, even if they have multiple tags + self.version_number = 1 else: raise ValueError("entity_id is required to create a submission") diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 75dde8610..ffd56d489 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,9 +1,21 @@ from dataclasses import dataclass -from typing import TYPE_CHECKING, AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union +from typing import ( + TYPE_CHECKING, + AsyncGenerator, + Dict, + Generator, + Optional, + Protocol, + Union, +) from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, wrap_async_generator_to_sync_generator +from synapseclient.core.async_utils import ( + async_to_sync, + skip_async_to_sync, + wrap_async_generator_to_sync_generator, +) if TYPE_CHECKING: from synapseclient.models.submission import Submission diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 5ef52ed4e..f7cb4fdda 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -290,7 +290,7 @@ async def test_get_evaluation_submissions_async_generator_behavior( async for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) @@ -320,7 +320,7 @@ async def test_get_user_submissions_async_generator_behavior( async for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index 5f4273826..b24f8c0ef 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -148,7 +148,7 @@ async def test_get_evaluation_submission_bundles_async_generator_behavior( async for bundle in bundles_generator: assert isinstance(bundle, SubmissionBundle) bundles.append(bundle) - + # AND all bundles should be valid SubmissionBundle objects assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) @@ -209,7 +209,9 @@ async def test_get_evaluation_submission_bundles_with_automatic_pagination_async # THEN I should get all bundles for the evaluation assert all_bundles is not None - assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions + assert len(all_bundles) >= len( + multiple_submissions + ) # At least our test submissions # AND each bundle should be valid for bundle in all_bundles: @@ -286,7 +288,9 @@ async def test_get_user_submission_bundles_with_automatic_pagination_async( # THEN I should get all bundles for the user in this evaluation assert all_bundles is not None - assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions + assert len(all_bundles) >= len( + multiple_submissions + ) # At least our test submissions # AND each bundle should be valid for bundle in all_bundles: @@ -634,7 +638,7 @@ async def test_get_submission_bundles_with_default_parameters_async( synapse_client=self.syn, ): eval_bundles.append(bundle) - + user_bundles = [] async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index 16fa52b1a..e89800042 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -239,9 +239,11 @@ async def test_get_evaluation_submissions( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get all submissions for an evaluation - submissions = list(Submission.get_evaluation_submissions( - evaluation_id=test_evaluation.id, synapse_client=self.syn - )) + submissions = list( + Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + ) # THEN I should get a list of submission objects assert len(submissions) > 0 @@ -255,11 +257,13 @@ async def test_get_evaluation_submissions_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get submissions filtered by status - submissions = list(Submission.get_evaluation_submissions( - evaluation_id=test_evaluation.id, - status="RECEIVED", - synapse_client=self.syn, - )) + submissions = list( + Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # The test submission should be in the results (initially in RECEIVED status) submission_ids = [sub.id for sub in submissions] @@ -279,7 +283,7 @@ async def test_get_evaluation_submissions_generator_behavior( for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) @@ -308,7 +312,7 @@ async def test_get_user_submissions_generator_behavior( for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index cf62e4e22..70d2bbf79 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -112,10 +112,12 @@ async def test_get_evaluation_submission_bundles_basic( ): """Test getting submission bundles for an evaluation.""" # WHEN I get submission bundles for an evaluation using generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get at least our test submission assert len(bundles) >= 1 # At least our test submission @@ -151,7 +153,7 @@ async def test_get_evaluation_submission_bundles_generator_behavior( for bundle in bundles_generator: assert isinstance(bundle, SubmissionBundle) bundles.append(bundle) - + # AND all bundles should be valid SubmissionBundle objects assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) @@ -160,11 +162,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter( ): """Test getting submission bundles filtered by status.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - status="RECEIVED", - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # THEN the bundles should be retrieved assert bundles is not None @@ -176,11 +180,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - status="NONEXISTENT_STATUS", - synapse_client=self.syn, - )) + list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + ) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) @@ -191,10 +197,12 @@ async def test_get_evaluation_submission_bundles_generator_behavior_with_multipl ): """Test generator behavior when getting submission bundles with multiple submissions.""" # WHEN I get submission bundles using the generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get all available bundles (at least the ones we created) assert bundles is not None @@ -206,27 +214,30 @@ async def test_get_evaluation_submission_bundles_generator_behavior_with_multipl evaluation_id=test_evaluation.id, synapse_client=self.syn, ) - + # THEN I should get the same bundles when iterating again bundles_second_iteration = list(bundles_generator) assert len(bundles_second_iteration) == len(bundles) - + # AND all created submissions should be found bundle_submission_ids = { bundle.submission.id for bundle in bundles if bundle.submission } created_submission_ids = {sub.id for sub in multiple_submissions} - assert created_submission_ids.issubset(bundle_submission_ids), \ - "All created submissions should be found in bundles" + assert created_submission_ids.issubset( + bundle_submission_ids + ), "All created submissions should be found in bundles" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id="syn999999999999", - synapse_client=self.syn, - )) + list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + ) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -236,10 +247,12 @@ async def test_get_user_submission_bundles_basic( ): """Test getting user submission bundles for an evaluation.""" # WHEN I get user submission bundles for an evaluation - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN the bundles should be retrieved assert bundles is not None @@ -266,10 +279,12 @@ async def test_get_user_submission_bundles_generator_behavior_with_multiple( ): """Test generator behavior when getting user submission bundles with multiple submissions.""" # WHEN I get user submission bundles using the generator - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get all available bundles (at least the ones we created) assert bundles is not None @@ -281,18 +296,19 @@ async def test_get_user_submission_bundles_generator_behavior_with_multiple( evaluation_id=test_evaluation.id, synapse_client=self.syn, ) - + # THEN I should get the same bundles when iterating again bundles_second_iteration = list(bundles_generator) assert len(bundles_second_iteration) == len(bundles) - + # AND all created submissions should be found bundle_submission_ids = { bundle.submission.id for bundle in bundles if bundle.submission } created_submission_ids = {sub.id for sub in multiple_submissions} - assert created_submission_ids.issubset(bundle_submission_ids), \ - "All created submissions should be found in user bundles" + assert created_submission_ids.issubset( + bundle_submission_ids + ), "All created submissions should be found in user bundles" class TestSubmissionBundleDataIntegrity: @@ -369,10 +385,12 @@ async def test_submission_bundle_data_consistency( ): """Test that submission bundles maintain data consistency between submission and status.""" # WHEN I get submission bundles for the evaluation - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should find our test submission test_bundle = None @@ -414,10 +432,12 @@ async def test_submission_bundle_status_updates_reflected( updated_status = submission_status.store(synapse_client=self.syn) # AND I get submission bundles again - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN the bundle should reflect the updated status test_bundle = None @@ -445,10 +465,12 @@ async def test_submission_bundle_evaluation_id_propagation( ): """Test that evaluation_id is properly propagated from submission to status.""" # WHEN I get submission bundles - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN find our test bundle test_bundle = None @@ -503,10 +525,12 @@ async def test_get_evaluation_submission_bundles_empty_evaluation( ): """Test getting submission bundles from an evaluation with no submissions.""" # WHEN I get submission bundles from an evaluation with no submissions - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -518,10 +542,12 @@ async def test_get_user_submission_bundles_empty_evaluation( ): """Test getting user submission bundles from an evaluation with no submissions.""" # WHEN I get user submission bundles from an evaluation with no submissions - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -533,10 +559,12 @@ async def test_get_evaluation_submission_bundles_generator_consistency( ): """Test that the generator produces consistent results across multiple iterations.""" # WHEN I request bundles using the generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should work without error assert bundles is not None @@ -548,10 +576,12 @@ async def test_get_user_submission_bundles_generator_empty_results( ): """Test that user submission bundles generator handles empty results correctly.""" # WHEN I request bundles from an empty evaluation - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not error) assert bundles is not None @@ -563,14 +593,18 @@ async def test_get_submission_bundles_with_default_parameters( ): """Test that default parameters work correctly.""" # WHEN I call methods without optional parameters - eval_bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) - user_bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + eval_bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) + user_bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index a18f22fe8..3587c0cdb 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -488,7 +488,7 @@ async def test_get_evaluation_submissions_async(self) -> None: async def mock_async_gen(*args, **kwargs): submission_data = self.get_example_submission_response() yield submission_data - + # Make the mock return our async generator when called mock_get_submissions.side_effect = mock_async_gen diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index c1d2eb2c0..b7d6f2721 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -614,9 +614,7 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - await submission.cancel_async( - synapse_client=self.syn - ) + await submission.cancel_async(synapse_client=self.syn) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index f06f52c4b..cb5d44018 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -249,11 +249,13 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - status="RECEIVED", - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( @@ -300,10 +302,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called mock_get_bundles.assert_called_once_with( @@ -349,10 +353,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_user_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( @@ -390,10 +396,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_user_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( From 73c5f1a5b5c0c276ad7dafafcf45dfda3b8d4a75 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 13:28:01 -0500 Subject: [PATCH 39/60] import -> imports --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index a4df850f1..71965845d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -193,7 +193,7 @@ plugins: default_handler: python handlers: python: - import: + imports: - https://docs.python.org/3/objects.inv - https://python-markdown.github.io/objects.inv options: From 8db9e67db48fd342651b7dd6941fdd0b512720e8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 16:29:07 -0500 Subject: [PATCH 40/60] change back to import. patch uses synapse logger instance. --- mkdocs.yml | 2 +- tests/unit/synapseclient/unit_test_client.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mkdocs.yml b/mkdocs.yml index 71965845d..a4df850f1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -193,7 +193,7 @@ plugins: default_handler: python handlers: python: - imports: + import: - https://docs.python.org/3/objects.inv - https://python-markdown.github.io/objects.inv options: diff --git a/tests/unit/synapseclient/unit_test_client.py b/tests/unit/synapseclient/unit_test_client.py index 37fa070ef..209bf4e8f 100644 --- a/tests/unit/synapseclient/unit_test_client.py +++ b/tests/unit/synapseclient/unit_test_client.py @@ -1213,7 +1213,7 @@ def test_check_entity_restrictions_unmet_restriction_entity_file_with_download_f def test_check_entity_restrictions_unmet_restriction_entity_project_with_download_file_is_true( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", @@ -1236,7 +1236,7 @@ def test_check_entity_restrictions_unmet_restriction_entity_project_with_downloa def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download_file_is_true_and_no_token( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", @@ -1259,7 +1259,7 @@ def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download_file_is_true_and_no_credentials( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", @@ -1282,7 +1282,7 @@ def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download_file_is_true( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", @@ -1305,7 +1305,7 @@ def test_check_entity_restrictions_unmet_restriction_entity_folder_with_download def test_check_entity_restrictions__unmet_restriction_downloadFile_is_False( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", From 53d98523ab66bcbb3eefd0cc228eacde47d0a0d7 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 2 Dec 2025 14:46:18 -0500 Subject: [PATCH 41/60] lock mkdocstrings-python to >=2.0.0 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 8866ad630..c5c90d0cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -126,7 +126,7 @@ docs = mkdocs>=1.5.3 mkdocs-material>=9.4.14 mkdocstrings>=0.24.0 - mkdocstrings-python>=1.8.0 + mkdocstrings-python>=2.0.0 termynal>=0.11.1 mkdocs-open-in-new-tab~=1.0.3 markdown-include~=0.8.1 From 4d082c3086023e19166274b8332166072ff41408 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 2 Dec 2025 15:03:16 -0500 Subject: [PATCH 42/60] add Return description to create_submission --- synapseclient/api/evaluation_services.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 80fee69a8..134a4eb13 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -403,6 +403,9 @@ async def create_submission( etag: The current eTag of the Entity being submitted. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the created Submission model's attributes. """ from synapseclient import Synapse From 2272d474439d034663a6be9eaed92ed1f4a4b4c8 Mon Sep 17 00:00:00 2001 From: SageGJ Date: Wed, 3 Dec 2025 14:37:57 -0700 Subject: [PATCH 43/60] [SYNPY-1714] Fix Issues with Failing Tests (#1287) * always log warning * check assert called with * patch object on class instance * Revert "always log warning" This reverts commit ae63c814df29b5895826c1188fca26fa729a39af. * update patch target * re-add erroneously removed line --- tests/unit/synapseclient/unit_test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/synapseclient/unit_test_client.py b/tests/unit/synapseclient/unit_test_client.py index 209bf4e8f..57142e5bc 100644 --- a/tests/unit/synapseclient/unit_test_client.py +++ b/tests/unit/synapseclient/unit_test_client.py @@ -1172,7 +1172,7 @@ def init_syn(self, syn: Synapse) -> None: self.syn.credentials = SynapseAuthTokenCredentials(token="abc", username="def") def test_check_entity_restrictions_no_unmet_restriction(self) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", @@ -1189,7 +1189,7 @@ def test_check_entity_restrictions_no_unmet_restriction(self) -> None: def test_check_entity_restrictions_unmet_restriction_entity_file_with_download_file_is_true( self, ) -> None: - with patch("logging.Logger.warning") as mocked_warn: + with patch.object(self.syn.logger, "warning") as mocked_warn: bundle = { "entity": { "id": "syn123", From 134ad0cc2ec65b767c77069e3133b55105fd5ea9 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 12:05:18 -0500 Subject: [PATCH 44/60] no need to build out Annotations object from scratch (remove evaluation_id attribute) --- synapseclient/models/submission_bundle.py | 7 ------ synapseclient/models/submission_status.py | 30 +---------------------- 2 files changed, 1 insertion(+), 36 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index ffd56d489..50a620796 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -212,13 +212,6 @@ def fill_from_dict( self.submission_status = SubmissionStatus().fill_from_dict( submission_status_dict ) - # Manually set evaluation_id from the submission data if available - if ( - self.submission_status - and self.submission - and self.submission.evaluation_id - ): - self.submission_status.evaluation_id = self.submission.evaluation_id else: self.submission_status = None diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index d8be2c913..d345c34f4 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -231,8 +231,6 @@ class SubmissionStatus( private_status_annotations: Indicates whether the annotations (not to be confused with submission annotations) are private (True) or public (False). Default is True. This controls the visibility of the 'annotations' field. entity_id: The Synapse ID of the Entity in this Submission. - evaluation_id: The ID of the Evaluation to which this Submission belongs. This field is automatically - populated when retrieving a SubmissionStatus via get() and is required when updating annotations. version_number: The version number of the Entity in this Submission. status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. @@ -385,11 +383,6 @@ class SubmissionStatus( The Synapse ID of the Entity in this Submission. """ - evaluation_id: Optional[str] = None - """ - The ID of the Evaluation to which this Submission belongs. - """ - version_number: Optional[int] = field(default=None, compare=False) """ The version number of the Entity in this Submission. @@ -537,20 +530,8 @@ def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: request_body["cancelRequested"] = self.cancel_requested if self.annotations and len(self.annotations) > 0: - # evaluation_id is required when annotations are provided for scopeId - if self.evaluation_id is None: - raise ValueError( - "Your submission status object is missing the 'evaluation_id' attribute. This attribute is required when submissions are updated with annotations. Please retrieve your submission status with .get() to populate this field." - ) - - # Add required objectId and scopeId to annotations dict as per Synapse API requirements - # https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/annotation/Annotations.html - annotations_with_metadata = self.annotations.copy() - annotations_with_metadata["objectId"] = self.id - annotations_with_metadata["scopeId"] = self.evaluation_id - request_body["annotations"] = to_submission_status_annotations( - annotations_with_metadata, self.private_status_annotations + self.annotations, self.private_status_annotations ) if self.submission_annotations and len(self.submission_annotations) > 0: @@ -606,13 +587,6 @@ async def get_async( ) self.fill_from_dict(response) - # Fetch evaluation_id from the associated submission since it's not in the SubmissionStatus response - if not self.evaluation_id: - submission_response = await evaluation_services.get_submission( - submission_id=self.id, synapse_client=synapse_client - ) - self.evaluation_id = submission_response.get("evaluationId", None) - self._set_last_persistent_instance() return self @@ -757,8 +731,6 @@ async def get_all_submission_statuses_async( for status_dict in response.get("results", []): submission_status = SubmissionStatus() submission_status.fill_from_dict(status_dict) - # Manually set evaluation_id since it's not in the SubmissionStatus response - submission_status.evaluation_id = evaluation_id submission_status._set_last_persistent_instance() submission_statuses.append(submission_status) From 48468c6fd129fcdf177334e828015b7ea2c617ca Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 14:12:21 -0500 Subject: [PATCH 45/60] remove Dict, List imports --- synapseclient/api/evaluation_services.py | 22 ++++----- synapseclient/models/submission.py | 16 +++---- synapseclient/models/submission_bundle.py | 3 +- synapseclient/models/submission_status.py | 46 +++++++++---------- .../synchronous/test_submission_status.py | 1 - 5 files changed, 43 insertions(+), 45 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 134a4eb13..60205f4af 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -4,7 +4,7 @@ """ import json -from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional from synapseclient.api.api_client import rest_get_paginated_async @@ -105,12 +105,12 @@ async def get_evaluations_by_project( project_id: str, access_type: Optional[str] = None, active_only: Optional[bool] = None, - evaluation_ids: Optional[List[str]] = None, + evaluation_ids: Optional[list[str]] = None, offset: Optional[int] = None, limit: Optional[int] = None, *, synapse_client: Optional["Synapse"] = None, -) -> List[dict]: +) -> list[dict]: """ Gets Evaluations tied to a project. Note: The response will contain only those Evaluations on which the caller is granted the ACCESS_TYPE.READ permission, unless specified otherwise with the accessType parameter. @@ -159,12 +159,12 @@ async def get_evaluations_by_project( async def get_all_evaluations( access_type: Optional[str] = None, active_only: Optional[bool] = None, - evaluation_ids: Optional[List[str]] = None, + evaluation_ids: Optional[list[str]] = None, offset: Optional[int] = None, limit: Optional[int] = None, *, synapse_client: Optional["Synapse"] = None, -) -> List[dict]: +) -> list[dict]: """ Get a list of all Evaluations, within a given range. Note: The response will contain only those Evaluations on which the caller is granted the ACCESS_TYPE.READ permission, unless specified otherwise with the accessType parameter. @@ -211,12 +211,12 @@ async def get_all_evaluations( async def get_available_evaluations( active_only: Optional[bool] = None, - evaluation_ids: Optional[List[str]] = None, + evaluation_ids: Optional[list[str]] = None, offset: Optional[int] = None, limit: Optional[int] = None, *, synapse_client: Optional["Synapse"] = None, -) -> List[dict]: +) -> list[dict]: """ Get a list of Evaluations to which the user has SUBMIT permission, within a given range. Note: The response will contain only those Evaluations on which the caller is granted the ACCESS_TYPE.SUBMIT permission. @@ -455,7 +455,7 @@ async def get_evaluation_submissions( status: Optional[str] = None, *, synapse_client: Optional["Synapse"] = None, -) -> AsyncGenerator[Dict[str, Any], None]: +) -> AsyncGenerator[dict[str, Any], None]: """ Generator to get all Submissions for a specified Evaluation queue. @@ -492,7 +492,7 @@ async def get_user_submissions( user_id: Optional[str] = None, *, synapse_client: Optional["Synapse"] = None, -) -> AsyncGenerator[Dict[str, Any], None]: +) -> AsyncGenerator[dict[str, Any], None]: """ Generator to get all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. @@ -788,7 +788,7 @@ async def get_evaluation_submission_bundles( status: Optional[str] = None, *, synapse_client: Optional["Synapse"] = None, -) -> AsyncGenerator[Dict[str, Any], None]: +) -> AsyncGenerator[dict[str, Any], None]: """ Generator to get all bundled Submissions and SubmissionStatuses to a given Evaluation. @@ -827,7 +827,7 @@ async def get_user_submission_bundles( evaluation_id: str, *, synapse_client: Optional["Synapse"] = None, -) -> AsyncGenerator[Dict[str, Any], None]: +) -> AsyncGenerator[dict[str, Any], None]: """ Generator to get all user bundled Submissions and SubmissionStatuses for a specified Evaluation. diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index f90660d26..fa3270816 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union +from typing import AsyncGenerator, Generator, Optional, Protocol, Union from typing_extensions import Self @@ -248,7 +248,7 @@ def get_submission_count( status: Optional[str] = None, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> dict: """ Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. @@ -368,12 +368,12 @@ class Submission( The ID of the team that submitted this submission (if it's a team submission). """ - contributors: List[str] = field(default_factory=list) + contributors: list[str] = field(default_factory=list) """ User IDs of team members who contributed to this submission (if it's a team submission). """ - submission_status: Optional[Dict] = None + submission_status: Optional[dict] = None """ The status of this Submission. """ @@ -403,7 +403,7 @@ class Submission( """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" def fill_from_dict( - self, synapse_submission: Dict[str, Union[bool, str, int, List]] + self, synapse_submission: dict[str, Union[bool, str, int, list]] ) -> "Submission": """ Converts a response from the REST API into this dataclass. @@ -435,7 +435,7 @@ def fill_from_dict( async def _fetch_latest_entity( self, *, synapse_client: Optional[Synapse] = None - ) -> Dict: + ) -> dict: """ Fetch the latest entity information from Synapse. @@ -491,7 +491,7 @@ async def _fetch_latest_entity( f"Unable to fetch entity information for {self.entity_id}: {e}" ) - def to_synapse_request(self) -> Dict: + def to_synapse_request(self) -> dict: """Creates a request body expected of the Synapse REST API for the Submission model. Returns: @@ -774,7 +774,7 @@ async def get_submission_count_async( status: Optional[str] = None, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> dict: """ Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 50a620796..e6c786003 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -2,7 +2,6 @@ from typing import ( TYPE_CHECKING, AsyncGenerator, - Dict, Generator, Optional, Protocol, @@ -187,7 +186,7 @@ class SubmissionBundle(SubmissionBundleSynchronousProtocol): def fill_from_dict( self, - synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]], + synapse_submission_bundle: dict[str, Union[bool, str, int, dict]], ) -> "SubmissionBundle": """ Converts a response from the REST API into this dataclass. diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index d345c34f4..1a3224995 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,6 +1,6 @@ from dataclasses import dataclass, field, replace from datetime import date, datetime -from typing import Dict, List, Optional, Protocol, Union +from typing import Optional, Protocol, Union from typing_extensions import Self @@ -105,7 +105,7 @@ def get_all_submission_statuses( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionStatus"]: + ) -> list["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -148,13 +148,13 @@ def get_all_submission_statuses( @staticmethod def batch_update_submission_statuses( evaluation_id: str, - statuses: List["SubmissionStatus"], + statuses: list["SubmissionStatus"], is_first_batch: bool = True, is_last_batch: bool = True, batch_token: Optional[str] = None, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> dict: """ Update multiple SubmissionStatuses. The maximum batch size is 500. @@ -346,30 +346,30 @@ class SubmissionStatus( """ annotations: Optional[ - Dict[ + dict[ str, Union[ - List[str], - List[bool], - List[float], - List[int], - List[date], - List[datetime], + list[str], + list[bool], + list[float], + list[int], + list[date], + list[datetime], ], ] ] = field(default_factory=dict) """Primary container object for Annotations on a Synapse object.""" submission_annotations: Optional[ - Dict[ + dict[ str, Union[ - List[str], - List[bool], - List[float], - List[int], - List[date], - List[datetime], + list[str], + list[bool], + list[float], + list[int], + list[date], + list[datetime], ], ] ] = field(default_factory=dict) @@ -439,7 +439,7 @@ def _set_last_persistent_instance(self) -> None: def fill_from_dict( self, - synapse_submission_status: Dict[str, Union[bool, str, int, float, List]], + synapse_submission_status: dict[str, Union[bool, str, int, float, list]], ) -> "SubmissionStatus": """ Converts a response from the REST API into this dataclass. @@ -478,7 +478,7 @@ def fill_from_dict( return self - def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: + def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> dict: """ Creates a request body expected by the Synapse REST API for the SubmissionStatus model. @@ -681,7 +681,7 @@ async def get_all_submission_statuses_async( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionStatus"]: + ) -> list["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -739,13 +739,13 @@ async def get_all_submission_statuses_async( @staticmethod async def batch_update_submission_statuses_async( evaluation_id: str, - statuses: List["SubmissionStatus"], + statuses: list["SubmissionStatus"], is_first_batch: bool = True, is_last_batch: bool = True, batch_token: Optional[str] = None, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> dict: """ Update multiple SubmissionStatuses. The maximum batch size is 500. diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index b4ae45372..2b3f5c1c6 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -2,7 +2,6 @@ import os import tempfile -import test import uuid from typing import Callable From 23a27790a24fa65b825b8e1982606e1d7b75ef57 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 14:47:46 -0500 Subject: [PATCH 46/60] fix broken tests due to removed evaluation_id attr --- .../async/test_submission_bundle_async.py | 6 +-- .../async/test_submission_status_async.py | 15 -------- .../synchronous/test_submission_bundle.py | 27 ------------- .../synchronous/test_submission_status.py | 15 -------- .../unit_test_submission_bundle_async.py | 8 ---- .../unit_test_submission_status_async.py | 32 +--------------- .../unit_test_submission_bundle.py | 30 +-------------- .../unit_test_submission_status.py | 38 ++----------------- 8 files changed, 8 insertions(+), 163 deletions(-) diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index b24f8c0ef..2cbfc081c 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -412,7 +412,6 @@ async def test_submission_bundle_data_consistency_async( if test_bundle.submission_status: assert test_bundle.submission_status.id == test_submission.id assert test_bundle.submission_status.entity_id == test_file.id - assert test_bundle.submission_status.evaluation_id == test_evaluation.id async def test_submission_bundle_status_updates_reflected_async( self, test_evaluation: Evaluation, test_submission: Submission @@ -482,10 +481,9 @@ async def test_submission_bundle_evaluation_id_propagation_async( assert test_bundle is not None - # AND both submission and status should have the correct evaluation_id + # AND submission should have the correct evaluation_id assert test_bundle.submission.evaluation_id == test_evaluation.id - if test_bundle.submission_status: - assert test_bundle.submission_status.evaluation_id == test_evaluation.id + # submission_status no longer has evaluation_id attribute class TestSubmissionBundleEdgeCasesAsync: diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index bf72a8158..bbad7997b 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -98,7 +98,6 @@ async def test_get_submission_status_by_id( # THEN the submission status should be retrieved correctly assert submission_status.id == test_submission.id assert submission_status.entity_id == test_submission.entity_id - assert submission_status.evaluation_id == test_evaluation.id assert ( submission_status.status is not None ) # Should have some status (e.g., "RECEIVED") @@ -571,7 +570,6 @@ async def test_get_all_submission_statuses( # AND each status should have proper attributes for status in statuses: assert status.id is not None - assert status.evaluation_id == test_evaluation.id assert status.status is not None assert status.etag is not None @@ -589,7 +587,6 @@ async def test_get_all_submission_statuses_with_status_filter( # THEN I should only get statuses with the specified status for status in statuses: assert status.status == "RECEIVED" - assert status.evaluation_id == test_evaluation.id async def test_get_all_submission_statuses_with_pagination( self, test_evaluation: Evaluation, test_submissions: list[Submission] @@ -792,17 +789,6 @@ async def test_to_synapse_request_missing_required_attributes(self): with pytest.raises(ValueError, match="missing the 'status_version' attribute"): submission_status.to_synapse_request(synapse_client=self.syn) - async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): - """Test that annotations require evaluation_id async.""" - # WHEN I try to create a request with annotations but no evaluation_id - submission_status = SubmissionStatus( - id="123", etag="some-etag", status_version=1, annotations={"test": "value"} - ) - - # THEN it should raise a ValueError - with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): - submission_status.to_synapse_request(synapse_client=self.syn) - async def test_to_synapse_request_valid_attributes(self): """Test that to_synapse_request works with valid attributes async.""" # WHEN I create a request with all required attributes @@ -811,7 +797,6 @@ async def test_to_synapse_request_valid_attributes(self): etag="some-etag", status_version=1, status="SCORED", - evaluation_id="eval123", submission_annotations={"score": 85.5}, ) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index 70d2bbf79..713c89f8b 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -411,7 +411,6 @@ async def test_submission_bundle_data_consistency( if test_bundle.submission_status: assert test_bundle.submission_status.id == test_submission.id assert test_bundle.submission_status.entity_id == test_file.id - assert test_bundle.submission_status.evaluation_id == test_evaluation.id async def test_submission_bundle_status_updates_reflected( self, test_evaluation: Evaluation, test_submission: Submission @@ -460,32 +459,6 @@ async def test_submission_bundle_status_updates_reflected( submission_status.submission_annotations = {} submission_status.store(synapse_client=self.syn) - async def test_submission_bundle_evaluation_id_propagation( - self, test_evaluation: Evaluation, test_submission: Submission - ): - """Test that evaluation_id is properly propagated from submission to status.""" - # WHEN I get submission bundles - bundles = list( - SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - ) - ) - - # THEN find our test bundle - test_bundle = None - for bundle in bundles: - if bundle.submission and bundle.submission.id == test_submission.id: - test_bundle = bundle - break - - assert test_bundle is not None - - # AND both submission and status should have the correct evaluation_id - assert test_bundle.submission.evaluation_id == test_evaluation.id - if test_bundle.submission_status: - assert test_bundle.submission_status.evaluation_id == test_evaluation.id - class TestSubmissionBundleEdgeCases: """Tests for edge cases and error handling in SubmissionBundle operations.""" diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index 2b3f5c1c6..741a09c94 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -97,7 +97,6 @@ async def test_get_submission_status_by_id( # THEN the submission status should be retrieved correctly assert submission_status.id == test_submission.id assert submission_status.entity_id == test_submission.entity_id - assert submission_status.evaluation_id == test_evaluation.id assert ( submission_status.status is not None ) # Should have some status (e.g., "RECEIVED") @@ -551,7 +550,6 @@ async def test_get_all_submission_statuses( # AND each status should have proper attributes for status in statuses: assert status.id is not None - assert status.evaluation_id == test_evaluation.id assert status.status is not None assert status.etag is not None @@ -569,7 +567,6 @@ async def test_get_all_submission_statuses_with_status_filter( # THEN I should only get statuses with the specified status for status in statuses: assert status.status == "RECEIVED" - assert status.evaluation_id == test_evaluation.id async def test_get_all_submission_statuses_with_pagination( self, test_evaluation: Evaluation, test_submissions: list[Submission] @@ -765,17 +762,6 @@ async def test_to_synapse_request_missing_required_attributes(self): with pytest.raises(ValueError, match="missing the 'status_version' attribute"): submission_status.to_synapse_request(synapse_client=self.syn) - async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): - """Test that annotations require evaluation_id.""" - # WHEN I try to create a request with annotations but no evaluation_id - submission_status = SubmissionStatus( - id="123", etag="some-etag", status_version=1, annotations={"test": "value"} - ) - - # THEN it should raise a ValueError - with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): - submission_status.to_synapse_request(synapse_client=self.syn) - async def test_to_synapse_request_valid_attributes(self): """Test that to_synapse_request works with valid attributes.""" # WHEN I create a request with all required attributes @@ -784,7 +770,6 @@ async def test_to_synapse_request_valid_attributes(self): etag="some-etag", status_version=1, status="SCORED", - evaluation_id="eval123", submission_annotations={"score": 85.5}, ) diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py index 5253c2f27..03dc00fda 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -139,9 +139,6 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert ( - bundle.submission_status.evaluation_id == EVALUATION_ID - ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -201,7 +198,6 @@ def test_fill_from_dict_evaluation_id_setting(self) -> None: assert bundle.submission is not None assert bundle.submission_status is not None assert bundle.submission.evaluation_id == EVALUATION_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID async def test_get_evaluation_submission_bundles_async(self) -> None: """Test getting submission bundles for an evaluation.""" @@ -274,9 +270,6 @@ async def mock_async_gen(*args, **kwargs): assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert ( - result[0].submission_status.evaluation_id == EVALUATION_ID - ) # set from submission # Check second bundle assert result[1].submission is not None @@ -377,7 +370,6 @@ async def mock_async_gen(*args, **kwargs): assert result[0].submission_status is not None assert result[0].submission_status.id == "789" assert result[0].submission_status.status == "VALIDATED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID async def test_get_user_submission_bundles_async_default_params(self) -> None: """Test getting user submission bundles with default parameters.""" diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py index c48ad2e1e..ff4137c9e 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py @@ -83,14 +83,12 @@ def test_init_submission_status(self) -> None: id=SUBMISSION_STATUS_ID, status=STATUS, entity_id=ENTITY_ID, - evaluation_id=EVALUATION_ID, ) # THEN the SubmissionStatus should have the expected attributes assert submission_status.id == SUBMISSION_STATUS_ID assert submission_status.status == STATUS assert submission_status.entity_id == ENTITY_ID - assert submission_status.evaluation_id == EVALUATION_ID assert submission_status.can_cancel is False # default value assert submission_status.cancel_requested is False # default value assert submission_status.private_status_annotations is True # default value @@ -155,25 +153,17 @@ async def test_get_async(self) -> None: "synapseclient.api.evaluation_services.get_submission_status", new_callable=AsyncMock, return_value=self.get_example_submission_status_dict(), - ) as mock_get_status, patch( - "synapseclient.api.evaluation_services.get_submission", - new_callable=AsyncMock, - return_value=self.get_example_submission_dict(), - ) as mock_get_submission: + ) as mock_get_status: result = await submission_status.get_async(synapse_client=self.syn) # THEN the submission status should be retrieved mock_get_status.assert_called_once_with( submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn ) - mock_get_submission.assert_called_once_with( - submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn - ) # AND the result should have the expected data assert result.id == SUBMISSION_STATUS_ID assert result.status == STATUS - assert result.evaluation_id == EVALUATION_ID assert result._last_persistent_instance is not None async def test_get_async_without_id(self) -> None: @@ -196,7 +186,6 @@ async def test_store_async(self) -> None: etag=ETAG, status_version=STATUS_VERSION, status="SCORED", - evaluation_id=EVALUATION_ID, ) submission_status._set_last_persistent_instance() @@ -281,21 +270,6 @@ def test_to_synapse_request_missing_status_version(self) -> None: with pytest.raises(ValueError, match="missing the 'status_version' attribute"): submission_status.to_synapse_request(synapse_client=self.syn) - def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: - """Test to_synapse_request with annotations but missing evaluation_id.""" - # GIVEN a SubmissionStatus with annotations but no evaluation_id - submission_status = SubmissionStatus( - id=SUBMISSION_STATUS_ID, - etag=ETAG, - status_version=STATUS_VERSION, - annotations={"test": "value"}, - ) - - # WHEN I call to_synapse_request - # THEN it should raise a ValueError - with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): - submission_status.to_synapse_request(synapse_client=self.syn) - def test_to_synapse_request_valid(self) -> None: """Test to_synapse_request with valid attributes.""" # GIVEN a SubmissionStatus with all required attributes @@ -304,7 +278,6 @@ def test_to_synapse_request_valid(self) -> None: etag=ETAG, status_version=STATUS_VERSION, status="SCORED", - evaluation_id=EVALUATION_ID, submission_annotations={"score": 85.5}, annotations={"internal_note": "test"}, ) @@ -429,14 +402,12 @@ async def test_batch_update_submission_statuses_async(self) -> None: etag="etag1", status_version=1, status="VALIDATED", - evaluation_id=EVALUATION_ID, ), SubmissionStatus( id="456", etag="etag2", status_version=1, status="SCORED", - evaluation_id=EVALUATION_ID, ), ] @@ -482,7 +453,6 @@ async def test_batch_update_with_batch_token(self) -> None: etag="etag1", status_version=1, status="VALIDATED", - evaluation_id=EVALUATION_ID, ) ] batch_token = "previous_batch_token" diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index cb5d44018..f2bb348ac 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -139,9 +139,6 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert ( - bundle.submission_status.evaluation_id == EVALUATION_ID - ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -181,28 +178,7 @@ def test_fill_from_dict_no_submission(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS - def test_fill_from_dict_evaluation_id_setting(self) -> None: - """Test that evaluation_id is properly set from submission to submission_status.""" - # GIVEN a bundle response where submission_status doesn't have evaluation_id - submission_dict = self.get_example_submission_dict() - status_dict = self.get_example_submission_status_dict() - # Remove evaluation_id from status_dict to simulate API response - status_dict.pop("evaluationId", None) - - bundle_data = { - "submission": submission_dict, - "submissionStatus": status_dict, - } - - # WHEN I fill a SubmissionBundle from the response - bundle = SubmissionBundle().fill_from_dict(bundle_data) - - # THEN submission_status should get evaluation_id from submission - assert bundle.submission is not None - assert bundle.submission_status is not None - assert bundle.submission.evaluation_id == EVALUATION_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID - + def test_get_evaluation_submission_bundles(self) -> None: """Test getting submission bundles for an evaluation using sync method.""" # GIVEN mock response data @@ -274,9 +250,6 @@ async def mock_async_gen(*args, **kwargs): assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert ( - result[0].submission_status.evaluation_id == EVALUATION_ID - ) # set from submission # Check second bundle assert result[1].submission is not None @@ -377,7 +350,6 @@ async def mock_async_gen(*args, **kwargs): assert result[0].submission_status is not None assert result[0].submission_status.id == "789" assert result[0].submission_status.status == "VALIDATED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID def test_get_user_submission_bundles_default_params(self) -> None: """Test getting user submission bundles with default parameters using sync method.""" diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py index e724e64f0..adc656899 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -83,14 +83,12 @@ def test_init_submission_status(self) -> None: id=SUBMISSION_STATUS_ID, status=STATUS, entity_id=ENTITY_ID, - evaluation_id=EVALUATION_ID, ) # THEN the SubmissionStatus should have the expected attributes assert submission_status.id == SUBMISSION_STATUS_ID assert submission_status.status == STATUS assert submission_status.entity_id == ENTITY_ID - assert submission_status.evaluation_id == EVALUATION_ID assert submission_status.can_cancel is False # default value assert submission_status.cancel_requested is False # default value assert submission_status.private_status_annotations is True # default value @@ -155,25 +153,17 @@ def test_get(self) -> None: "synapseclient.api.evaluation_services.get_submission_status", new_callable=AsyncMock, return_value=self.get_example_submission_status_dict(), - ) as mock_get_status, patch( - "synapseclient.api.evaluation_services.get_submission", - new_callable=AsyncMock, - return_value=self.get_example_submission_dict(), - ) as mock_get_submission: + ) as mock_get_status: result = submission_status.get(synapse_client=self.syn) # THEN the submission status should be retrieved mock_get_status.assert_called_once_with( submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn ) - mock_get_submission.assert_called_once_with( - submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn - ) # AND the result should have the expected data assert result.id == SUBMISSION_STATUS_ID assert result.status == STATUS - assert result.evaluation_id == EVALUATION_ID def test_get_without_id(self) -> None: """Test that getting a SubmissionStatus without ID raises ValueError.""" @@ -195,7 +185,6 @@ def test_store(self) -> None: etag=ETAG, status_version=STATUS_VERSION, status="SCORED", - evaluation_id=EVALUATION_ID, ) submission_status._set_last_persistent_instance() @@ -279,21 +268,6 @@ def test_to_synapse_request_missing_status_version(self) -> None: with pytest.raises(ValueError, match="missing the 'status_version' attribute"): submission_status.to_synapse_request(synapse_client=self.syn) - def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: - """Test to_synapse_request with annotations but missing evaluation_id.""" - # GIVEN a SubmissionStatus with annotations but no evaluation_id - submission_status = SubmissionStatus( - id=SUBMISSION_STATUS_ID, - etag=ETAG, - status_version=STATUS_VERSION, - annotations={"test": "value"}, - ) - - # WHEN I call to_synapse_request - # THEN it should raise a ValueError - with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): - submission_status.to_synapse_request(synapse_client=self.syn) - def test_to_synapse_request_valid(self) -> None: """Test to_synapse_request with valid attributes.""" # GIVEN a SubmissionStatus with all required attributes @@ -302,7 +276,6 @@ def test_to_synapse_request_valid(self) -> None: etag=ETAG, status_version=STATUS_VERSION, status="SCORED", - evaluation_id=EVALUATION_ID, submission_annotations={"score": 85.5}, annotations={"internal_note": "test"}, ) @@ -426,15 +399,13 @@ def test_batch_update_submission_statuses(self) -> None: id="123", etag="etag1", status_version=1, - status="VALIDATED", - evaluation_id=EVALUATION_ID, + status="VALIDATED" ), SubmissionStatus( id="456", etag="etag2", status_version=1, - status="SCORED", - evaluation_id=EVALUATION_ID, + status="SCORED" ), ] @@ -479,8 +450,7 @@ def test_batch_update_with_batch_token(self) -> None: id="123", etag="etag1", status_version=1, - status="VALIDATED", - evaluation_id=EVALUATION_ID, + status="VALIDATED" ) ] batch_token = "previous_batch_token" From 72a4dd05628ecb748df481d4c6bb296faf21324a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 14:49:22 -0500 Subject: [PATCH 47/60] style --- synapseclient/models/submission_bundle.py | 9 +-------- .../synchronous/unit_test_submission_bundle.py | 1 - .../synchronous/unit_test_submission_status.py | 17 +++-------------- 3 files changed, 4 insertions(+), 23 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index e6c786003..a44048843 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,12 +1,5 @@ from dataclasses import dataclass -from typing import ( - TYPE_CHECKING, - AsyncGenerator, - Generator, - Optional, - Protocol, - Union, -) +from typing import TYPE_CHECKING, AsyncGenerator, Generator, Optional, Protocol, Union from synapseclient import Synapse from synapseclient.api import evaluation_services diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index f2bb348ac..92f61f186 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -178,7 +178,6 @@ def test_fill_from_dict_no_submission(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS - def test_get_evaluation_submission_bundles(self) -> None: """Test getting submission bundles for an evaluation using sync method.""" # GIVEN mock response data diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py index adc656899..5243dc6ab 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -396,17 +396,9 @@ def test_batch_update_submission_statuses(self) -> None: # GIVEN a list of SubmissionStatus objects statuses = [ SubmissionStatus( - id="123", - etag="etag1", - status_version=1, - status="VALIDATED" - ), - SubmissionStatus( - id="456", - etag="etag2", - status_version=1, - status="SCORED" + id="123", etag="etag1", status_version=1, status="VALIDATED" ), + SubmissionStatus(id="456", etag="etag2", status_version=1, status="SCORED"), ] # AND mock response @@ -447,10 +439,7 @@ def test_batch_update_with_batch_token(self) -> None: # GIVEN a list of SubmissionStatus objects and a batch token statuses = [ SubmissionStatus( - id="123", - etag="etag1", - status_version=1, - status="VALIDATED" + id="123", etag="etag1", status_version=1, status="VALIDATED" ) ] batch_token = "previous_batch_token" From 8dd2db467954226125fd15bd7e90f24e7f135881 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 15:43:29 -0500 Subject: [PATCH 48/60] import classes directly from typing. remove Dict and List. --- synapseclient/annotations.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index fb2933b1f..4e05a5730 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -72,7 +72,7 @@ import collections import datetime -import typing +from typing import Any, Callable, Mapping, Optional, Union from deprecated import deprecated @@ -95,8 +95,8 @@ def raise_anno_type_error(anno_type: str): raise ValueError(f"Unknown type in annotations response: {anno_type}") -ANNO_TYPE_TO_FUNC: typing.Dict[ - str, typing.Callable[[str], typing.Union[str, int, float, datetime.datetime]] +ANNO_TYPE_TO_FUNC: dict[ + str, Callable[[str], Union[str, int, float, datetime.datetime]] ] = collections.defaultdict( raise_anno_type_error, { @@ -109,7 +109,7 @@ def raise_anno_type_error(anno_type: str): ) -def is_synapse_annotations(annotations: typing.Mapping) -> bool: +def is_synapse_annotations(annotations: Mapping) -> bool: """Tests if the given object is a Synapse-style Annotations object. Arguments: @@ -125,7 +125,7 @@ def is_synapse_annotations(annotations: typing.Mapping) -> bool: return annotations.keys() >= {"id", "etag", "annotations"} -def _annotation_value_list_element_type(annotation_values: typing.List): +def _annotation_value_list_element_type(annotation_values: list): if not annotation_values: raise ValueError("annotations value list can not be empty") @@ -229,11 +229,11 @@ def to_submission_status_annotations(annotations, is_private=True): def to_submission_annotations( - id: typing.Union[str, int], + id: Union[str, int], etag: str, - annotations: typing.Dict[str, typing.Any], - logger: typing.Optional[typing.Any] = None, -) -> typing.Dict[str, typing.Any]: + annotations: dict[str, Any], + logger: Optional[Any] = None, +) -> dict[str, Any]: """ Converts a normal dictionary to the format used for submission annotations, which is different from the format used to annotate entities. @@ -468,9 +468,9 @@ class Annotations(dict): def __init__( self, - id: typing.Union[str, int, Entity], + id: Union[str, int, Entity], etag: str, - values: typing.Dict = None, + values: dict = None, **kwargs, ): """ @@ -525,7 +525,7 @@ def etag(self, value): self._etag = str(value) -def to_synapse_annotations(annotations: Annotations) -> typing.Dict[str, typing.Any]: +def to_synapse_annotations(annotations: Annotations) -> dict[str, Any]: """Transforms a simple flat dictionary to a Synapse-style Annotation object. See the [Synapse API](https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/annotation/v2/Annotations.html) documentation for more information on Synapse-style Annotation objects. @@ -581,7 +581,7 @@ def _convert_to_annotations_list(annotations): def from_synapse_annotations( - raw_annotations: typing.Dict[str, typing.Any] + raw_annotations: dict[str, Any] ) -> Annotations: """Transforms a Synapse-style Annotation object to a simple flat dictionary. From b75c5662dfefae60103cad4cdded54ad797075b2 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 15:43:57 -0500 Subject: [PATCH 49/60] style --- synapseclient/annotations.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index 4e05a5730..cb6c6d3c5 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -580,9 +580,7 @@ def _convert_to_annotations_list(annotations): return nested_annos -def from_synapse_annotations( - raw_annotations: dict[str, Any] -) -> Annotations: +def from_synapse_annotations(raw_annotations: dict[str, Any]) -> Annotations: """Transforms a Synapse-style Annotation object to a simple flat dictionary. Arguments: From be4278a716c8f16eccca21cdddc5c8c583b6252e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 16:08:12 -0500 Subject: [PATCH 50/60] add API reference links to _fetch_latest_entity --- synapseclient/models/submission.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index fa3270816..89be6e03f 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -439,6 +439,13 @@ async def _fetch_latest_entity( """ Fetch the latest entity information from Synapse. + + + If the object is a DockerRepository, this will also fetch the DockerCommit object with the latest createdOn value + and attach it to the final dictionary: + + + Arguments: synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created From 5a70bd1a078e26f13a297a68fd89df27352d535d Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 16:34:07 -0500 Subject: [PATCH 51/60] type hint should be logging.Logger instance (generic or Synapse client) --- synapseclient/annotations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index cb6c6d3c5..43e6262b0 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -72,6 +72,7 @@ import collections import datetime +import logging from typing import Any, Callable, Mapping, Optional, Union from deprecated import deprecated @@ -232,7 +233,7 @@ def to_submission_annotations( id: Union[str, int], etag: str, annotations: dict[str, Any], - logger: Optional[Any] = None, + logger: Optional[logging.Logger] = None, ) -> dict[str, Any]: """ Converts a normal dictionary to the format used for submission annotations, which is different from the format From a54ade1fe02ec11d38954355cb1de935cddd270b Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 16:35:07 -0500 Subject: [PATCH 52/60] explicit import to follow the other imports --- synapseclient/annotations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index 43e6262b0..fb3bcafde 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -72,8 +72,8 @@ import collections import datetime -import logging from typing import Any, Callable, Mapping, Optional, Union +from logging import Logger from deprecated import deprecated @@ -233,7 +233,7 @@ def to_submission_annotations( id: Union[str, int], etag: str, annotations: dict[str, Any], - logger: Optional[logging.Logger] = None, + logger: Optional[Logger] = None, ) -> dict[str, Any]: """ Converts a normal dictionary to the format used for submission annotations, which is different from the format From 890005e6af362bf6862dd56b006bf4801f43b968 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 16:36:04 -0500 Subject: [PATCH 53/60] import order --- synapseclient/annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index fb3bcafde..3e68d990c 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -72,8 +72,8 @@ import collections import datetime -from typing import Any, Callable, Mapping, Optional, Union from logging import Logger +from typing import Any, Callable, Mapping, Optional, Union from deprecated import deprecated From 0b0fa0c69ebf8a3dcf6214931c5ee9e7f397a93b Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 4 Dec 2025 16:48:22 -0500 Subject: [PATCH 54/60] minimize indenting by using if not: --- synapseclient/models/submission.py | 39 +++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 89be6e03f..6a5d8f230 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -583,27 +583,27 @@ async def create_submission_example(): ``` """ - if self.entity_id: - entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + if not self.entity_id: + raise ValueError("entity_id is required to create a submission") - self.entity_etag = entity_info.get("etag") + entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) - if ( - entity_info.get("concreteType") - == "org.sagebionetworks.repo.model.FileEntity" - ): - self.version_number = entity_info.get("versionNumber") - elif ( - entity_info.get("concreteType") - == "org.sagebionetworks.repo.model.docker.DockerRepository" - ): - self.docker_repository_name = entity_info.get("repositoryName") - self.docker_digest = entity_info.get("digest") - self.docker_tag = entity_info.get("tag") - # All docker repositories are assigned version number 1, even if they have multiple tags - self.version_number = 1 - else: - raise ValueError("entity_id is required to create a submission") + self.entity_etag = entity_info.get("etag") + + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.FileEntity" + ): + self.version_number = entity_info.get("versionNumber") + elif ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + self.docker_repository_name = entity_info.get("repositoryName") + self.docker_digest = entity_info.get("digest") + self.docker_tag = entity_info.get("tag") + # All docker repositories are assigned version number 1, even if they have multiple tags + self.version_number = 1 if not self.entity_etag: raise ValueError("Unable to fetch etag for entity") @@ -615,6 +615,7 @@ async def create_submission_example(): request_body, self.entity_etag, synapse_client=synapse_client ) self.fill_from_dict(response) + return self @otel_trace_method( From 96f2dd766e56cde1f0b777142a1f45274873b7ea Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 10:04:55 -0500 Subject: [PATCH 55/60] raise ValueError(no etag for entity) sooner --- synapseclient/models/submission.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 6a5d8f230..6813449fd 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -590,6 +590,9 @@ async def create_submission_example(): self.entity_etag = entity_info.get("etag") + if not self.entity_etag: + raise ValueError("Unable to fetch etag for entity") + if ( entity_info.get("concreteType") == "org.sagebionetworks.repo.model.FileEntity" @@ -605,9 +608,6 @@ async def create_submission_example(): # All docker repositories are assigned version number 1, even if they have multiple tags self.version_number = 1 - if not self.entity_etag: - raise ValueError("Unable to fetch etag for entity") - # Build the request body now that all the necessary dataclass attributes are set request_body = self.to_synapse_request() From acedf2646beb2ebd4d0f56e4f7d2f858ccccbd3e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 10:10:05 -0500 Subject: [PATCH 56/60] remove docker submission async integration test --- .../models/async/test_submission_async.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index f7cb4fdda..11afafcad 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -124,25 +124,6 @@ async def test_store_submission_without_evaluation_id_async(self, test_file: Fil with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): await submission.store_async(synapse_client=self.syn) - # async def test_store_submission_with_docker_repository_async( - # self, test_evaluation: Evaluation - # ): - # # GIVEN we would need a Docker repository entity (mocked for this test) - # # This test demonstrates the expected behavior for Docker repository submissions - - # # WHEN I create a submission for a Docker repository entity using async method - # # TODO: This would require a real Docker repository entity in a full integration test - # submission = Submission( - # entity_id="syn123456789", # Would be a Docker repository ID - # evaluation_id=test_evaluation.id, - # name=f"Docker Submission {uuid.uuid4()}", - # ) - - # # THEN the submission should handle Docker-specific attributes - # # (This test would need to be expanded with actual Docker repository setup) - # assert submission.entity_id == "syn123456789" - # assert submission.evaluation_id == test_evaluation.id - class TestSubmissionRetrievalAsync: @pytest.fixture(autouse=True, scope="function") From 75bad9698e90d943312c2224ec8b4474e37089a3 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 10:38:55 -0500 Subject: [PATCH 57/60] link Jira ticket to TODO item --- .../synapseclient/models/synchronous/test_submission.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index e89800042..473232422 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -127,6 +127,7 @@ async def test_store_submission_with_docker_repository( # WHEN I create a submission for a Docker repository entity # TODO: This would require a real Docker repository entity in a full integration test + # Jira: https://sagebionetworks.jira.com/browse/SYNPY-1720 submission = Submission( entity_id="syn123456789", # Would be a Docker repository ID evaluation_id=test_evaluation.id, From ea5242489d96f0f16c27f4b88bb763c3c7b11236 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 10:50:34 -0500 Subject: [PATCH 58/60] remove old cancel submission test --- .../models/synchronous/test_submission.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index 473232422..75fbae9e5 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -479,25 +479,6 @@ async def test_file( finally: os.unlink(temp_file_path) - # TODO: Add with SubmissionStatus model tests - # async def test_cancel_submission_successfully( - # self, test_evaluation: Evaluation, test_file: File - # ): - # # GIVEN a submission - # submission = Submission( - # entity_id=test_file.id, - # evaluation_id=test_evaluation.id, - # name=f"Test Submission for Cancellation {uuid.uuid4()}", - # ) - # created_submission = submission.store(synapse_client=self.syn) - # self.schedule_for_cleanup(created_submission.id) - - # # WHEN I cancel the submission - # cancelled_submission = created_submission.cancel(synapse_client=self.syn) - - # # THEN the submission should be cancelled - # assert cancelled_submission.id == created_submission.id - async def test_cancel_submission_without_id(self): # WHEN I try to cancel a submission without an ID submission = Submission(entity_id="syn123", evaluation_id="456") From f08a6add9fd0b130246db0adf80e64087ea54ef1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 10:52:14 -0500 Subject: [PATCH 59/60] remove old cancel submission test (async) --- .../models/async/test_submission_async.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 11afafcad..ec6757f8e 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -472,24 +472,6 @@ async def test_file( finally: os.unlink(temp_file_path) - # async def test_cancel_submission_successfully_async( - # self, test_evaluation: Evaluation, test_file: File - # ): - # # GIVEN a submission created with async method - # submission = Submission( - # entity_id=test_file.id, - # evaluation_id=test_evaluation.id, - # name=f"Test Submission for Cancellation {uuid.uuid4()}", - # ) - # created_submission = await submission.store_async(synapse_client=self.syn) - # self.schedule_for_cleanup(created_submission.id) - - # # WHEN I cancel the submission using async method - # cancelled_submission = await created_submission.cancel_async(synapse_client=self.syn) - - # # THEN the submission should be cancelled - # assert cancelled_submission.id == created_submission.id - async def test_cancel_submission_without_id_async(self): # WHEN I try to cancel a submission without an ID using async method submission = Submission(entity_id="syn123", evaluation_id="456") From e5189d6fc36928e2b925cb447dc9444512222c78 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 5 Dec 2025 11:42:48 -0500 Subject: [PATCH 60/60] assert the SubmissionStatus object has not changed --- .../models/async/test_submission_status_async.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index bbad7997b..e02785e50 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -385,6 +385,10 @@ async def test_store_submission_status_without_changes( """Test that storing a submission status without changes shows warning async.""" # GIVEN a submission status that hasn't been modified # (it already has _last_persistent_instance set from get()) + assert ( + test_submission_status._last_persistent_instance == test_submission_status + ) + assert not test_submission_status.has_changed # WHEN I try to store it without making changes result = await test_submission_status.store_async(synapse_client=self.syn)