From a061fb4f64c0631d73338e21a1cc543693424d1a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 29 Sep 2025 16:34:45 -0400 Subject: [PATCH 01/46] tdd initial tests --- .../models/async/test_submission_async.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 tests/integration/synapseclient/models/async/test_submission_async.py diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py new file mode 100644 index 000000000..2b856d4e5 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -0,0 +1,199 @@ +def test_create_submission_async(): + # WHEN an evaluation is retrieved + evaluation = Evaluation(id=evaluation_id).get() + + # AND an entity is retrieved + file = File(name="test.txt", parentId=project.id).get() + + # THEN the entity can be submitted to the evaluation + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + +def test_get_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is retrieved by ID + retrieved_submission = Submission(id=submission.id).get() + + # THEN the retrieved submission matches the created one + assert retrieved_submission.id == submission.id + assert retrieved_submission.name == submission.name + + # AND the user_id matches the current user + current_user = syn.getUserProfile()().id + assert retrieved_submission.user_id == current_user + +def test_get_evaluation_submissions_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN submissions are retrieved for the evaluation + submissions = Submission.get_evaluation_submissions(evaluation.id) + + # THEN the submissions list is not empty + assert len(submissions) > 0 + + # AND each submission belongs to the evaluation + for submission in submissions: + assert submission.evaluation_id == evaluation.id + +def test_get_user_submissions_async(): + # GIVEN a user has made submissions + current_user = syn.getUserProfile()().id + + # WHEN submissions are retrieved for the user + submissions = Submission.get_user_submissions(current_user) + + # THEN the submissions list is not empty + assert len(submissions) > 0 + + # AND each submission belongs to the user + for submission in submissions: + assert submission.user_id == current_user + +def test_get_submission_count_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission count is retrieved for the evaluation + count = Submission.get_submission_count(evaluation.id) + + # THEN the count is greater than zero + assert count > 0 + +def test_delete_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is deleted + submission.delete() + + # THEN retrieving the submission should raise an error + try: + Submission(id=submission.id).get() + assert False, "Expected an error when retrieving a deleted submission" + except SynapseError as e: + assert e.response.status_code == 404 + +def test_cancel_submission_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission is canceled + submission.cancel() + + # THEN the submission status should be 'CANCELED' + updated_submission = Submission(id=submission.id).get() + assert updated_submission.status == 'CANCELED' + +def test_get_submission_status_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission status is retrieved + status = submission.get_status() + + # THEN the status should be 'RECEIVED' + assert status == 'RECEIVED' + +def test_update_submission_status_async(): + # GIVEN a submission has been created + submission = Submission( + name="Test Submission", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the submission status is retrieved + status = submission.get_status() + assert status != 'SCORED' + + # AND the submission status is updated to 'SCORED' + submission.update_status('SCORED') + + # THEN the submission status should be 'SCORED' + updated_submission = Submission(id=submission.id).get() + assert updated_submission.status == 'SCORED' + +def test_get_evaluation_submission_statuses_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission statuses are retrieved for the evaluation + statuses = Submission.get_evaluation_submission_statuses(evaluation.id) + + # THEN the statuses list is not empty + assert len(statuses) > 0 + +def test_batch_update_statuses_async(): + # GIVEN multiple submissions have been created + submission1 = Submission( + name="Test Submission 1", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + submission2 = Submission( + name="Test Submission 2", + entity_id=file.id, + evaluation_id=evaluation.id, + version_number=1 + ).store() + + # WHEN the statuses of the submissions are batch updated to 'SCORED' + Submission.batch_update_statuses( + [submission1.id, submission2.id], + 'SCORED' + ) + + # THEN each submission status should be 'SCORED' + updated_submission1 = Submission(id=submission1.id).get() + updated_submission2 = Submission(id=submission2.id).get() + assert updated_submission1.status == 'SCORED' + assert updated_submission2.status == 'SCORED' + +def test_get_evaluation_submission_bundles_async(): + # GIVEN an evaluation has submissions + evaluation = Evaluation(id=evaluation_id).get() + + # WHEN the submission bundles are retrieved for the evaluation + bundles = Submission.get_evaluation_submission_bundles(evaluation.id) + + # THEN the bundles list is not empty + assert len(bundles) > 0 + +def test_get_user_submission_bundles_async(): + # GIVEN a user has made submissions + current_user = syn.getUserProfile()().id + + # WHEN the submission bundles are retrieved for the user + bundles = Submission.get_user_submission_bundles(current_user) + + # THEN the bundles list is not empty + assert len(bundles) > 0 From d85a9d7aebe27b71c2e4a78839006b2728c07519 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 29 Sep 2025 16:43:57 -0400 Subject: [PATCH 02/46] initial intro of dataclasses --- synapseclient/models/submission.py | 209 +++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 synapseclient/models/submission.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py new file mode 100644 index 000000000..0af9359db --- /dev/null +++ b/synapseclient/models/submission.py @@ -0,0 +1,209 @@ +from collections import OrderedDict +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Dict, List, Optional, Protocol, TypeVar, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.constants import concrete_types +from synapseclient.core.utils import delete_none_keys +from synapseclient.models import Activity, Annotations +from synapseclient.models.mixins.access_control import AccessControllable +from synapseclient.models.mixins.table_components import ( + DeleteMixin, + GetMixin, +) + + +class SubmissionSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for Submission operations.""" + + def get( + self, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a Submission from Synapse. + + Arguments: + include_activity: Whether to include the activity in the returned submission. + Defaults to False. Setting this to True will include the activity + record associated with this submission. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Example: Retrieving a submission by ID. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234").get() + print(submission) + ``` + """ + return self + + def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Example: Delete a submission. +   + + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + submission.delete() + print("Deleted Submission.") + ``` + """ + pass + + +@dataclass +@async_to_sync +class Submission( + SubmissionSynchronousProtocol, + AccessControllable, + GetMixin, + DeleteMixin, +): + """A `Submission` object represents a Synapse Submission, which is created when a user + submits an entity to an evaluation queue. + + + Attributes: + id: The unique ID of this Submission. + user_id: The ID of the user that submitted this Submission. + submitter_alias: The name of the user that submitted this Submission. + entity_id: The ID of the entity being submitted. + version_number: The version number of the entity at submission. + evaluation_id: The ID of the Evaluation to which this Submission belongs. + name: The name of this Submission. + created_on: The date this Submission was created. + team_id: The ID of the team that submitted this submission (if it's a team submission). + contributors: User IDs of team members who contributed to this submission (if it's a team submission). + submission_status: The status of this Submission. + entity_bundle_json: The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + docker_repository_name: For Docker repositories, the repository name. + docker_digest: For Docker repositories, the digest of the submitted Docker image. + activity: The Activity model represents the main record of Provenance in Synapse. + + Example: Retrieve a Submission. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn123456").get() + print(submission) + ``` + """ + + id: Optional[str] = None + """ + The unique ID of this Submission. + """ + + user_id: Optional[str] = None + """ + The ID of the user that submitted this Submission. + """ + + submitter_alias: Optional[str] = None + """ + The name of the user that submitted this Submission. + """ + + entity_id: Optional[str] = None + """ + The ID of the entity being submitted. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the entity at submission. + """ + + evaluation_id: Optional[str] = None + """ + The ID of the Evaluation to which this Submission belongs. + """ + + name: Optional[str] = None + """ + The name of this Submission. + """ + + created_on: Optional[str] = field(default=None, compare=False) + """ + The date this Submission was created. + """ + + team_id: Optional[str] = None + """ + The ID of the team that submitted this submission (if it's a team submission). + """ + + contributors: List[str] = field(default_factory=list) + """ + User IDs of team members who contributed to this submission (if it's a team submission). + """ + + submission_status: Optional[Dict] = None + """ + The status of this Submission. + """ + + entity_bundle_json: Optional[str] = None + """ + The bundled entity information at submission. This includes the entity, annotations, + file handles, and other metadata. + """ + + docker_repository_name: Optional[str] = None + """ + For Docker repositories, the repository name. + """ + + docker_digest: Optional[str] = None + """ + For Docker repositories, the digest of the submitted Docker image. + """ + + activity: Optional[Activity] = field(default=None, compare=False) + """The Activity model represents the main record of Provenance in Synapse. It is + analogous to the Activity defined in the + [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" + + _last_persistent_instance: Optional["Submission"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" From 50498b7ec7953d20270fea85f86f3286621eead4 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 30 Sep 2025 17:58:11 -0400 Subject: [PATCH 03/46] expose api services for submission object --- synapseclient/api/submission_services.py | 228 +++++++++++++++++++++++ 1 file changed, 228 insertions(+) create mode 100644 synapseclient/api/submission_services.py diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py new file mode 100644 index 000000000..226b04056 --- /dev/null +++ b/synapseclient/api/submission_services.py @@ -0,0 +1,228 @@ +# TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. + +import json +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: + from synapseclient import Synapse + + +async def create_submission(request_body: dict, synapse_client: Optional["Synapse"] = None) -> dict: + """ + Creates a Submission and sends a submission notification email to the submitter's team members. + + + + Arguments: + request_body: The request body to send to the server. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = "/evaluation/submission" + + response = await client.rest_post_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] = None) -> dict: + """ + Retrieves a Submission by its ID. + + + + Arguments: + submission_id: The ID of the submission to fetch. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + response = await client.rest_get_async(uri) + + return response + + +async def get_evaluation_submissions( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + # TODO: Support pagination in the return type. + A response JSON containing a paginated list of submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/all" + query_params = { + "limit": limit, + "offset": offset + } + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission" + query_params = { + "limit": limit, + "offset": offset + } + + if user_id: + query_params["userId"] = user_id + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/count" + query_params = {} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def delete_submission( + submission_id: str, + synapse_client: Optional["Synapse"] = None +) -> None: + """ + Deletes a Submission and its SubmissionStatus. + + + + Arguments: + submission_id: The ID of the submission to delete. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + await client.rest_delete_async(uri) + + +async def cancel_submission( + submission_id: str, + synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Cancels a Submission. Only the user who created the Submission may cancel it. + + + + Arguments: + submission_id: The ID of the submission to cancel. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + The canceled Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/cancellation" + + response = await client.rest_put_async(uri) + + return response \ No newline at end of file From 94cedc62619dde1c5d6aa73dd0f212ee5ed2f975 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 30 Sep 2025 17:59:31 -0400 Subject: [PATCH 04/46] style and update docstring --- synapseclient/api/submission_services.py | 86 +++++++++---------- synapseclient/models/submission.py | 7 +- .../models/async/test_submission_async.py | 47 ++++++---- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index 226b04056..385423308 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -7,7 +7,9 @@ from synapseclient import Synapse -async def create_submission(request_body: dict, synapse_client: Optional["Synapse"] = None) -> dict: +async def create_submission( + request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: """ Creates a Submission and sends a submission notification email to the submitter's team members. @@ -23,13 +25,15 @@ async def create_submission(request_body: dict, synapse_client: Optional["Synaps client = Synapse.get_client(synapse_client=synapse_client) uri = "/evaluation/submission" - + response = await client.rest_post_async(uri, body=json.dumps(request_body)) return response -async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] = None) -> dict: +async def get_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: """ Retrieves a Submission by its ID. @@ -39,7 +43,7 @@ async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] submission_id: The ID of the submission to fetch. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: The requested Submission. """ @@ -48,18 +52,18 @@ async def get_submission(submission_id: str, synapse_client: Optional["Synapse"] client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}" - + response = await client.rest_get_async(uri) return response async def get_evaluation_submissions( - evaluation_id: str, + evaluation_id: str, status: Optional[str] = None, limit: int = 20, offset: int = 0, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Retrieves all Submissions for a specified Evaluation queue. @@ -68,14 +72,14 @@ async def get_evaluation_submissions( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, + status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. + offset: The offset index determines where this page will start from. An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: # TODO: Support pagination in the return type. A response JSON containing a paginated list of submissions for the evaluation queue. @@ -85,14 +89,11 @@ async def get_evaluation_submissions( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = { - "limit": limit, - "offset": offset - } - + query_params = {"limit": limit, "offset": offset} + if status: query_params["status"] = status - + response = await client.rest_get_async(uri, **query_params) return response @@ -103,7 +104,7 @@ async def get_user_submissions( user_id: Optional[str] = None, limit: int = 20, offset: int = 0, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Retrieves Submissions for a specified Evaluation queue and user. @@ -113,14 +114,14 @@ async def get_user_submissions( Arguments: evaluation_id: The ID of the evaluation queue. - user_id: Optionally specify the ID of the user whose submissions will be returned. + user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. + offset: The offset index determines where this page will start from. An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: A response JSON containing a paginated list of user submissions for the evaluation queue. """ @@ -129,14 +130,11 @@ async def get_user_submissions( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission" - query_params = { - "limit": limit, - "offset": offset - } - + query_params = {"limit": limit, "offset": offset} + if user_id: query_params["userId"] = user_id - + response = await client.rest_get_async(uri, **query_params) return response @@ -145,7 +143,7 @@ async def get_user_submissions( async def get_submission_count( evaluation_id: str, status: Optional[str] = None, - synapse_client: Optional["Synapse"] = None + synapse_client: Optional["Synapse"] = None, ) -> dict: """ Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. @@ -154,11 +152,11 @@ async def get_submission_count( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, + status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: A response JSON containing the submission count. """ @@ -168,18 +166,17 @@ async def get_submission_count( uri = f"/evaluation/{evaluation_id}/submission/count" query_params = {} - + if status: query_params["status"] = status - + response = await client.rest_get_async(uri, **query_params) return response async def delete_submission( - submission_id: str, - synapse_client: Optional["Synapse"] = None + submission_id: str, synapse_client: Optional["Synapse"] = None ) -> None: """ Deletes a Submission and its SubmissionStatus. @@ -188,7 +185,7 @@ async def delete_submission( Arguments: submission_id: The ID of the submission to delete. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. """ from synapseclient import Synapse @@ -196,13 +193,12 @@ async def delete_submission( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}" - + await client.rest_delete_async(uri) async def cancel_submission( - submission_id: str, - synapse_client: Optional["Synapse"] = None + submission_id: str, synapse_client: Optional["Synapse"] = None ) -> dict: """ Cancels a Submission. Only the user who created the Submission may cancel it. @@ -211,18 +207,18 @@ async def cancel_submission( Arguments: submission_id: The ID of the submission to cancel. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - + Returns: - The canceled Submission. + The Submission response object for the canceled submission as a JSON dict. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/cancellation" - + response = await client.rest_put_async(uri) - return response \ No newline at end of file + return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 0af9359db..4e65fc07b 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -11,10 +11,7 @@ from synapseclient.core.utils import delete_none_keys from synapseclient.models import Activity, Annotations from synapseclient.models.mixins.access_control import AccessControllable -from synapseclient.models.mixins.table_components import ( - DeleteMixin, - GetMixin, -) +from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin class SubmissionSynchronousProtocol(Protocol): @@ -112,7 +109,7 @@ class Submission( docker_repository_name: For Docker repositories, the repository name. docker_digest: For Docker repositories, the digest of the submitted Docker image. activity: The Activity model represents the main record of Provenance in Synapse. - + Example: Retrieve a Submission. ```python from synapseclient import Synapse diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 2b856d4e5..da5fdc46f 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -10,16 +10,17 @@ def test_create_submission_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() + def test_get_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is retrieved by ID @@ -33,6 +34,7 @@ def test_get_submission_async(): current_user = syn.getUserProfile()().id assert retrieved_submission.user_id == current_user + def test_get_evaluation_submissions_async(): # GIVEN an evaluation has submissions evaluation = Evaluation(id=evaluation_id).get() @@ -47,6 +49,7 @@ def test_get_evaluation_submissions_async(): for submission in submissions: assert submission.evaluation_id == evaluation.id + def test_get_user_submissions_async(): # GIVEN a user has made submissions current_user = syn.getUserProfile()().id @@ -61,6 +64,7 @@ def test_get_user_submissions_async(): for submission in submissions: assert submission.user_id == current_user + def test_get_submission_count_async(): # GIVEN an evaluation has submissions evaluation = Evaluation(id=evaluation_id).get() @@ -71,13 +75,14 @@ def test_get_submission_count_async(): # THEN the count is greater than zero assert count > 0 + def test_delete_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is deleted @@ -90,13 +95,14 @@ def test_delete_submission_async(): except SynapseError as e: assert e.response.status_code == 404 + def test_cancel_submission_async(): # GIVEN a submission has been created submission = Submission( name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission is canceled @@ -104,7 +110,8 @@ def test_cancel_submission_async(): # THEN the submission status should be 'CANCELED' updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'CANCELED' + assert updated_submission.status == "CANCELED" + def test_get_submission_status_async(): # GIVEN a submission has been created @@ -112,14 +119,15 @@ def test_get_submission_status_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission status is retrieved status = submission.get_status() # THEN the status should be 'RECEIVED' - assert status == 'RECEIVED' + assert status == "RECEIVED" + def test_update_submission_status_async(): # GIVEN a submission has been created @@ -127,19 +135,20 @@ def test_update_submission_status_async(): name="Test Submission", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the submission status is retrieved status = submission.get_status() - assert status != 'SCORED' + assert status != "SCORED" # AND the submission status is updated to 'SCORED' - submission.update_status('SCORED') + submission.update_status("SCORED") # THEN the submission status should be 'SCORED' updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == 'SCORED' + assert updated_submission.status == "SCORED" + def test_get_evaluation_submission_statuses_async(): # GIVEN an evaluation has submissions @@ -151,32 +160,31 @@ def test_get_evaluation_submission_statuses_async(): # THEN the statuses list is not empty assert len(statuses) > 0 + def test_batch_update_statuses_async(): # GIVEN multiple submissions have been created submission1 = Submission( name="Test Submission 1", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() submission2 = Submission( name="Test Submission 2", entity_id=file.id, evaluation_id=evaluation.id, - version_number=1 + version_number=1, ).store() # WHEN the statuses of the submissions are batch updated to 'SCORED' - Submission.batch_update_statuses( - [submission1.id, submission2.id], - 'SCORED' - ) + Submission.batch_update_statuses([submission1.id, submission2.id], "SCORED") # THEN each submission status should be 'SCORED' updated_submission1 = Submission(id=submission1.id).get() updated_submission2 = Submission(id=submission2.id).get() - assert updated_submission1.status == 'SCORED' - assert updated_submission2.status == 'SCORED' + assert updated_submission1.status == "SCORED" + assert updated_submission2.status == "SCORED" + def test_get_evaluation_submission_bundles_async(): # GIVEN an evaluation has submissions @@ -188,6 +196,7 @@ def test_get_evaluation_submission_bundles_async(): # THEN the bundles list is not empty assert len(bundles) > 0 + def test_get_user_submission_bundles_async(): # GIVEN a user has made submissions current_user = syn.getUserProfile()().id From cd19e0ca831faeed09ca68f2490e71f53852d063 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 10 Oct 2025 10:32:41 -0400 Subject: [PATCH 05/46] add submission and submissionstatus models --- synapseclient/models/submission_status.py | 637 ++++++++++++++++++++++ synapseclient/models/submissionstatus.py | 0 2 files changed, 637 insertions(+) create mode 100644 synapseclient/models/submission_status.py create mode 100644 synapseclient/models/submissionstatus.py diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py new file mode 100644 index 000000000..30cfc0568 --- /dev/null +++ b/synapseclient/models/submission_status.py @@ -0,0 +1,637 @@ +from dataclasses import dataclass, field +from datetime import date, datetime +from typing import Dict, List, Optional, Protocol, Union + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.core.utils import delete_none_keys +from synapseclient.models import Annotations +from synapseclient.models.mixins.access_control import AccessControllable + + +class SubmissionStatusSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionStatus operations.""" + + def get( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Example: Retrieving a submission status by ID. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + print(status) + ``` + """ + return self + + def store( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus instance. + + Example: Update a submission status. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = SubmissionStatus(id="syn1234").get() + status.status = "SCORED" + status = status.store() + print("Updated SubmissionStatus.") + ``` + """ + return self + + @staticmethod + def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return {} + + @staticmethod + def batch_update_submission_statuses( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + return {} + + +@dataclass +@async_to_sync +class SubmissionStatus( + SubmissionStatusSynchronousProtocol, + AccessControllable, +): + """A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + + Attributes: + id: The unique, immutable Synapse ID of the Submission. + etag: Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + modified_on: The date on which this SubmissionStatus was last modified. + status: The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + score: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + report: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + annotations: Primary container object for Annotations on a Synapse object. + submission_annotations: Annotations are additional key-value pair metadata that are associated with an object. + entity_id: The Synapse ID of the Entity in this Submission. + version_number: The version number of the Entity in this Submission. + status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + cancel_requested: Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + + Example: Retrieve and update a SubmissionStatus. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get a submission status + status = SubmissionStatus(id="syn123456").get() + + # Update the status + status.status = "SCORED" + status.submission_annotations = {"score": [85.5], "feedback": ["Good work!"]} + status = status.store() + print(status) + ``` + """ + + id: Optional[str] = None + """ + The unique, immutable Synapse ID of the Submission. + """ + + etag: Optional[str] = field(default=None, compare=False) + """ + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. The eTag changes every time a SubmissionStatus is updated; + it is used to detect when a client's copy of a SubmissionStatus is out-of-date. + """ + + modified_on: Optional[str] = field(default=None, compare=False) + """ + The date on which this SubmissionStatus was last modified. + """ + + status: Optional[str] = None + """ + The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). + """ + + score: Optional[float] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + report: Optional[str] = None + """ + This field is deprecated and should not be used. Use the 'submission_annotations' field instead. + """ + + annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Primary container object for Annotations on a Synapse object.""" + + submission_annotations: Optional[ + Dict[ + str, + Union[ + List[str], + List[bool], + List[float], + List[int], + List[date], + List[datetime], + ], + ] + ] = field(default_factory=dict, compare=False) + """Annotations are additional key-value pair metadata that are associated with an object.""" + + entity_id: Optional[str] = None + """ + The Synapse ID of the Entity in this Submission. + """ + + version_number: Optional[int] = field(default=None, compare=False) + """ + The version number of the Entity in this Submission. + """ + + status_version: Optional[int] = field(default=None, compare=False) + """ + A version of the status, auto-generated and auto-incremented by the system and read-only to the client. + """ + + can_cancel: Optional[bool] = field(default=False, compare=False) + """ + Can this submission be cancelled? By default, this will be set to False. Users can read this value. + Only the queue's scoring application can change this value. + """ + + cancel_requested: Optional[bool] = field(default=False, compare=False) + """ + Has user requested to cancel this submission? By default, this will be set to False. + Submission owner can read and request to change this value. + """ + + _last_persistent_instance: Optional["SubmissionStatus"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + def has_changed(self) -> bool: + """Determines if the object has been changed and needs to be updated in Synapse.""" + return ( + not self._last_persistent_instance or self._last_persistent_instance != self + ) + + def _set_last_persistent_instance(self) -> None: + """Stash the last time this object interacted with Synapse. This is used to + determine if the object has been changed and needs to be updated in Synapse.""" + import dataclasses + del self._last_persistent_instance + self._last_persistent_instance = dataclasses.replace(self) + + def fill_from_dict( + self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] + ) -> "SubmissionStatus": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_status: The response from the REST API. + + Returns: + The SubmissionStatus object. + """ + self.id = synapse_submission_status.get("id", None) + self.etag = synapse_submission_status.get("etag", None) + self.modified_on = synapse_submission_status.get("modifiedOn", None) + self.status = synapse_submission_status.get("status", None) + self.score = synapse_submission_status.get("score", None) + self.report = synapse_submission_status.get("report", None) + self.entity_id = synapse_submission_status.get("entityId", None) + self.version_number = synapse_submission_status.get("versionNumber", None) + self.status_version = synapse_submission_status.get("statusVersion", None) + self.can_cancel = synapse_submission_status.get("canCancel", False) + self.cancel_requested = synapse_submission_status.get("cancelRequested", False) + + # Handle annotations + annotations_dict = synapse_submission_status.get("annotations", {}) + if annotations_dict: + self.annotations = Annotations.from_dict(annotations_dict) + + # Handle submission annotations + submission_annotations_dict = synapse_submission_status.get("submissionAnnotations", {}) + if submission_annotations_dict: + self.submission_annotations = Annotations.from_dict(submission_annotations_dict) + + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Get: {self.id}" + ) + async def get_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Retrieve a SubmissionStatus from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus instance retrieved from Synapse. + + Raises: + ValueError: If the submission status does not have an ID to get. + + Example: Retrieving a submission status by ID + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + status = await SubmissionStatus(id="syn1234").get_async() + print(status) + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to get.") + + response = await submission_services.get_submission_status( + submission_id=self.id, + synapse_client=synapse_client + ) + + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Store: {self.id if self.id else 'new_status'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "SubmissionStatus": + """ + Store (update) the SubmissionStatus in Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object. + + Raises: + ValueError: If the submission status is missing required fields. + + Example: Update a submission status + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get existing status + status = await SubmissionStatus(id="syn1234").get_async() + + # Update fields + status.status = "SCORED" + status.submission_annotations = {"score": [85.5]} + + # Store the update + status = await status.store_async() + print(f"Updated status: {status.status}") + ``` + """ + if not self.id: + raise ValueError("The submission status must have an ID to update.") + + # Prepare request body + request_body = delete_none_keys({ + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + }) + + # Add annotations if present + if self.annotations: + # Convert annotations to the format expected by the API + request_body["annotations"] = self.annotations + + # Add submission annotations if present + if self.submission_annotations: + # Convert submission annotations to the format expected by the API + request_body["submissionAnnotations"] = self.submission_annotations + + # Update the submission status using the service + response = await submission_services.update_submission_status( + submission_id=self.id, + request_body=request_body, + synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + self._set_last_persistent_instance() + return self + + @staticmethod + async def get_all_submission_statuses_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Example: Getting all submission statuses for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + response = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(response['results'])} submission statuses") + ``` + """ + return await submission_services.get_all_submission_statuses( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client + ) + + @staticmethod + async def batch_update_submission_statuses_async( + evaluation_id: str, + statuses: List["SubmissionStatus"], + is_first_batch: bool = True, + is_last_batch: bool = True, + batch_token: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + statuses: List of SubmissionStatus objects to update. + is_first_batch: Boolean indicating if this is the first batch in the series. Default True. + is_last_batch: Boolean indicating if this is the last batch in the series. Default True. + batch_token: Token from previous batch response (required for all but first batch). + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Example: Batch update submission statuses + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Prepare list of status updates + statuses = [ + SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), + SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) + ] + + response = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id="9614543", + statuses=statuses, + is_first_batch=True, + is_last_batch=True + ) + print(f"Batch update completed: {response}") + ``` + """ + # Convert SubmissionStatus objects to dictionaries + status_dicts = [] + for status in statuses: + status_dict = delete_none_keys({ + "id": status.id, + "etag": status.etag, + "status": status.status, + "score": status.score, + "report": status.report, + "entityId": status.entity_id, + "versionNumber": status.version_number, + "canCancel": status.can_cancel, + "cancelRequested": status.cancel_requested, + }) + + # Add annotations if present + if status.annotations: + status_dict["annotations"] = status.annotations + + # Add submission annotations if present + if status.submission_annotations: + status_dict["submissionAnnotations"] = status.submission_annotations + + status_dicts.append(status_dict) + + # Prepare the batch request body + request_body = { + "statuses": status_dicts, + "isFirstBatch": is_first_batch, + "isLastBatch": is_last_batch, + } + + # Add batch token if provided (required for all but first batch) + if batch_token: + request_body["batchToken"] = batch_token + + return await submission_services.batch_update_submission_statuses( + evaluation_id=evaluation_id, + request_body=request_body, + synapse_client=synapse_client + ) diff --git a/synapseclient/models/submissionstatus.py b/synapseclient/models/submissionstatus.py new file mode 100644 index 000000000..e69de29bb From 99cba20d98a4cd428233c1666d2e4d251e019321 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 10 Oct 2025 10:43:27 -0400 Subject: [PATCH 06/46] add submission status retrieval and update methods; remove empty submissionstatus file --- synapseclient/api/submission_services.py | 258 +++++++++++++- synapseclient/models/submission.py | 397 +++++++++++++++++++++- synapseclient/models/submission_status.py | 72 ++-- synapseclient/models/submissionstatus.py | 0 4 files changed, 687 insertions(+), 40 deletions(-) delete mode 100644 synapseclient/models/submissionstatus.py diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index 385423308..feea626fc 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -1,7 +1,7 @@ # TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. import json -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: from synapseclient import Synapse @@ -222,3 +222,259 @@ async def cancel_submission( response = await client.rest_put_async(uri) return response + + +async def get_submission_status( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the SubmissionStatus object associated with a specified Submission. + + + + Arguments: + submission_id: The ID of the submission to get the status for. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus object as a JSON dict. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatus. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_get_async(uri) + + return response + + +async def update_submission_status( + submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Updates a SubmissionStatus object. + + + + Arguments: + submission_id: The ID of the SubmissionStatus being updated. + request_body: The SubmissionStatus object to update as a dictionary. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object as a JSON dict. + + Note: + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Each time a SubmissionStatus is updated a new etag will be + issued to the SubmissionStatus. When an update is requested, Synapse will compare + the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. + If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED + (412) response. When this occurs, the caller should fetch the latest copy of the + SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatuses. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/status/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def batch_update_submission_statuses( + evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + request_body: The SubmissionStatusBatch object as a dictionary containing: + - statuses: List of SubmissionStatus objects to update + - isFirstBatch: Boolean indicating if this is the first batch in the series + - isLastBatch: Boolean indicating if this is the last batch in the series + - batchToken: Token from previous batch response (required for all but first batch) + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Note: + To allow upload of more than the maximum batch size (500), the system supports + uploading a series of batches. Synapse employs optimistic concurrency on the + series in the form of a batch token. Each request (except the first) must include + the 'batch token' returned in the response to the previous batch. If another client + begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be + generated and upload must restart from the first batch. + + After the final batch is uploaded, the data for the Evaluation queue will be + mirrored to the tables which support querying. Therefore uploaded data will not + appear in Evaluation queries until after the final batch is successfully uploaded. + + It is the client's responsibility to note in each batch request: + 1. Whether it is the first batch in the series (isFirstBatch) + 2. Whether it is the last batch (isLastBatch) + + For a single batch both flags are set to 'true'. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/statusBatch" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission bundles for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, **query_params) + + return response + + +async def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of the requesting user's submission bundles for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle" + query_params = {"limit": limit, "offset": offset} + + response = await client.rest_get_async(uri, **query_params) + + return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 4e65fc07b..4aa538131 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -1,15 +1,12 @@ -from collections import OrderedDict from dataclasses import dataclass, field -from datetime import date, datetime -from typing import Dict, List, Optional, Protocol, TypeVar, Union +from typing import Dict, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse -from synapseclient.core.async_utils import async_to_sync -from synapseclient.core.constants import concrete_types +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import delete_none_keys -from synapseclient.models import Activity, Annotations from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin @@ -194,7 +191,8 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ - activity: Optional[Activity] = field(default=None, compare=False) + # TODO + activity: Optional[Dict] = field(default=None, compare=False) """The Activity model represents the main record of Provenance in Synapse. It is analogous to the Activity defined in the [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" @@ -204,3 +202,388 @@ class Submission( ) """The last persistent instance of this object. This is used to determine if the object has been changed and needs to be updated in Synapse.""" + + def fill_from_dict( + self, synapse_submission: Dict[str, Union[bool, str, int, List]] + ) -> "Submission": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission: The response from the REST API. + + Returns: + The Submission object. + """ + self.id = synapse_submission.get("id", None) + self.user_id = synapse_submission.get("userId", None) + self.submitter_alias = synapse_submission.get("submitterAlias", None) + self.entity_id = synapse_submission.get("entityId", None) + self.version_number = synapse_submission.get("versionNumber", None) + self.evaluation_id = synapse_submission.get("evaluationId", None) + self.name = synapse_submission.get("name", None) + self.created_on = synapse_submission.get("createdOn", None) + self.team_id = synapse_submission.get("teamId", None) + self.contributors = synapse_submission.get("contributors", []) + self.submission_status = synapse_submission.get("submissionStatus", None) + self.entity_bundle_json = synapse_submission.get("entityBundleJSON", None) + self.docker_repository_name = synapse_submission.get( + "dockerRepositoryName", None + ) + self.docker_digest = synapse_submission.get("dockerDigest", None) + + activity_dict = synapse_submission.get("activity", None) + if activity_dict: + # TODO: Implement Activity class and its fill_from_dict method + self.activity = {} + + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Store: {self.id if self.id else 'new_submission'}" + ) + async def store_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Store the submission in Synapse. This creates a new submission in an evaluation queue. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission object with the ID set. + + Raises: + ValueError: If the submission is missing required fields. + + Example: Creating a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ) + submission = await submission.store_async() + print(submission.id) + ``` + """ + if not self.entity_id: + raise ValueError("The submission must have an entity_id to store.") + if not self.evaluation_id: + raise ValueError("The submission must have an evaluation_id to store.") + + # Prepare request body + request_body = delete_none_keys( + { + "entityId": self.entity_id, + "evaluationId": self.evaluation_id, + "name": self.name, + "teamId": self.team_id, + "contributors": self.contributors if self.contributors else None, + "dockerRepositoryName": self.docker_repository_name, + "dockerDigest": self.docker_digest, + } + ) + + # Create the submission using the service + response = await submission_services.create_submission( + request_body=request_body, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + return self + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Get: {self.id}" + ) + async def get_async( + self, + include_activity: bool = False, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Retrieve a Submission from Synapse. + + Arguments: + include_activity: Whether to include the activity in the returned submission. + Defaults to False. Setting this to True will include the activity + record associated with this submission. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission instance retrieved from Synapse. + + Raises: + ValueError: If the submission does not have an ID to get. + + Example: Retrieving a submission by ID + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = await Submission(id="syn1234").get_async() + print(submission) + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to get.") + + # Get the submission using the service + response = await submission_services.get_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + + # Handle activity if requested + if include_activity and self.activity: + # The activity should be included in the response by default + # but if we need to fetch it separately, we would do it here + pass + + return self + + @staticmethod + async def get_evaluation_submissions_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of submissions for the evaluation queue. + + Example: Getting submissions for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_evaluation_submissions_async( + evaluation_id="9614543", + status="SCORED", + limit=10 + ) + print(f"Found {len(response['results'])} submissions") + ``` + """ + return await submission_services.get_evaluation_submissions( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_user_submissions_async( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + + Example: Getting user submissions + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_user_submissions_async( + evaluation_id="9614543", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + ``` + """ + return await submission_services.get_user_submissions( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + @staticmethod + async def get_submission_count_async( + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + + Example: Getting submission count + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = await Submission.get_submission_count_async( + evaluation_id="9614543", + status="SCORED" + ) + print(f"Found {response['count']} submissions") + ``` + """ + return await submission_services.get_submission_count( + evaluation_id=evaluation_id, status=status, synapse_client=synapse_client + ) + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Delete: {self.id}" + ) + async def delete_submission_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> None: + """ + Delete a Submission from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Raises: + ValueError: If the submission does not have an ID to delete. + + Example: Delete a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + await submission.delete_submission_async() + print("Deleted Submission.") + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to delete.") + + await submission_services.delete_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + @otel_trace_method( + method_to_trace_name=lambda self, **kwargs: f"Submission_Cancel: {self.id}" + ) + async def cancel_submission_async( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Submission": + """ + Cancel a Submission. Only the user who created the Submission may cancel it. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated Submission object. + + Raises: + ValueError: If the submission does not have an ID to cancel. + + Example: Cancel a submission + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + canceled_submission = await submission.cancel_submission_async() + print(f"Canceled submission: {canceled_submission.id}") + ``` + """ + if not self.id: + raise ValueError("The submission must have an ID to cancel.") + + response = await submission_services.cancel_submission( + submission_id=self.id, synapse_client=synapse_client + ) + + # Update this object with the response + self.fill_from_dict(response) + return self diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 30cfc0568..c468f0e79 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -214,7 +214,7 @@ class SubmissionStatus( # Get a submission status status = SubmissionStatus(id="syn123456").get() - + # Update the status status.status = "SCORED" status.submission_annotations = {"score": [85.5], "feedback": ["Good work!"]} @@ -328,6 +328,7 @@ def _set_last_persistent_instance(self) -> None: """Stash the last time this object interacted with Synapse. This is used to determine if the object has been changed and needs to be updated in Synapse.""" import dataclasses + del self._last_persistent_instance self._last_persistent_instance = dataclasses.replace(self) @@ -361,9 +362,13 @@ def fill_from_dict( self.annotations = Annotations.from_dict(annotations_dict) # Handle submission annotations - submission_annotations_dict = synapse_submission_status.get("submissionAnnotations", {}) + submission_annotations_dict = synapse_submission_status.get( + "submissionAnnotations", {} + ) if submission_annotations_dict: - self.submission_annotations = Annotations.from_dict(submission_annotations_dict) + self.submission_annotations = Annotations.from_dict( + submission_annotations_dict + ) return self @@ -405,8 +410,7 @@ async def get_async( raise ValueError("The submission status must have an ID to get.") response = await submission_services.get_submission_status( - submission_id=self.id, - synapse_client=synapse_client + submission_id=self.id, synapse_client=synapse_client ) self.fill_from_dict(response) @@ -445,11 +449,11 @@ async def store_async( # Get existing status status = await SubmissionStatus(id="syn1234").get_async() - + # Update fields status.status = "SCORED" status.submission_annotations = {"score": [85.5]} - + # Store the update status = await status.store_async() print(f"Updated status: {status.status}") @@ -459,17 +463,19 @@ async def store_async( raise ValueError("The submission status must have an ID to update.") # Prepare request body - request_body = delete_none_keys({ - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - }) + request_body = delete_none_keys( + { + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + } + ) # Add annotations if present if self.annotations: @@ -485,7 +491,7 @@ async def store_async( response = await submission_services.update_submission_status( submission_id=self.id, request_body=request_body, - synapse_client=synapse_client + synapse_client=synapse_client, ) # Update this object with the response @@ -541,7 +547,7 @@ async def get_all_submission_statuses_async( status=status, limit=limit, offset=offset, - synapse_client=synapse_client + synapse_client=synapse_client, ) @staticmethod @@ -597,17 +603,19 @@ async def batch_update_submission_statuses_async( # Convert SubmissionStatus objects to dictionaries status_dicts = [] for status in statuses: - status_dict = delete_none_keys({ - "id": status.id, - "etag": status.etag, - "status": status.status, - "score": status.score, - "report": status.report, - "entityId": status.entity_id, - "versionNumber": status.version_number, - "canCancel": status.can_cancel, - "cancelRequested": status.cancel_requested, - }) + status_dict = delete_none_keys( + { + "id": status.id, + "etag": status.etag, + "status": status.status, + "score": status.score, + "report": status.report, + "entityId": status.entity_id, + "versionNumber": status.version_number, + "canCancel": status.can_cancel, + "cancelRequested": status.cancel_requested, + } + ) # Add annotations if present if status.annotations: @@ -633,5 +641,5 @@ async def batch_update_submission_statuses_async( return await submission_services.batch_update_submission_statuses( evaluation_id=evaluation_id, request_body=request_body, - synapse_client=synapse_client + synapse_client=synapse_client, ) diff --git a/synapseclient/models/submissionstatus.py b/synapseclient/models/submissionstatus.py deleted file mode 100644 index e69de29bb..000000000 From 9ca2448ab685931f54d040bb88394953e30e8fc6 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 14 Oct 2025 10:12:07 -0400 Subject: [PATCH 07/46] pipe query params directly into restAPI httpx requests --- synapseclient/api/submission_services.py | 12 ++++++------ synapseclient/models/__init__.py | 6 ++++++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py index feea626fc..cc9202954 100644 --- a/synapseclient/api/submission_services.py +++ b/synapseclient/api/submission_services.py @@ -94,7 +94,7 @@ async def get_evaluation_submissions( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -135,7 +135,7 @@ async def get_user_submissions( if user_id: query_params["userId"] = user_id - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -170,7 +170,7 @@ async def get_submission_count( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -339,7 +339,7 @@ async def get_all_submission_statuses( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -438,7 +438,7 @@ async def get_evaluation_submission_bundles( if status: query_params["status"] = status - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response @@ -475,6 +475,6 @@ async def get_user_submission_bundles( uri = f"/evaluation/{evaluation_id}/submission/bundle" query_params = {"limit": limit, "offset": offset} - response = await client.rest_get_async(uri, **query_params) + response = await client.rest_get_async(uri, params=query_params) return response diff --git a/synapseclient/models/__init__.py b/synapseclient/models/__init__.py index 9a5322727..700087ab5 100644 --- a/synapseclient/models/__init__.py +++ b/synapseclient/models/__init__.py @@ -25,6 +25,9 @@ from synapseclient.models.recordset import RecordSet from synapseclient.models.schema_organization import JSONSchema, SchemaOrganization from synapseclient.models.services import FailureStrategy +from synapseclient.models.submission import Submission +from synapseclient.models.submission_bundle import SubmissionBundle +from synapseclient.models.submission_status import SubmissionStatus from synapseclient.models.submissionview import SubmissionView from synapseclient.models.table import Table from synapseclient.models.table_components import ( @@ -128,6 +131,9 @@ "EntityRef", "DatasetCollection", # Submission models + "Submission", + "SubmissionBundle", + "SubmissionStatus", "SubmissionView", # JSON Schema models "SchemaOrganization", From f0d7ecc0d3cc68698e1df8fed8853b92ff460ca1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 14 Oct 2025 10:12:30 -0400 Subject: [PATCH 08/46] new dataclass object submission_bundle --- synapseclient/models/submission_bundle.py | 313 ++++++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 synapseclient/models/submission_bundle.py diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py new file mode 100644 index 000000000..7476bad18 --- /dev/null +++ b/synapseclient/models/submission_bundle.py @@ -0,0 +1,313 @@ +from dataclasses import dataclass, field +from typing import Dict, List, Optional, Protocol, Union, TYPE_CHECKING + +from typing_extensions import Self + +from synapseclient import Synapse +from synapseclient.api import submission_services +from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.models.mixins.access_control import AccessControllable + +if TYPE_CHECKING: + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + +class SubmissionBundleSynchronousProtocol(Protocol): + """Protocol defining the synchronous interface for SubmissionBundle operations.""" + + @staticmethod + def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + @staticmethod + def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + return [] + + +@dataclass +@async_to_sync +class SubmissionBundle( + SubmissionBundleSynchronousProtocol, + AccessControllable, +): + """A `SubmissionBundle` object represents a bundle containing a Synapse Submission + and its accompanying SubmissionStatus. This bundle provides convenient access to both + the submission data and its current status in a single object. + + + Attributes: + submission: A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + submission_status: A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + + Example: Retrieve submission bundles for an evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + # Get all submission bundles for an evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="9614543", + status="SCORED" + ) + + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") + ``` + """ + + submission: Optional["Submission"] = None + """ + A Submission to a Synapse Evaluation is a pointer to a versioned Entity. + Submissions are immutable, so we archive a copy of the EntityBundle at the time of submission. + """ + + submission_status: Optional["SubmissionStatus"] = None + """ + A SubmissionStatus is a secondary, mutable object associated with a Submission. + This object should be used to contain scoring data about the Submission. + """ + + _last_persistent_instance: Optional["SubmissionBundle"] = field( + default=None, repr=False, compare=False + ) + """The last persistent instance of this object. This is used to determine if the + object has been changed and needs to be updated in Synapse.""" + + def fill_from_dict( + self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] + ) -> "SubmissionBundle": + """ + Converts a response from the REST API into this dataclass. + + Arguments: + synapse_submission_bundle: The response from the REST API. + + Returns: + The SubmissionBundle object. + """ + from synapseclient.models.submission import Submission + from synapseclient.models.submission_status import SubmissionStatus + + submission_dict = synapse_submission_bundle.get("submission", None) + if submission_dict: + self.submission = Submission().fill_from_dict(submission_dict) + else: + self.submission = None + + submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) + if submission_status_dict: + self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + else: + self.submission_status = None + + return self + + @staticmethod + async def get_evaluation_submission_bundles_async( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the submission bundles + for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + + Example: Getting submission bundles for an evaluation + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="9614543", + status="SCORED", + limit=50 + ) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await submission_services.get_evaluation_submission_bundles( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles + + @staticmethod + async def get_user_submission_bundles_async( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> List["SubmissionBundle"]: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A list of SubmissionBundle objects containing the requesting user's + submission bundles for the evaluation queue. + + Example: Getting user submission bundles + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id="9614543", + limit=25 + ) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + ``` + """ + response = await submission_services.get_user_submission_bundles( + evaluation_id=evaluation_id, + limit=limit, + offset=offset, + synapse_client=synapse_client, + ) + + # Convert response to list of SubmissionBundle objects + bundles = [] + for bundle_dict in response.get("results", []): + bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundles.append(bundle) + + return bundles \ No newline at end of file From 91b947b4f4c5f98d3a4d5a7cbb502d660a894a82 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 5 Nov 2025 10:08:58 -0500 Subject: [PATCH 09/46] move submission services functions to evaluation_services.py --- synapseclient/api/evaluation_services.py | 472 ++++++++++++++++++++++ synapseclient/api/submission_services.py | 480 ----------------------- 2 files changed, 472 insertions(+), 480 deletions(-) delete mode 100644 synapseclient/api/submission_services.py diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 8149618b3..ce39105d6 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -386,3 +386,475 @@ async def get_evaluation_permissions( uri = f"/evaluation/{evaluation_id}/permissions" return await client.rest_get_async(uri) + +async def create_submission( + request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Creates a Submission and sends a submission notification email to the submitter's team members. + + + + Arguments: + request_body: The request body to send to the server. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = "/evaluation/submission" + + response = await client.rest_post_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Retrieves a Submission by its ID. + + + + Arguments: + submission_id: The ID of the submission to fetch. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The requested Submission. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + response = await client.rest_get_async(uri) + + return response + + +async def get_evaluation_submissions( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves all Submissions for a specified Evaluation queue. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + # TODO: Support pagination in the return type. + A response JSON containing a paginated list of submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission" + query_params = {"limit": limit, "offset": offset} + + if user_id: + query_params["userId"] = user_id + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/count" + query_params = {} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def delete_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> None: + """ + Deletes a Submission and its SubmissionStatus. + + + + Arguments: + submission_id: The ID of the submission to delete. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}" + + await client.rest_delete_async(uri) + + +async def cancel_submission( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Cancels a Submission. Only the user who created the Submission may cancel it. + + + + Arguments: + submission_id: The ID of the submission to cancel. + synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` + this will use the last created instance from the Synapse class constructor. + + Returns: + The Submission response object for the canceled submission as a JSON dict. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/cancellation" + + response = await client.rest_put_async(uri) + + return response + + +async def get_submission_status( + submission_id: str, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Gets the SubmissionStatus object associated with a specified Submission. + + + + Arguments: + submission_id: The ID of the submission to get the status for. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The SubmissionStatus object as a JSON dict. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatus. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_get_async(uri) + + return response + + +async def update_submission_status( + submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Updates a SubmissionStatus object. + + + + Arguments: + submission_id: The ID of the SubmissionStatus being updated. + request_body: The SubmissionStatus object to update as a dictionary. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated SubmissionStatus object as a JSON dict. + + Note: + Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle + concurrent updates. Each time a SubmissionStatus is updated a new etag will be + issued to the SubmissionStatus. When an update is requested, Synapse will compare + the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. + If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED + (412) response. When this occurs, the caller should fetch the latest copy of the + SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/submission/{submission_id}/status" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_all_submission_statuses( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission statuses by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission statuses for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. + Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION + to see all data marked as "private" in the SubmissionStatuses. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/status/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def batch_update_submission_statuses( + evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None +) -> dict: + """ + Update multiple SubmissionStatuses. The maximum batch size is 500. + + + + Arguments: + evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. + request_body: The SubmissionStatusBatch object as a dictionary containing: + - statuses: List of SubmissionStatus objects to update + - isFirstBatch: Boolean indicating if this is the first batch in the series + - isLastBatch: Boolean indicating if this is the last batch in the series + - batchToken: Token from previous batch response (required for all but first batch) + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A BatchUploadResponse object as a JSON dict containing the batch token + and other response information. + + Note: + To allow upload of more than the maximum batch size (500), the system supports + uploading a series of batches. Synapse employs optimistic concurrency on the + series in the form of a batch token. Each request (except the first) must include + the 'batch token' returned in the response to the previous batch. If another client + begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be + generated and upload must restart from the first batch. + + After the final batch is uploaded, the data for the Evaluation queue will be + mirrored to the tables which support querying. Therefore uploaded data will not + appear in Evaluation queries until after the final batch is successfully uploaded. + + It is the client's responsibility to note in each batch request: + 1. Whether it is the first batch in the series (isFirstBatch) + 2. Whether it is the last batch (isLastBatch) + + For a single batch both flags are set to 'true'. + + The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/statusBatch" + + response = await client.rest_put_async(uri, body=json.dumps(request_body)) + + return response + + +async def get_evaluation_submission_bundles( + evaluation_id: str, + status: Optional[str] = None, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + status: Optionally filter submission bundles by status. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10, max value 100. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of submission bundles for the evaluation queue. + + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle/all" + query_params = {"limit": limit, "offset": offset} + + if status: + query_params["status"] = status + + response = await client.rest_get_async(uri, params=query_params) + + return response + + +async def get_user_submission_bundles( + evaluation_id: str, + limit: int = 10, + offset: int = 0, + synapse_client: Optional["Synapse"] = None, +) -> dict: + """ + Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + + + + Arguments: + evaluation_id: The ID of the specified Evaluation. + limit: Limits the number of entities that will be fetched for this page. + When null it will default to 10. Default to 10. + offset: The offset index determines where this page will start from. + An index of 0 is the first entity. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A PaginatedResults object as a JSON dict containing + a paginated list of the requesting user's submission bundles for the evaluation queue. + """ + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + uri = f"/evaluation/{evaluation_id}/submission/bundle" + query_params = {"limit": limit, "offset": offset} + + response = await client.rest_get_async(uri, params=query_params) + + return response diff --git a/synapseclient/api/submission_services.py b/synapseclient/api/submission_services.py deleted file mode 100644 index cc9202954..000000000 --- a/synapseclient/api/submission_services.py +++ /dev/null @@ -1,480 +0,0 @@ -# TODO: The functions here should be moved into the `evaluation_services.py` file, once this branch is rebased onto those changes. - -import json -from typing import TYPE_CHECKING, Optional - -if TYPE_CHECKING: - from synapseclient import Synapse - - -async def create_submission( - request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Creates a Submission and sends a submission notification email to the submitter's team members. - - - - Arguments: - request_body: The request body to send to the server. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = "/evaluation/submission" - - response = await client.rest_post_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Retrieves a Submission by its ID. - - - - Arguments: - submission_id: The ID of the submission to fetch. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The requested Submission. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}" - - response = await client.rest_get_async(uri) - - return response - - -async def get_evaluation_submissions( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 20, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Retrieves all Submissions for a specified Evaluation queue. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - # TODO: Support pagination in the return type. - A response JSON containing a paginated list of submissions for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_user_submissions( - evaluation_id: str, - user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Retrieves Submissions for a specified Evaluation queue and user. - If user_id is omitted, this returns the submissions of the caller. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - user_id: Optionally specify the ID of the user whose submissions will be returned. - If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission" - query_params = {"limit": limit, "offset": offset} - - if user_id: - query_params["userId"] = user_id - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_submission_count( - evaluation_id: str, - status: Optional[str] = None, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. - - - - Arguments: - evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - A response JSON containing the submission count. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/count" - query_params = {} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def delete_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> None: - """ - Deletes a Submission and its SubmissionStatus. - - - - Arguments: - submission_id: The ID of the submission to delete. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}" - - await client.rest_delete_async(uri) - - -async def cancel_submission( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Cancels a Submission. Only the user who created the Submission may cancel it. - - - - Arguments: - submission_id: The ID of the submission to cancel. - synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` - this will use the last created instance from the Synapse class constructor. - - Returns: - The Submission response object for the canceled submission as a JSON dict. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/cancellation" - - response = await client.rest_put_async(uri) - - return response - - -async def get_submission_status( - submission_id: str, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Gets the SubmissionStatus object associated with a specified Submission. - - - - Arguments: - submission_id: The ID of the submission to get the status for. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The SubmissionStatus object as a JSON dict. - - Note: - The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. - Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION - to see all data marked as "private" in the SubmissionStatus. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/status" - - response = await client.rest_get_async(uri) - - return response - - -async def update_submission_status( - submission_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Updates a SubmissionStatus object. - - - - Arguments: - submission_id: The ID of the SubmissionStatus being updated. - request_body: The SubmissionStatus object to update as a dictionary. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - The updated SubmissionStatus object as a JSON dict. - - Note: - Synapse employs an Optimistic Concurrency Control (OCC) scheme to handle - concurrent updates. Each time a SubmissionStatus is updated a new etag will be - issued to the SubmissionStatus. When an update is requested, Synapse will compare - the etag of the passed SubmissionStatus with the current etag of the SubmissionStatus. - If the etags do not match, then the update will be rejected with a PRECONDITION_FAILED - (412) response. When this occurs, the caller should fetch the latest copy of the - SubmissionStatus and re-apply any changes, then re-attempt the SubmissionStatus update. - - The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/submission/{submission_id}/status" - - response = await client.rest_put_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_all_submission_statuses( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets a collection of SubmissionStatuses to a specified Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - status: Optionally filter submission statuses by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. - - Note: - The caller must be granted the ACCESS_TYPE.READ on the specified Evaluation. - Furthermore, the caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION - to see all data marked as "private" in the SubmissionStatuses. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/status/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def batch_update_submission_statuses( - evaluation_id: str, request_body: dict, synapse_client: Optional["Synapse"] = None -) -> dict: - """ - Update multiple SubmissionStatuses. The maximum batch size is 500. - - - - Arguments: - evaluation_id: The ID of the Evaluation to which the SubmissionStatus objects belong. - request_body: The SubmissionStatusBatch object as a dictionary containing: - - statuses: List of SubmissionStatus objects to update - - isFirstBatch: Boolean indicating if this is the first batch in the series - - isLastBatch: Boolean indicating if this is the last batch in the series - - batchToken: Token from previous batch response (required for all but first batch) - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A BatchUploadResponse object as a JSON dict containing the batch token - and other response information. - - Note: - To allow upload of more than the maximum batch size (500), the system supports - uploading a series of batches. Synapse employs optimistic concurrency on the - series in the form of a batch token. Each request (except the first) must include - the 'batch token' returned in the response to the previous batch. If another client - begins batch upload simultaneously, a PRECONDITION_FAILED (412) response will be - generated and upload must restart from the first batch. - - After the final batch is uploaded, the data for the Evaluation queue will be - mirrored to the tables which support querying. Therefore uploaded data will not - appear in Evaluation queries until after the final batch is successfully uploaded. - - It is the client's responsibility to note in each batch request: - 1. Whether it is the first batch in the series (isFirstBatch) - 2. Whether it is the last batch (isLastBatch) - - For a single batch both flags are set to 'true'. - - The caller must be granted the ACCESS_TYPE.UPDATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/statusBatch" - - response = await client.rest_put_async(uri, body=json.dumps(request_body)) - - return response - - -async def get_evaluation_submission_bundles( - evaluation_id: str, - status: Optional[str] = None, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission bundles for the evaluation queue. - - Note: - The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/bundle/all" - query_params = {"limit": limit, "offset": offset} - - if status: - query_params["status"] = status - - response = await client.rest_get_async(uri, params=query_params) - - return response - - -async def get_user_submission_bundles( - evaluation_id: str, - limit: int = 10, - offset: int = 0, - synapse_client: Optional["Synapse"] = None, -) -> dict: - """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. - - - - Arguments: - evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. - synapse_client: If not passed in and caching was not disabled by - `Synapse.allow_client_caching(False)` this will use the last created - instance from the Synapse class constructor. - - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of the requesting user's submission bundles for the evaluation queue. - """ - from synapseclient import Synapse - - client = Synapse.get_client(synapse_client=synapse_client) - - uri = f"/evaluation/{evaluation_id}/submission/bundle" - query_params = {"limit": limit, "offset": offset} - - response = await client.rest_get_async(uri, params=query_params) - - return response From a6b46d9445892d0e0ce83938cbef3f3bf4791350 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 10:26:06 -0500 Subject: [PATCH 10/46] renaming imports, to_synapse_request, request body refactor --- synapseclient/api/__init__.py | 27 ++++++++ synapseclient/api/evaluation_services.py | 1 + synapseclient/models/submission.py | 75 +++++++++++++++-------- synapseclient/models/submission_bundle.py | 36 +++++------ synapseclient/models/submission_status.py | 10 +-- 5 files changed, 100 insertions(+), 49 deletions(-) diff --git a/synapseclient/api/__init__.py b/synapseclient/api/__init__.py index 09c578ed2..1e02bf24c 100644 --- a/synapseclient/api/__init__.py +++ b/synapseclient/api/__init__.py @@ -64,15 +64,28 @@ update_entity_acl, ) from .evaluation_services import ( + batch_update_submission_statuses, + cancel_submission, create_or_update_evaluation, + create_submission, delete_evaluation, + delete_submission, get_all_evaluations, + get_all_submission_statuses, get_available_evaluations, get_evaluation, get_evaluation_acl, get_evaluation_permissions, + get_evaluation_submission_bundles, + get_evaluation_submissions, get_evaluations_by_project, + get_submission, + get_submission_count, + get_submission_status, + get_user_submission_bundles, + get_user_submissions, update_evaluation_acl, + update_submission_status, ) from .file_services import ( AddPartResponse, @@ -282,4 +295,18 @@ "get_evaluation_acl", "update_evaluation_acl", "get_evaluation_permissions", + # submission-related evaluation services + "create_submission", + "get_submission", + "get_evaluation_submissions", + "get_user_submissions", + "get_submission_count", + "delete_submission", + "cancel_submission", + "get_submission_status", + "update_submission_status", + "get_all_submission_statuses", + "batch_update_submission_statuses", + "get_evaluation_submission_bundles", + "get_user_submission_bundles", ] diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index ce39105d6..ae06932d6 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -387,6 +387,7 @@ async def get_evaluation_permissions( return await client.rest_get_async(uri) + async def create_submission( request_body: dict, synapse_client: Optional["Synapse"] = None ) -> dict: diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 4aa538131..f8ff4b206 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -4,9 +4,8 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.api import submission_services +from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys from synapseclient.models.mixins.access_control import AccessControllable from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin @@ -239,6 +238,44 @@ def fill_from_dict( return self + def to_synapse_request(self) -> Dict: + """Creates a request body expected of the Synapse REST API for the Submission model. + + Returns: + A dictionary containing the request body for creating a submission. + + Raises: + ValueError: If any required attributes are missing. + """ + # These attributes are required for creating a submission + required_attributes = ["entity_id", "evaluation_id"] + + for attribute in required_attributes: + if not getattr(self, attribute): + raise ValueError( + f"Your submission object is missing the '{attribute}' attribute. This attribute is required to create a submission" + ) + + # Build a request body for creating a submission + request_body = { + "entityId": self.entity_id, + "evaluationId": self.evaluation_id, + } + + # Add optional fields if they are set + if self.name is not None: + request_body["name"] = self.name + if self.team_id is not None: + request_body["teamId"] = self.team_id + if self.contributors: + request_body["contributors"] = self.contributors + if self.docker_repository_name is not None: + request_body["dockerRepositoryName"] = self.docker_repository_name + if self.docker_digest is not None: + request_body["dockerDigest"] = self.docker_digest + + return request_body + @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Store: {self.id if self.id else 'new_submission'}" ) @@ -262,6 +299,7 @@ async def store_async( ValueError: If the submission is missing required fields. Example: Creating a submission +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -278,26 +316,11 @@ async def store_async( print(submission.id) ``` """ - if not self.entity_id: - raise ValueError("The submission must have an entity_id to store.") - if not self.evaluation_id: - raise ValueError("The submission must have an evaluation_id to store.") - - # Prepare request body - request_body = delete_none_keys( - { - "entityId": self.entity_id, - "evaluationId": self.evaluation_id, - "name": self.name, - "teamId": self.team_id, - "contributors": self.contributors if self.contributors else None, - "dockerRepositoryName": self.docker_repository_name, - "dockerDigest": self.docker_digest, - } - ) + # Create the submission using the new to_synapse_request method + request_body = self.to_synapse_request() # Create the submission using the service - response = await submission_services.create_submission( + response = await evaluation_services.create_submission( request_body=request_body, synapse_client=synapse_client ) @@ -347,7 +370,7 @@ async def get_async( raise ValueError("The submission must have an ID to get.") # Get the submission using the service - response = await submission_services.get_submission( + response = await evaluation_services.get_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -404,7 +427,7 @@ async def get_evaluation_submissions_async( print(f"Found {len(response['results'])} submissions") ``` """ - return await submission_services.get_evaluation_submissions( + return await evaluation_services.get_evaluation_submissions( evaluation_id=evaluation_id, status=status, limit=limit, @@ -455,7 +478,7 @@ async def get_user_submissions_async( print(f"Found {len(response['results'])} user submissions") ``` """ - return await submission_services.get_user_submissions( + return await evaluation_services.get_user_submissions( evaluation_id=evaluation_id, user_id=user_id, limit=limit, @@ -499,7 +522,7 @@ async def get_submission_count_async( print(f"Found {response['count']} submissions") ``` """ - return await submission_services.get_submission_count( + return await evaluation_services.get_submission_count( evaluation_id=evaluation_id, status=status, synapse_client=synapse_client ) @@ -538,7 +561,7 @@ async def delete_submission_async( if not self.id: raise ValueError("The submission must have an ID to delete.") - await submission_services.delete_submission( + await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -580,7 +603,7 @@ async def cancel_submission_async( if not self.id: raise ValueError("The submission must have an ID to cancel.") - response = await submission_services.cancel_submission( + response = await evaluation_services.cancel_submission( submission_id=self.id, synapse_client=synapse_client ) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 7476bad18..ce4c736de 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,11 +1,9 @@ from dataclasses import dataclass, field -from typing import Dict, List, Optional, Protocol, Union, TYPE_CHECKING - -from typing_extensions import Self +from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union from synapseclient import Synapse -from synapseclient.api import submission_services -from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.api import evaluation_services +from synapseclient.core.async_utils import async_to_sync from synapseclient.models.mixins.access_control import AccessControllable if TYPE_CHECKING: @@ -85,7 +83,7 @@ def get_user_submission_bundles( instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's + A list of SubmissionBundle objects containing the requesting user's submission bundles for the evaluation queue. Example: Getting user submission bundles @@ -138,7 +136,7 @@ class SubmissionBundle( evaluation_id="9614543", status="SCORED" ) - + for bundle in bundles: print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") @@ -177,16 +175,18 @@ def fill_from_dict( """ from synapseclient.models.submission import Submission from synapseclient.models.submission_status import SubmissionStatus - + submission_dict = synapse_submission_bundle.get("submission", None) if submission_dict: self.submission = Submission().fill_from_dict(submission_dict) else: self.submission = None - + submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + self.submission_status = SubmissionStatus().fill_from_dict( + submission_status_dict + ) else: self.submission_status = None @@ -240,19 +240,19 @@ async def get_evaluation_submission_bundles_async( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - response = await submission_services.get_evaluation_submission_bundles( + response = await evaluation_services.get_evaluation_submission_bundles( evaluation_id=evaluation_id, status=status, limit=limit, offset=offset, synapse_client=synapse_client, ) - + bundles = [] for bundle_dict in response.get("results", []): bundle = SubmissionBundle().fill_from_dict(bundle_dict) bundles.append(bundle) - + return bundles @staticmethod @@ -277,7 +277,7 @@ async def get_user_submission_bundles_async( instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's + A list of SubmissionBundle objects containing the requesting user's submission bundles for the evaluation queue. Example: Getting user submission bundles @@ -297,17 +297,17 @@ async def get_user_submission_bundles_async( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - response = await submission_services.get_user_submission_bundles( + response = await evaluation_services.get_user_submission_bundles( evaluation_id=evaluation_id, limit=limit, offset=offset, synapse_client=synapse_client, ) - + # Convert response to list of SubmissionBundle objects bundles = [] for bundle_dict in response.get("results", []): bundle = SubmissionBundle().fill_from_dict(bundle_dict) bundles.append(bundle) - - return bundles \ No newline at end of file + + return bundles diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index c468f0e79..cc0376b7f 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -5,7 +5,7 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.api import submission_services +from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import delete_none_keys from synapseclient.models import Annotations @@ -409,7 +409,7 @@ async def get_async( if not self.id: raise ValueError("The submission status must have an ID to get.") - response = await submission_services.get_submission_status( + response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) @@ -488,7 +488,7 @@ async def store_async( request_body["submissionAnnotations"] = self.submission_annotations # Update the submission status using the service - response = await submission_services.update_submission_status( + response = await evaluation_services.update_submission_status( submission_id=self.id, request_body=request_body, synapse_client=synapse_client, @@ -542,7 +542,7 @@ async def get_all_submission_statuses_async( print(f"Found {len(response['results'])} submission statuses") ``` """ - return await submission_services.get_all_submission_statuses( + return await evaluation_services.get_all_submission_statuses( evaluation_id=evaluation_id, status=status, limit=limit, @@ -638,7 +638,7 @@ async def batch_update_submission_statuses_async( if batch_token: request_body["batchToken"] = batch_token - return await submission_services.batch_update_submission_statuses( + return await evaluation_services.batch_update_submission_statuses( evaluation_id=evaluation_id, request_body=request_body, synapse_client=synapse_client, From b68bd8e5c2a71148317eae83470b2d7646ac6661 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 11:37:44 -0500 Subject: [PATCH 11/46] patching up store method signature --- synapseclient/api/evaluation_services.py | 10 +++- synapseclient/models/submission.py | 61 +++++++++++++++++++++--- 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index ae06932d6..478976137 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -389,7 +389,7 @@ async def get_evaluation_permissions( async def create_submission( - request_body: dict, synapse_client: Optional["Synapse"] = None + request_body: dict, etag: str, synapse_client: Optional["Synapse"] = None ) -> dict: """ Creates a Submission and sends a submission notification email to the submitter's team members. @@ -398,6 +398,7 @@ async def create_submission( Arguments: request_body: The request body to send to the server. + etag: The current eTag of the Entity being submitted. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. """ @@ -407,7 +408,12 @@ async def create_submission( uri = "/evaluation/submission" - response = await client.rest_post_async(uri, body=json.dumps(request_body)) + # Add etag as query parameter if provided + params = {"etag": etag} + + response = await client.rest_post_async( + uri, body=json.dumps(request_body), params=params + ) return response diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index f8ff4b206..95ec2d294 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -141,7 +141,7 @@ class Submission( version_number: Optional[int] = field(default=None, compare=False) """ - The version number of the entity at submission. + The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. """ evaluation_id: Optional[str] = None @@ -196,6 +196,9 @@ class Submission( analogous to the Activity defined in the [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" + etag: Optional[str] = None + """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" + _last_persistent_instance: Optional["Submission"] = field( default=None, repr=False, compare=False ) @@ -238,6 +241,38 @@ def fill_from_dict( return self + async def _fetch_latest_entity( + self, *, synapse_client: Optional[Synapse] = None + ) -> Dict: + """ + Fetch the latest entity information from Synapse. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Dictionary containing entity information from the REST API. + + Raises: + ValueError: If entity_id is not set or if unable to fetch entity information. + """ + if not self.entity_id: + raise ValueError("entity_id must be set to fetch entity information") + + from synapseclient import Synapse + + client = Synapse.get_client(synapse_client=synapse_client) + + try: + entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") + return entity_info + except Exception as e: + raise ValueError( + f"Unable to fetch entity information for {self.entity_id}: {e}" + ) + def to_synapse_request(self) -> Dict: """Creates a request body expected of the Synapse REST API for the Submission model. @@ -296,7 +331,7 @@ async def store_async( The Submission object with the ID set. Raises: - ValueError: If the submission is missing required fields. + ValueError: If the submission is missing required fields, or if unable to fetch entity etag. Example: Creating a submission   @@ -319,12 +354,23 @@ async def store_async( # Create the submission using the new to_synapse_request method request_body = self.to_synapse_request() - # Create the submission using the service + if self.entity_id: + entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + self.entity_etag = entity_info.get("etag") + self.version_number = entity_info.get("versionNumber") + + # version number is required in the request body + if self.version_number is not None: + request_body["versionNumber"] = self.version_number + else: + raise ValueError("entity_id is required to create a submission") + + if not self.entity_etag: + raise ValueError("Unable to fetch etag for entity") + response = await evaluation_services.create_submission( - request_body=request_body, synapse_client=synapse_client + request_body, self.entity_etag, synapse_client=synapse_client ) - - # Update this object with the response self.fill_from_dict(response) return self @@ -355,6 +401,7 @@ async def get_async( ValueError: If the submission does not have an ID to get. Example: Retrieving a submission by ID +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -362,7 +409,7 @@ async def get_async( syn = Synapse() syn.login() - submission = await Submission(id="syn1234").get_async() + submission = await Submission(id="9999999").get_async() print(submission) ``` """ From 5a44f073593899103c4655168937490cb1f941d1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 12:14:01 -0500 Subject: [PATCH 12/46] update docs --- synapseclient/models/submission.py | 148 +++++++++++++++++------------ 1 file changed, 85 insertions(+), 63 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 95ec2d294..f4c1602ff 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -15,7 +15,6 @@ class SubmissionSynchronousProtocol(Protocol): def get( self, - include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, ) -> "Self": @@ -104,7 +103,6 @@ class Submission( file handles, and other metadata. docker_repository_name: For Docker repositories, the repository name. docker_digest: For Docker repositories, the digest of the submitted Docker image. - activity: The Activity model represents the main record of Provenance in Synapse. Example: Retrieve a Submission. ```python @@ -190,12 +188,6 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ - # TODO - activity: Optional[Dict] = field(default=None, compare=False) - """The Activity model represents the main record of Provenance in Synapse. It is - analogous to the Activity defined in the - [W3C Specification](https://www.w3.org/TR/prov-n/) on Provenance.""" - etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" @@ -234,11 +226,6 @@ def fill_from_dict( ) self.docker_digest = synapse_submission.get("dockerDigest", None) - activity_dict = synapse_submission.get("activity", None) - if activity_dict: - # TODO: Implement Activity class and its fill_from_dict method - self.activity = {} - return self async def _fetch_latest_entity( @@ -336,19 +323,24 @@ async def store_async( Example: Creating a submission   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission( - entity_id="syn123456", - evaluation_id="9614543", - name="My Submission" - ) - submission = await submission.store_async() - print(submission.id) + async def create_submission_example(): + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ) + submission = await submission.store_async() + print(submission.id) + + asyncio.run(create_submission_example()) ``` """ # Create the submission using the new to_synapse_request method @@ -379,7 +371,6 @@ async def store_async( ) async def get_async( self, - include_activity: bool = False, *, synapse_client: Optional[Synapse] = None, ) -> "Submission": @@ -387,9 +378,6 @@ async def get_async( Retrieve a Submission from Synapse. Arguments: - include_activity: Whether to include the activity in the returned submission. - Defaults to False. Setting this to True will include the activity - record associated with this submission. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -403,33 +391,30 @@ async def get_async( Example: Retrieving a submission by ID   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = await Submission(id="9999999").get_async() - print(submission) + async def get_submission_example(): + + submission = await Submission(id="9999999").get_async() + print(submission) + + asyncio.run(get_submission_example()) ``` """ if not self.id: raise ValueError("The submission must have an ID to get.") - # Get the submission using the service response = await evaluation_services.get_submission( submission_id=self.id, synapse_client=synapse_client ) - # Update this object with the response self.fill_from_dict(response) - # Handle activity if requested - if include_activity and self.activity: - # The activity should be included in the response by default - # but if we need to fetch it separately, we would do it here - pass - return self @staticmethod @@ -446,11 +431,11 @@ async def get_evaluation_submissions_async( Arguments: evaluation_id: The ID of the evaluation queue. - status: Optionally filter submissions by a submission status, such as SCORED, VALID, - INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. + status: Optionally filter submissions by a submission status. + Submission status can be one of + limit: Limits the number of submissions in a single response. Defaults to 20. offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. + An index of 0 is the first submission. Defaults to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -459,19 +444,24 @@ async def get_evaluation_submissions_async( A response JSON containing a paginated list of submissions for the evaluation queue. Example: Getting submissions for an evaluation +   + Get SCORED submissions from a specific evaluation. ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_evaluation_submissions_async( - evaluation_id="9614543", - status="SCORED", - limit=10 - ) - print(f"Found {len(response['results'])} submissions") + async def get_evaluation_submissions_example(): + response = await Submission.get_evaluation_submissions_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {len(response['results'])} submissions") + + asyncio.run(get_evaluation_submissions_example()) ``` """ return await evaluation_services.get_evaluation_submissions( @@ -511,18 +501,22 @@ async def get_user_submissions_async( Example: Getting user submissions ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_user_submissions_async( - evaluation_id="9614543", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + async def get_user_submissions_example(): + response = await Submission.get_user_submissions_async( + evaluation_id="9999999", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + + asyncio.run(get_user_submissions_example()) ``` """ return await evaluation_services.get_user_submissions( @@ -555,18 +549,24 @@ async def get_submission_count_async( A response JSON containing the submission count. Example: Getting submission count +   + Get the total number of SCORED submissions from a specific evaluation. ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - response = await Submission.get_submission_count_async( - evaluation_id="9614543", - status="SCORED" - ) - print(f"Found {response['count']} submissions") + async def get_submission_count_example(): + response = await Submission.get_submission_count_async( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {response['count']} submissions") + + asyncio.run(get_submission_count_example()) ``` """ return await evaluation_services.get_submission_count( @@ -576,7 +576,7 @@ async def get_submission_count_async( @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Delete: {self.id}" ) - async def delete_submission_async( + async def delete_async( self, *, synapse_client: Optional[Synapse] = None, @@ -593,16 +593,22 @@ async def delete_submission_async( ValueError: If the submission does not have an ID to delete. Example: Delete a submission +   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission(id="syn1234") - await submission.delete_submission_async() - print("Deleted Submission.") + async def delete_submission_example(): + submission = Submission(id="9999999") + await submission.delete_async() + print("Submission deleted successfully") + + # Run the async function + asyncio.run(delete_submission_example()) ``` """ if not self.id: @@ -611,11 +617,16 @@ async def delete_submission_async( await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) + + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been deleted.") @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"Submission_Cancel: {self.id}" ) - async def cancel_submission_async( + async def cancel_async( self, *, synapse_client: Optional[Synapse] = None, @@ -635,16 +646,22 @@ async def cancel_submission_async( ValueError: If the submission does not have an ID to cancel. Example: Cancel a submission +   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import Submission syn = Synapse() syn.login() - submission = Submission(id="syn1234") - canceled_submission = await submission.cancel_submission_async() - print(f"Canceled submission: {canceled_submission.id}") + async def cancel_submission_example(): + submission = Submission(id="syn1234") + canceled_submission = await submission.cancel_async() + print(f"Canceled submission: {canceled_submission.id}") + + # Run the async function + asyncio.run(cancel_submission_example()) ``` """ if not self.id: @@ -654,6 +671,11 @@ async def cancel_submission_async( submission_id=self.id, synapse_client=synapse_client ) + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + logger.info(f"Submission {self.id} has successfully been cancelled.") + # Update this object with the response self.fill_from_dict(response) return self From 65664d106a04d9a1cb614f7048175afa1900ff59 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 16:55:51 -0500 Subject: [PATCH 13/46] new suite of tests --- synapseclient/models/submission.py | 52 +- .../models/async/test_submission_async.py | 834 +++++++++++++----- .../models/synchronous/test_submission.py | 610 +++++++++++++ .../async/unit_test_submission_async.py | 604 +++++++++++++ .../synchronous/unit_test_submission.py | 805 +++++++++++++++++ 5 files changed, 2680 insertions(+), 225 deletions(-) create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission.py create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_async.py create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index f4c1602ff..b2e0e34f1 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -7,7 +7,6 @@ from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.models.mixins.access_control import AccessControllable -from synapseclient.models.mixins.table_components import DeleteMixin, GetMixin class SubmissionSynchronousProtocol(Protocol): @@ -80,8 +79,6 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: class Submission( SubmissionSynchronousProtocol, AccessControllable, - GetMixin, - DeleteMixin, ): """A `Submission` object represents a Synapse Submission, which is created when a user submits an entity to an evaluation queue. @@ -191,12 +188,6 @@ class Submission( etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" - _last_persistent_instance: Optional["Submission"] = field( - default=None, repr=False, compare=False - ) - """The last persistent instance of this object. This is used to determine if the - object has been changed and needs to be updated in Synapse.""" - def fill_from_dict( self, synapse_submission: Dict[str, Union[bool, str, int, List]] ) -> "Submission": @@ -254,6 +245,25 @@ async def _fetch_latest_entity( try: entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") + + # If this is a DockerRepository, fetch docker image tag & digest, and add it to the entity_info dict + if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": + docker_tag_response = await client.rest_get_async(f"/entity/{self.entity_id}/dockerTag") + + # Get the latest digest from the docker tag results + if "results" in docker_tag_response and docker_tag_response["results"]: + # Sort by createdOn timestamp to get the latest entry + # Convert ISO timestamp strings to datetime objects for comparison + from datetime import datetime + + latest_result = max( + docker_tag_response["results"], + key=lambda x: datetime.fromisoformat(x["createdOn"].replace("Z", "+00:00")) + ) + + # Add the latest result to entity_info + entity_info.update(latest_result) + return entity_info except Exception as e: raise ValueError( @@ -282,6 +292,7 @@ def to_synapse_request(self) -> Dict: request_body = { "entityId": self.entity_id, "evaluationId": self.evaluation_id, + "versionNumber": self.version_number } # Add optional fields if they are set @@ -343,23 +354,27 @@ async def create_submission_example(): asyncio.run(create_submission_example()) ``` """ - # Create the submission using the new to_synapse_request method - request_body = self.to_synapse_request() if self.entity_id: entity_info = await self._fetch_latest_entity(synapse_client=synapse_client) + self.entity_etag = entity_info.get("etag") - self.version_number = entity_info.get("versionNumber") - # version number is required in the request body - if self.version_number is not None: - request_body["versionNumber"] = self.version_number + if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.FileEntity": + self.version_number = entity_info.get("versionNumber") + elif entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": + self.version_number = 1 # TODO: Docker repositories do not have version numbers + self.docker_repository_name = entity_info.get("repositoryName") + self.docker_digest = entity_info.get("digest") else: raise ValueError("entity_id is required to create a submission") if not self.entity_etag: raise ValueError("Unable to fetch etag for entity") + # Build the request body now that all the necessary dataclass attributes are set + request_body = self.to_synapse_request() + response = await evaluation_services.create_submission( request_body, self.entity_etag, synapse_client=synapse_client ) @@ -417,6 +432,7 @@ async def get_submission_example(): return self + # TODO: Have all staticmethods return generators for pagination @staticmethod async def get_evaluation_submissions_async( evaluation_id: str, @@ -564,7 +580,7 @@ async def get_submission_count_example(): evaluation_id="9999999", status="SCORED" ) - print(f"Found {response['count']} submissions") + print(f"Found {response} submissions") asyncio.run(get_submission_count_example()) ``` @@ -617,8 +633,9 @@ async def delete_submission_example(): await evaluation_services.delete_submission( submission_id=self.id, synapse_client=synapse_client ) - + from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger logger.info(f"Submission {self.id} has successfully been deleted.") @@ -672,6 +689,7 @@ async def cancel_submission_example(): ) from synapseclient import Synapse + client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger logger.info(f"Submission {self.id} has successfully been cancelled.") diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index da5fdc46f..e5fb73631 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -1,208 +1,626 @@ -def test_create_submission_async(): - # WHEN an evaluation is retrieved - evaluation = Evaluation(id=evaluation_id).get() - - # AND an entity is retrieved - file = File(name="test.txt", parentId=project.id).get() - - # THEN the entity can be submitted to the evaluation - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - -def test_get_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is retrieved by ID - retrieved_submission = Submission(id=submission.id).get() - - # THEN the retrieved submission matches the created one - assert retrieved_submission.id == submission.id - assert retrieved_submission.name == submission.name - - # AND the user_id matches the current user - current_user = syn.getUserProfile()().id - assert retrieved_submission.user_id == current_user - - -def test_get_evaluation_submissions_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN submissions are retrieved for the evaluation - submissions = Submission.get_evaluation_submissions(evaluation.id) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the evaluation - for submission in submissions: - assert submission.evaluation_id == evaluation.id - - -def test_get_user_submissions_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN submissions are retrieved for the user - submissions = Submission.get_user_submissions(current_user) - - # THEN the submissions list is not empty - assert len(submissions) > 0 - - # AND each submission belongs to the user - for submission in submissions: - assert submission.user_id == current_user - - -def test_get_submission_count_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission count is retrieved for the evaluation - count = Submission.get_submission_count(evaluation.id) - - # THEN the count is greater than zero - assert count > 0 - - -def test_delete_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is deleted - submission.delete() - - # THEN retrieving the submission should raise an error - try: - Submission(id=submission.id).get() - assert False, "Expected an error when retrieving a deleted submission" - except SynapseError as e: - assert e.response.status_code == 404 - - -def test_cancel_submission_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission is canceled - submission.cancel() - - # THEN the submission status should be 'CANCELED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == "CANCELED" - - -def test_get_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - - # THEN the status should be 'RECEIVED' - assert status == "RECEIVED" - - -def test_update_submission_status_async(): - # GIVEN a submission has been created - submission = Submission( - name="Test Submission", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the submission status is retrieved - status = submission.get_status() - assert status != "SCORED" - - # AND the submission status is updated to 'SCORED' - submission.update_status("SCORED") - - # THEN the submission status should be 'SCORED' - updated_submission = Submission(id=submission.id).get() - assert updated_submission.status == "SCORED" - - -def test_get_evaluation_submission_statuses_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission statuses are retrieved for the evaluation - statuses = Submission.get_evaluation_submission_statuses(evaluation.id) - - # THEN the statuses list is not empty - assert len(statuses) > 0 - - -def test_batch_update_statuses_async(): - # GIVEN multiple submissions have been created - submission1 = Submission( - name="Test Submission 1", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - submission2 = Submission( - name="Test Submission 2", - entity_id=file.id, - evaluation_id=evaluation.id, - version_number=1, - ).store() - - # WHEN the statuses of the submissions are batch updated to 'SCORED' - Submission.batch_update_statuses([submission1.id, submission2.id], "SCORED") - - # THEN each submission status should be 'SCORED' - updated_submission1 = Submission(id=submission1.id).get() - updated_submission2 = Submission(id=submission2.id).get() - assert updated_submission1.status == "SCORED" - assert updated_submission2.status == "SCORED" - - -def test_get_evaluation_submission_bundles_async(): - # GIVEN an evaluation has submissions - evaluation = Evaluation(id=evaluation_id).get() - - # WHEN the submission bundles are retrieved for the evaluation - bundles = Submission.get_evaluation_submission_bundles(evaluation.id) - - # THEN the bundles list is not empty - assert len(bundles) > 0 - - -def test_get_user_submission_bundles_async(): - # GIVEN a user has made submissions - current_user = syn.getUserProfile()().id - - # WHEN the submission bundles are retrieved for the user - bundles = Submission.get_user_submission_bundles(current_user) - - # THEN the bundles list is not empty - assert len(bundles) > 0 +"""Async integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest +import pytest_asyncio + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data using async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id_async(self, test_evaluation: Evaluation): + # WHEN I try to create a submission without entity_id using async method + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id is required to create a submission"): + await submission.store_async(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id_async(self, test_file: File): + # WHEN I try to create a submission without evaluation_id using async method + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + await submission.store_async(synapse_client=self.syn) + + # async def test_store_submission_with_docker_repository_async( + # self, test_evaluation: Evaluation + # ): + # # GIVEN we would need a Docker repository entity (mocked for this test) + # # This test demonstrates the expected behavior for Docker repository submissions + + # # WHEN I create a submission for a Docker repository entity using async method + # # TODO: This would require a real Docker repository entity in a full integration test + # submission = Submission( + # entity_id="syn123456789", # Would be a Docker repository ID + # evaluation_id=test_evaluation.id, + # name=f"Docker Submission {uuid.uuid4()}", + # ) + + # # THEN the submission should handle Docker-specific attributes + # # (This test would need to be expanded with actual Docker repository setup) + # assert submission.entity_id == "syn123456789" + # assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrievalAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest_asyncio.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id_async( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID using async method + retrieved_submission = await Submission(id=test_submission.id).get_async( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination_async( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters using async method + response = await Submission.get_evaluation_submissions_async( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions_async(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user using async method + response = await Submission.get_user_submissions_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count_async(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation using async method + response = await Submission.get_submission_count_async( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + + +class TestSubmissionDeletionAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully_async( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission created with async method + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=self.syn) + + # WHEN I delete the submission using async method + await created_submission.delete_async(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + await Submission(id=created_submission.id).get_async(synapse_client=self.syn) + + async def test_delete_submission_without_id_async(self): + # WHEN I try to delete a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + +class TestSubmissionCancelAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest_asyncio.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest_asyncio.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest_asyncio.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = await File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store_async(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # async def test_cancel_submission_successfully_async( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission created with async method + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = await submission.store_async(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission using async method + # cancelled_submission = await created_submission.cancel_async(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id_async(self): + # WHEN I try to cancel a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + +class TestSubmissionValidationAsync: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id_async(self): + # WHEN I try to get a submission without an ID using async method + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id_async(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id_async(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data_async(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data_async(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + async def test_fetch_latest_entity_success_async(self): + # GIVEN a submission with a valid entity_id + submission = Submission(entity_id="syn123456", evaluation_id="789") + + # Note: This test would need a real entity ID to work in practice + # For now, we test the validation logic + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + async def test_fetch_latest_entity_without_entity_id_async(self): + # GIVEN a submission without entity_id + submission = Submission(evaluation_id="789") + + # WHEN I try to fetch entity information + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id must be set"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + +class TestSubmissionDataMappingAsync: + async def test_fill_from_dict_complete_data_async(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data_async(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py new file mode 100644 index 000000000..c595cc509 --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -0,0 +1,610 @@ +"""Integration tests for the synapseclient.models.Submission class.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission + + +class TestSubmissionCreation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + # Create a temporary file + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + async def test_store_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # WHEN I create a submission with valid data + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + self.schedule_for_cleanup(created_submission.id) + + # THEN the submission should be created successfully + assert created_submission.id is not None + assert created_submission.entity_id == test_file.id + assert created_submission.evaluation_id == test_evaluation.id + assert created_submission.name == submission.name + assert created_submission.user_id is not None + assert created_submission.created_on is not None + assert created_submission.version_number is not None + + async def test_store_submission_without_entity_id(self, test_evaluation: Evaluation): + # WHEN I try to create a submission without entity_id + submission = Submission( + evaluation_id=test_evaluation.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id is required"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_without_evaluation_id(self, test_file: File): + # WHEN I try to create a submission without evaluation_id + submission = Submission( + entity_id=test_file.id, + name="Test Submission", + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.store(synapse_client=self.syn) + + async def test_store_submission_with_docker_repository( + self, test_evaluation: Evaluation + ): + # GIVEN we would need a Docker repository entity (mocked for this test) + # This test demonstrates the expected behavior for Docker repository submissions + + # WHEN I create a submission for a Docker repository entity + # TODO: This would require a real Docker repository entity in a full integration test + submission = Submission( + entity_id="syn123456789", # Would be a Docker repository ID + evaluation_id=test_evaluation.id, + name=f"Docker Submission {uuid.uuid4()}", + ) + + # THEN the submission should handle Docker-specific attributes + # (This test would need to be expanded with actual Docker repository setup) + assert submission.entity_id == "syn123456789" + assert submission.evaluation_id == test_evaluation.id + + +class TestSubmissionRetrieval: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for retrieval tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_by_id( + self, test_submission: Submission, test_evaluation: Evaluation, test_file: File + ): + # WHEN I get a submission by ID + retrieved_submission = Submission(id=test_submission.id).get( + synapse_client=self.syn + ) + + # THEN the submission should be retrieved correctly + assert retrieved_submission.id == test_submission.id + assert retrieved_submission.entity_id == test_file.id + assert retrieved_submission.evaluation_id == test_evaluation.id + assert retrieved_submission.name == test_submission.name + assert retrieved_submission.user_id is not None + assert retrieved_submission.created_on is not None + + async def test_get_evaluation_submissions( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get all submissions for an evaluation + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with submissions + assert "results" in response + assert len(response["results"]) > 0 + + # AND the submission should be in the results + submission_ids = [sub.get("id") for sub in response["results"]] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_with_status_filter( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submissions filtered by status + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should get submissions with the specified status + assert "results" in response + for submission in response["results"]: + if submission.get("id") == test_submission.id: + # The submission should be in RECEIVED status initially + break + else: + pytest.fail("Test submission not found in filtered results") + + async def test_get_evaluation_submissions_with_pagination( + self, test_evaluation: Evaluation + ): + # WHEN I get submissions with pagination parameters + response = Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + limit=5, + offset=0, + synapse_client=self.syn, + ) + + # THEN the response should respect pagination + assert "results" in response + assert len(response["results"]) <= 5 + + async def test_get_user_submissions(self, test_evaluation: Evaluation): + # WHEN I get submissions for the current user + response = Submission.get_user_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a response with user submissions + assert "results" in response + # Note: Could be empty if user hasn't made submissions to this evaluation + + async def test_get_submission_count(self, test_evaluation: Evaluation): + # WHEN I get the submission count for an evaluation + response = Submission.get_submission_count( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + + # THEN I should get a count response + assert isinstance(response, int) + assert response >= 0 + + +class TestSubmissionDeletion: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + async def test_delete_submission_successfully( + self, test_evaluation: Evaluation, test_file: File + ): + # GIVEN a submission + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission for Deletion {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=self.syn) + + # WHEN I delete the submission + created_submission.delete(synapse_client=self.syn) + + # THEN attempting to retrieve it should raise an error + with pytest.raises(SynapseHTTPError): + Submission(id=created_submission.id).get(synapse_client=self.syn) + + async def test_delete_submission_without_id(self): + # WHEN I try to delete a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + submission.delete(synapse_client=self.syn) + + +class TestSubmissionCancel: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + """Create a test project for submission tests.""" + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission tests", + content_source=test_project.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> File: + """Create a test file for submission tests.""" + import tempfile + import os + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + temp_file.write("This is test content for submission testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=test_project.id + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + os.unlink(temp_file_path) + + # TODO: Add with SubmissionStatus model tests + # async def test_cancel_submission_successfully( + # self, test_evaluation: Evaluation, test_file: File + # ): + # # GIVEN a submission + # submission = Submission( + # entity_id=test_file.id, + # evaluation_id=test_evaluation.id, + # name=f"Test Submission for Cancellation {uuid.uuid4()}", + # ) + # created_submission = submission.store(synapse_client=self.syn) + # self.schedule_for_cleanup(created_submission.id) + + # # WHEN I cancel the submission + # cancelled_submission = created_submission.cancel(synapse_client=self.syn) + + # # THEN the submission should be cancelled + # assert cancelled_submission.id == created_submission.id + + async def test_cancel_submission_without_id(self): + # WHEN I try to cancel a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + submission.cancel(synapse_client=self.syn) + + +class TestSubmissionValidation: + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_get_submission_without_id(self): + # WHEN I try to get a submission without an ID + submission = Submission(entity_id="syn123", evaluation_id="456") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + submission.get(synapse_client=self.syn) + + async def test_to_synapse_request_missing_entity_id(self): + # WHEN I try to create a request without entity_id + submission = Submission(evaluation_id="456", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="Your submission object is missing the 'entity_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_missing_evaluation_id(self): + # WHEN I try to create a request without evaluation_id + submission = Submission(entity_id="syn123", name="Test") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + async def test_to_synapse_request_valid_data(self): + # WHEN I create a request with valid required data + submission = Submission( + entity_id="syn123456", + evaluation_id="789", + name="Test Submission", + team_id="team123", + contributors=["user1", "user2"], + docker_repository_name="test/repo", + docker_digest="sha256:abc123", + ) + + request_body = submission.to_synapse_request() + + # THEN it should create a valid request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert request_body["name"] == "Test Submission" + assert request_body["teamId"] == "team123" + assert request_body["contributors"] == ["user1", "user2"] + assert request_body["dockerRepositoryName"] == "test/repo" + assert request_body["dockerDigest"] == "sha256:abc123" + + async def test_to_synapse_request_minimal_data(self): + # WHEN I create a request with only required data + submission = Submission(entity_id="syn123456", evaluation_id="789") + + request_body = submission.to_synapse_request() + + # THEN it should create a minimal request body + assert request_body["entityId"] == "syn123456" + assert request_body["evaluationId"] == "789" + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + +class TestSubmissionDataMapping: + async def test_fill_from_dict_complete_data(self): + # GIVEN a complete submission response from the REST API + api_response = { + "id": "123456", + "userId": "user123", + "submitterAlias": "testuser", + "entityId": "syn789", + "versionNumber": 1, + "evaluationId": "eval456", + "name": "Test Submission", + "createdOn": "2023-01-01T10:00:00.000Z", + "teamId": "team123", + "contributors": ["user1", "user2"], + "submissionStatus": {"status": "RECEIVED"}, + "entityBundleJSON": '{"entity": {"id": "syn789"}}', + "dockerRepositoryName": "test/repo", + "dockerDigest": "sha256:abc123", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN all fields should be mapped correctly + assert submission.id == "123456" + assert submission.user_id == "user123" + assert submission.submitter_alias == "testuser" + assert submission.entity_id == "syn789" + assert submission.version_number == 1 + assert submission.evaluation_id == "eval456" + assert submission.name == "Test Submission" + assert submission.created_on == "2023-01-01T10:00:00.000Z" + assert submission.team_id == "team123" + assert submission.contributors == ["user1", "user2"] + assert submission.submission_status == {"status": "RECEIVED"} + assert submission.entity_bundle_json == '{"entity": {"id": "syn789"}}' + assert submission.docker_repository_name == "test/repo" + assert submission.docker_digest == "sha256:abc123" + + async def test_fill_from_dict_minimal_data(self): + # GIVEN a minimal submission response from the REST API + api_response = { + "id": "123456", + "entityId": "syn789", + "evaluationId": "eval456", + } + + # WHEN I fill a submission object from the dict + submission = Submission() + submission.fill_from_dict(api_response) + + # THEN required fields should be set and optional fields should have defaults + assert submission.id == "123456" + assert submission.entity_id == "syn789" + assert submission.evaluation_id == "eval456" + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py new file mode 100644 index 000000000..d02c0e713 --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -0,0 +1,604 @@ +"""Async unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmissionAsync: + """Async tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + } + ] + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z" + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z" + } + ] + } + + def test_fill_from_dict_complete_data_async(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_complete_data_async(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success_async(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag") + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results_async(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection_async(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update({ + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + }) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert stored_submission.version_number == 1 # Docker repos get version 1 + assert stored_submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_to_synapse_request_minimal_data_async(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py new file mode 100644 index 000000000..77820d3a6 --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -0,0 +1,805 @@ +"""Unit tests for the synapseclient.models.Submission class.""" +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, MagicMock, call, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Submission + +SUBMISSION_ID = "9614543" +USER_ID = "123456" +SUBMITTER_ALIAS = "test_user" +ENTITY_ID = "syn789012" +VERSION_NUMBER = 1 +EVALUATION_ID = "9999999" +SUBMISSION_NAME = "Test Submission" +CREATED_ON = "2023-01-01T10:00:00.000Z" +TEAM_ID = "team123" +CONTRIBUTORS = ["user1", "user2", "user3"] +SUBMISSION_STATUS = {"status": "RECEIVED", "score": 85.5} +ENTITY_BUNDLE_JSON = '{"entity": {"id": "syn789012", "name": "test_entity"}}' +DOCKER_REPOSITORY_NAME = "test/repository" +DOCKER_DIGEST = "sha256:abc123def456" +ETAG = "etag_value" + + +class TestSubmission: + """Tests for the synapseclient.models.Submission class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_response(self) -> Dict[str, Union[str, int, List, Dict]]: + """Get a complete example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": SUBMITTER_ALIAS, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "evaluationId": EVALUATION_ID, + "name": SUBMISSION_NAME, + "createdOn": CREATED_ON, + "teamId": TEAM_ID, + "contributors": CONTRIBUTORS, + "submissionStatus": SUBMISSION_STATUS, + "entityBundleJSON": ENTITY_BUNDLE_JSON, + "dockerRepositoryName": DOCKER_REPOSITORY_NAME, + "dockerDigest": DOCKER_DIGEST, + } + + def get_minimal_submission_response(self) -> Dict[str, str]: + """Get a minimal example submission response from the REST API.""" + return { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + } + + def get_example_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example entity response for testing entity fetching.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "versionNumber": VERSION_NUMBER, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + } + + def get_example_docker_entity_response(self) -> Dict[str, Union[str, int]]: + """Get an example Docker repository entity response for testing.""" + return { + "id": ENTITY_ID, + "etag": ETAG, + "name": "test_docker_repo", + "concreteType": "org.sagebionetworks.repo.model.docker.DockerRepository", + "repositoryName": "test/repository", + } + + def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get an example Docker tag response for testing.""" + return { + "totalNumberOfResults": 2, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:older123def456", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + } + ] + } + + def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: + """Get a more complex Docker tag response with multiple versions to test sorting.""" + return { + "totalNumberOfResults": 4, + "results": [ + { + "tag": "v1.0", + "digest": "sha256:version1", + "createdOn": "2024-01-01T10:00:00.000Z" + }, + { + "tag": "v3.0", + "digest": "sha256:version3", + "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + }, + { + "tag": "v2.0", + "digest": "sha256:version2", + "createdOn": "2024-06-01T15:30:00.000Z" + }, + { + "tag": "v1.5", + "digest": "sha256:version1_5", + "createdOn": "2024-03-15T08:45:00.000Z" + } + ] + } + + def test_fill_from_dict_complete_data(self) -> None: + # GIVEN a complete submission response from the REST API + # WHEN I call fill_from_dict with the example submission response + submission = Submission().fill_from_dict(self.get_example_submission_response()) + + # THEN the Submission object should be filled with all the data + assert submission.id == SUBMISSION_ID + assert submission.user_id == USER_ID + assert submission.submitter_alias == SUBMITTER_ALIAS + assert submission.entity_id == ENTITY_ID + assert submission.version_number == VERSION_NUMBER + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.created_on == CREATED_ON + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.submission_status == SUBMISSION_STATUS + assert submission.entity_bundle_json == ENTITY_BUNDLE_JSON + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_fill_from_dict_minimal_data(self) -> None: + # GIVEN a minimal submission response from the REST API + # WHEN I call fill_from_dict with the minimal submission response + submission = Submission().fill_from_dict(self.get_minimal_submission_response()) + + # THEN the Submission object should be filled with required data and defaults for optional data + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.version_number is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + + def test_to_synapse_request_complete_data(self) -> None: + # GIVEN a submission with all optional fields set + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain all fields in the correct format + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert request_body["name"] == SUBMISSION_NAME + assert request_body["teamId"] == TEAM_ID + assert request_body["contributors"] == CONTRIBUTORS + assert request_body["dockerRepositoryName"] == DOCKER_REPOSITORY_NAME + assert request_body["dockerDigest"] == DOCKER_DIGEST + + def test_to_synapse_request_minimal_data(self) -> None: + # GIVEN a submission with only required fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + version_number=VERSION_NUMBER, + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN the request body should contain only required fields + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["versionNumber"] == VERSION_NUMBER + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body + assert "dockerRepositoryName" not in request_body + assert "dockerDigest" not in request_body + + def test_to_synapse_request_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'entity_id' attribute"): + submission.to_synapse_request() + + def test_to_synapse_request_missing_evaluation_id(self) -> None: + # GIVEN a submission without evaluation_id + submission = Submission(entity_id=ENTITY_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission.to_synapse_request() + + @pytest.mark.asyncio + async def test_fetch_latest_entity_success(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with a mocked successful response + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_rest_get: + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["versionNumber"] == VERSION_NUMBER + mock_rest_get.assert_called_once_with(f"/entity/{ENTITY_ID}") + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_repository(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with mocked Docker repository responses + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return different responses for different URLs + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_example_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information with latest docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should have the latest tag information (v2.0 based on createdOn date) + assert entity_info["tag"] == "v2.0" + assert entity_info["digest"] == "sha256:latest456abc789" + assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" + + # Verify both API calls were made + expected_calls = [ + call(f"/entity/{ENTITY_ID}"), + call(f"/entity/{ENTITY_ID}/dockerTag") + ] + mock_rest_get.assert_has_calls(expected_calls) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_empty_results(self) -> None: + # GIVEN a submission with a Docker repository entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with empty docker tag results + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return empty docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return {"totalNumberOfResults": 0, "results": []} + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should return the entity information without docker tag info + assert entity_info["id"] == ENTITY_ID + assert entity_info["etag"] == ETAG + assert entity_info["repositoryName"] == "test/repository" + # Should not have docker tag fields since results were empty + assert "tag" not in entity_info + assert "digest" not in entity_info + assert "createdOn" not in entity_info + + @pytest.mark.asyncio + async def test_fetch_latest_entity_docker_complex_tag_selection(self) -> None: + # GIVEN a submission with a Docker repository with multiple tags + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity with multiple docker tags with different dates + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + ) as mock_rest_get: + # Configure the mock to return complex docker tag results + def side_effect(url): + if url == f"/entity/{ENTITY_ID}": + return self.get_example_docker_entity_response() + elif url == f"/entity/{ENTITY_ID}/dockerTag": + return self.get_complex_docker_tag_response() + + mock_rest_get.side_effect = side_effect + + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) + + # THEN it should select the tag with the latest createdOn timestamp (v3.0) + assert entity_info["tag"] == "v3.0" + assert entity_info["digest"] == "sha256:version3" + assert entity_info["createdOn"] == "2024-08-15T12:00:00.000Z" + + @pytest.mark.asyncio + async def test_fetch_latest_entity_without_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="entity_id must be set to fetch entity information"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_fetch_latest_entity_api_error(self) -> None: + # GIVEN a submission with an entity_id + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call _fetch_latest_entity and the API returns an error + with patch.object( + self.syn, + "rest_get_async", + new_callable=AsyncMock, + side_effect=SynapseHTTPError("Entity not found"), + ) as mock_rest_get: + # THEN it should raise a ValueError with context about the original error + with pytest.raises(ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}"): + await submission._fetch_latest_entity(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_success(self) -> None: + # GIVEN a submission with valid data + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should fetch entity information, create the submission, and fill the object + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Check the call arguments to create_submission + call_args = mock_create_submission.call_args + request_body = call_args[0][0] + etag = call_args[0][1] + + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert request_body["name"] == SUBMISSION_NAME + assert request_body["versionNumber"] == VERSION_NUMBER + assert etag == ETAG + + # Verify the submission is filled with response data + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_docker_repository_success(self) -> None: + # GIVEN a submission with valid data for Docker repository + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async with mocked Docker repository entity + docker_entity_with_tag = self.get_example_docker_entity_response() + docker_entity_with_tag.update({ + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z" + }) + + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=docker_entity_with_tag, + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should handle Docker repository specific logic + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify Docker repository attributes are set correctly + assert submission.version_number == 1 # Docker repos get version 1 + assert submission.docker_repository_name == "test/repository" + assert stored_submission.docker_digest == DOCKER_DIGEST + + @pytest.mark.asyncio + async def test_store_async_with_team_data_success(self) -> None: + # GIVEN a submission with team information + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + ) + + # WHEN I call store_async with mocked dependencies + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + return_value=self.get_example_entity_response(), + ) as mock_fetch_entity, patch( + "synapseclient.api.evaluation_services.create_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_create_submission: + + stored_submission = await submission.store_async(synapse_client=self.syn) + + # THEN it should preserve team information in the stored submission + mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) + mock_create_submission.assert_called_once() + + # Verify team data is preserved + assert stored_submission.team_id == TEAM_ID + assert stored_submission.contributors == CONTRIBUTORS + assert stored_submission.id == SUBMISSION_ID + assert stored_submission.entity_id == ENTITY_ID + assert stored_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_store_async_missing_entity_id(self) -> None: + # GIVEN a submission without entity_id + submission = Submission(evaluation_id=EVALUATION_ID, name=SUBMISSION_NAME) + + # WHEN I call store_async + # THEN it should raise a ValueError during to_synapse_request + with pytest.raises(ValueError, match="entity_id is required to create a submission"): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_store_async_entity_fetch_failure(self) -> None: + # GIVEN a submission with valid data but entity fetch fails + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + ) + + # WHEN I call store_async and entity fetching fails + with patch.object( + submission, + "_fetch_latest_entity", + new_callable=AsyncMock, + side_effect=ValueError("Unable to fetch entity information"), + ) as mock_fetch_entity: + # THEN it should propagate the ValueError + with pytest.raises(ValueError, match="Unable to fetch entity information"): + await submission.store_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call get_async with a mocked successful response + with patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_get_submission: + + retrieved_submission = await submission.get_async(synapse_client=self.syn) + + # THEN it should call the API and fill the object + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + assert retrieved_submission.id == SUBMISSION_ID + assert retrieved_submission.entity_id == ENTITY_ID + assert retrieved_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_get_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call get_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to get"): + await submission.get_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_delete_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call delete_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.delete_submission", + new_callable=AsyncMock, + ) as mock_delete_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + await submission.delete_async(synapse_client=self.syn) + + # THEN it should call the API and log the deletion + mock_delete_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been deleted." + ) + + @pytest.mark.asyncio + async def test_delete_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call delete_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to delete"): + await submission.delete_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_cancel_async_success(self) -> None: + # GIVEN a submission with an ID + submission = Submission(id=SUBMISSION_ID) + + # WHEN I call cancel_async with mocked dependencies + with patch( + "synapseclient.api.evaluation_services.cancel_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_response(), + ) as mock_cancel_submission, patch( + "synapseclient.Synapse.get_client", + return_value=self.syn, + ): + # Mock the logger + self.syn.logger = MagicMock() + + cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + + # THEN it should call the API, log the cancellation, and update the object + mock_cancel_submission.assert_called_once_with( + submission_id=SUBMISSION_ID, + synapse_client=self.syn, + ) + self.syn.logger.info.assert_called_once_with( + f"Submission {SUBMISSION_ID} has successfully been cancelled." + ) + assert cancelled_submission.id == SUBMISSION_ID + assert cancelled_submission.entity_id == ENTITY_ID + assert cancelled_submission.evaluation_id == EVALUATION_ID + + @pytest.mark.asyncio + async def test_cancel_async_without_id(self) -> None: + # GIVEN a submission without an ID + submission = Submission(entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID) + + # WHEN I call cancel_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="must have an ID to cancel"): + await submission.cancel_async(synapse_client=self.syn) + + @pytest.mark.asyncio + async def test_get_evaluation_submissions_async(self) -> None: + # GIVEN evaluation parameters + evaluation_id = EVALUATION_ID + status = "SCORED" + limit = 10 + offset = 5 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_evaluation_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_submissions: + + response = await Submission.get_evaluation_submissions_async( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_user_submissions_async(self) -> None: + # GIVEN user submission parameters + evaluation_id = EVALUATION_ID + user_id = USER_ID + limit = 15 + offset = 0 + + expected_response = { + "results": [self.get_example_submission_response()], + "totalNumberOfResults": 1, + } + + # WHEN I call get_user_submissions_async + with patch( + "synapseclient.api.evaluation_services.get_user_submissions", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_user_submissions: + + response = await Submission.get_user_submissions_async( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_user_submissions.assert_called_once_with( + evaluation_id=evaluation_id, + user_id=user_id, + limit=limit, + offset=offset, + synapse_client=self.syn, + ) + assert response == expected_response + + @pytest.mark.asyncio + async def test_get_submission_count_async(self) -> None: + # GIVEN submission count parameters + evaluation_id = EVALUATION_ID + status = "VALID" + + expected_response = 42 + + # WHEN I call get_submission_count_async + with patch( + "synapseclient.api.evaluation_services.get_submission_count", + new_callable=AsyncMock, + return_value=expected_response, + ) as mock_get_count: + + response = await Submission.get_submission_count_async( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + + # THEN it should call the API with correct parameters + mock_get_count.assert_called_once_with( + evaluation_id=evaluation_id, + status=status, + synapse_client=self.syn, + ) + assert response == expected_response + + def test_default_values(self) -> None: + # GIVEN a new Submission object with no parameters + submission = Submission() + + # THEN all attributes should have their default values + assert submission.id is None + assert submission.user_id is None + assert submission.submitter_alias is None + assert submission.entity_id is None + assert submission.version_number is None + assert submission.evaluation_id is None + assert submission.name is None + assert submission.created_on is None + assert submission.team_id is None + assert submission.contributors == [] + assert submission.submission_status is None + assert submission.entity_bundle_json is None + assert submission.docker_repository_name is None + assert submission.docker_digest is None + assert submission.etag is None + + def test_constructor_with_values(self) -> None: + # GIVEN specific values for submission attributes + # WHEN I create a Submission object with those values + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=SUBMISSION_NAME, + team_id=TEAM_ID, + contributors=CONTRIBUTORS, + docker_repository_name=DOCKER_REPOSITORY_NAME, + docker_digest=DOCKER_DIGEST, + ) + + # THEN the object should be initialized with those values + assert submission.id == SUBMISSION_ID + assert submission.entity_id == ENTITY_ID + assert submission.evaluation_id == EVALUATION_ID + assert submission.name == SUBMISSION_NAME + assert submission.team_id == TEAM_ID + assert submission.contributors == CONTRIBUTORS + assert submission.docker_repository_name == DOCKER_REPOSITORY_NAME + assert submission.docker_digest == DOCKER_DIGEST + + def test_to_synapse_request_with_none_values(self) -> None: + # GIVEN a submission with some None values for optional fields + submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name=None, # Explicitly None + team_id=None, # Explicitly None + contributors=[], # Empty list (falsy) + ) + + # WHEN I call to_synapse_request + request_body = submission.to_synapse_request() + + # THEN None and empty values should not be included + assert request_body["entityId"] == ENTITY_ID + assert request_body["evaluationId"] == EVALUATION_ID + assert "name" not in request_body + assert "teamId" not in request_body + assert "contributors" not in request_body From d214673cf5660c2ee7f7c203f5d73e884e5e8e31 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 10 Nov 2025 18:07:19 -0500 Subject: [PATCH 14/46] submissionstatus rework as a mutable object --- synapseclient/models/submission.py | 39 ++++-- synapseclient/models/submission_status.py | 125 ++++++++++-------- .../models/async/test_submission_async.py | 86 +++++++----- .../models/synchronous/test_submission.py | 100 ++++++++------ .../async/unit_test_submission_async.py | 73 +++++----- .../synchronous/unit_test_submission.py | 89 +++++++------ 6 files changed, 293 insertions(+), 219 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index b2e0e34f1..5d7390050 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -245,25 +245,32 @@ async def _fetch_latest_entity( try: entity_info = await client.rest_get_async(f"/entity/{self.entity_id}") - + # If this is a DockerRepository, fetch docker image tag & digest, and add it to the entity_info dict - if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": - docker_tag_response = await client.rest_get_async(f"/entity/{self.entity_id}/dockerTag") - + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + docker_tag_response = await client.rest_get_async( + f"/entity/{self.entity_id}/dockerTag" + ) + # Get the latest digest from the docker tag results if "results" in docker_tag_response and docker_tag_response["results"]: # Sort by createdOn timestamp to get the latest entry # Convert ISO timestamp strings to datetime objects for comparison from datetime import datetime - + latest_result = max( docker_tag_response["results"], - key=lambda x: datetime.fromisoformat(x["createdOn"].replace("Z", "+00:00")) + key=lambda x: datetime.fromisoformat( + x["createdOn"].replace("Z", "+00:00") + ), ) - + # Add the latest result to entity_info entity_info.update(latest_result) - + return entity_info except Exception as e: raise ValueError( @@ -292,7 +299,7 @@ def to_synapse_request(self) -> Dict: request_body = { "entityId": self.entity_id, "evaluationId": self.evaluation_id, - "versionNumber": self.version_number + "versionNumber": self.version_number, } # Add optional fields if they are set @@ -360,10 +367,18 @@ async def create_submission_example(): self.entity_etag = entity_info.get("etag") - if entity_info.get("concreteType") == "org.sagebionetworks.repo.model.FileEntity": + if ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.FileEntity" + ): self.version_number = entity_info.get("versionNumber") - elif entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository": - self.version_number = 1 # TODO: Docker repositories do not have version numbers + elif ( + entity_info.get("concreteType") + == "org.sagebionetworks.repo.model.docker.DockerRepository" + ): + self.version_number = ( + 1 # TODO: Docker repositories do not have version numbers + ) self.docker_repository_name = entity_info.get("repositoryName") self.docker_digest = entity_info.get("digest") else: diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index cc0376b7f..740351504 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,13 +1,14 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse +from synapseclient.annotations import to_submission_status_annotations from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys +from synapseclient.core.utils import delete_none_keys, merge_dataclass_entities from synapseclient.models import Annotations from synapseclient.models.mixins.access_control import AccessControllable @@ -318,8 +319,9 @@ class SubmissionStatus( """The last persistent instance of this object. This is used to determine if the object has been changed and needs to be updated in Synapse.""" + @property def has_changed(self) -> bool: - """Determines if the object has been changed and needs to be updated in Synapse.""" + """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( not self._last_persistent_instance or self._last_persistent_instance != self ) @@ -327,10 +329,7 @@ def has_changed(self) -> bool: def _set_last_persistent_instance(self) -> None: """Stash the last time this object interacted with Synapse. This is used to determine if the object has been changed and needs to be updated in Synapse.""" - import dataclasses - - del self._last_persistent_instance - self._last_persistent_instance = dataclasses.replace(self) + self._last_persistent_instance = replace(self) def fill_from_dict( self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] @@ -372,6 +371,44 @@ def fill_from_dict( return self + def to_synapse_request(self) -> Dict: + """ + Creates a request body expected by the Synapse REST API for the SubmissionStatus model. + + Returns: + A dictionary containing the request body for updating a submission status. + """ + # Prepare request body with basic fields + request_body = delete_none_keys( + { + "id": self.id, + "etag": self.etag, + "status": self.status, + "score": self.score, + "report": self.report, + "entityId": self.entity_id, + "versionNumber": self.version_number, + "canCancel": self.can_cancel, + "cancelRequested": self.cancel_requested, + } + ) + + # Add annotations if present + if self.annotations and len(self.annotations) > 0: + # Convert annotations to the format expected by the API + request_body["annotations"] = to_submission_status_annotations( + self.annotations + ) + + # Add submission annotations if present + if self.submission_annotations and len(self.submission_annotations) > 0: + # Convert submission annotations to the format expected by the API + request_body["submissionAnnotations"] = to_submission_status_annotations( + self.submission_annotations + ) + + return request_body + @otel_trace_method( method_to_trace_name=lambda self, **kwargs: f"SubmissionStatus_Get: {self.id}" ) @@ -462,39 +499,40 @@ async def store_async( if not self.id: raise ValueError("The submission status must have an ID to update.") - # Prepare request body - request_body = delete_none_keys( - { - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - } - ) - - # Add annotations if present - if self.annotations: - # Convert annotations to the format expected by the API - request_body["annotations"] = self.annotations + # Get the client for logging + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + + # Check if there are changes to apply + if self._last_persistent_instance and self.has_changed: + # Merge with the last persistent instance to preserve system-managed fields + merge_dataclass_entities( + source=self._last_persistent_instance, + destination=self, + fields_to_preserve_from_source=[ + "id", + "etag", + "modified_on", + "entity_id", + "version_number", + "status_version", + ], + logger=logger, + ) + elif self._last_persistent_instance and not self.has_changed: + logger.warning( + f"SubmissionStatus (ID: {self.id}) has not changed since last 'store' or 'get' event, so it will not be updated in Synapse. Please get the submission status again if you want to refresh its state." + ) + return self - # Add submission annotations if present - if self.submission_annotations: - # Convert submission annotations to the format expected by the API - request_body["submissionAnnotations"] = self.submission_annotations + request_body = self.to_synapse_request() - # Update the submission status using the service response = await evaluation_services.update_submission_status( submission_id=self.id, request_body=request_body, synapse_client=synapse_client, ) - # Update this object with the response self.fill_from_dict(response) self._set_last_persistent_instance() return self @@ -603,28 +641,7 @@ async def batch_update_submission_statuses_async( # Convert SubmissionStatus objects to dictionaries status_dicts = [] for status in statuses: - status_dict = delete_none_keys( - { - "id": status.id, - "etag": status.etag, - "status": status.status, - "score": status.score, - "report": status.report, - "entityId": status.entity_id, - "versionNumber": status.version_number, - "canCancel": status.can_cancel, - "cancelRequested": status.cancel_requested, - } - ) - - # Add annotations if present - if status.annotations: - status_dict["annotations"] = status.annotations - - # Add submission annotations if present - if status.submission_annotations: - status_dict["submissionAnnotations"] = status.submission_annotations - + status_dict = status.to_synapse_request() status_dicts.append(status_dict) # Prepare the batch request body diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index e5fb73631..745a60960 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -49,22 +49,27 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - + import tempfile + # Create a temporary file - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -93,7 +98,9 @@ async def test_store_submission_successfully_async( assert created_submission.created_on is not None assert created_submission.version_number is not None - async def test_store_submission_without_entity_id_async(self, test_evaluation: Evaluation): + async def test_store_submission_without_entity_id_async( + self, test_evaluation: Evaluation + ): # WHEN I try to create a submission without entity_id using async method submission = Submission( evaluation_id=test_evaluation.id, @@ -101,7 +108,9 @@ async def test_store_submission_without_entity_id_async(self, test_evaluation: E ) # THEN it should raise a ValueError - with pytest.raises(ValueError, match="entity_id is required to create a submission"): + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): await submission.store_async(synapse_client=self.syn) async def test_store_submission_without_evaluation_id_async(self, test_file: File): @@ -120,7 +129,7 @@ async def test_store_submission_without_evaluation_id_async(self, test_file: Fil # ): # # GIVEN we would need a Docker repository entity (mocked for this test) # # This test demonstrates the expected behavior for Docker repository submissions - + # # WHEN I create a submission for a Docker repository entity using async method # # TODO: This would require a real Docker repository entity in a full integration test # submission = Submission( @@ -128,7 +137,7 @@ async def test_store_submission_without_evaluation_id_async(self, test_file: Fil # evaluation_id=test_evaluation.id, # name=f"Docker Submission {uuid.uuid4()}", # ) - + # # THEN the submission should handle Docker-specific attributes # # (This test would need to be expanded with actual Docker repository setup) # assert submission.entity_id == "syn123456789" @@ -173,21 +182,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -239,7 +253,7 @@ async def test_get_evaluation_submissions_async( # THEN I should get a response with submissions assert "results" in response assert len(response["results"]) > 0 - + # AND the submission should be in the results submission_ids = [sub.get("id") for sub in response["results"]] assert test_submission.id in submission_ids @@ -336,21 +350,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -373,7 +392,9 @@ async def test_delete_submission_successfully_async( # THEN attempting to retrieve it should raise an error with pytest.raises(SynapseHTTPError): - await Submission(id=created_submission.id).get_async(synapse_client=self.syn) + await Submission(id=created_submission.id).get_async( + synapse_client=self.syn + ) async def test_delete_submission_without_id_async(self): # WHEN I try to delete a submission without an ID using async method @@ -422,21 +443,26 @@ async def test_evaluation( @pytest_asyncio.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = await File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store_async(synapse_client=syn) schedule_for_cleanup(file.id) return file diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index c595cc509..e6943a006 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -21,9 +21,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -48,22 +46,27 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - + import tempfile + # Create a temporary file - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -92,7 +95,9 @@ async def test_store_submission_successfully( assert created_submission.created_on is not None assert created_submission.version_number is not None - async def test_store_submission_without_entity_id(self, test_evaluation: Evaluation): + async def test_store_submission_without_entity_id( + self, test_evaluation: Evaluation + ): # WHEN I try to create a submission without entity_id submission = Submission( evaluation_id=test_evaluation.id, @@ -119,7 +124,7 @@ async def test_store_submission_with_docker_repository( ): # GIVEN we would need a Docker repository entity (mocked for this test) # This test demonstrates the expected behavior for Docker repository submissions - + # WHEN I create a submission for a Docker repository entity # TODO: This would require a real Docker repository entity in a full integration test submission = Submission( @@ -127,7 +132,7 @@ async def test_store_submission_with_docker_repository( evaluation_id=test_evaluation.id, name=f"Docker Submission {uuid.uuid4()}", ) - + # THEN the submission should handle Docker-specific attributes # (This test would need to be expanded with actual Docker repository setup) assert submission.entity_id == "syn123456789" @@ -145,9 +150,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -172,21 +175,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -238,7 +246,7 @@ async def test_get_evaluation_submissions( # THEN I should get a response with submissions assert "results" in response assert len(response["results"]) > 0 - + # AND the submission should be in the results submission_ids = [sub.get("id") for sub in response["results"]] assert test_submission.id in submission_ids @@ -309,9 +317,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -336,21 +342,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -395,9 +406,7 @@ async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: """Create a test project for submission tests.""" - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -422,21 +431,26 @@ async def test_evaluation( @pytest.fixture(scope="function") async def test_file( - self, test_project: Project, syn: Synapse, schedule_for_cleanup: Callable[..., None] + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], ) -> File: """Create a test file for submission tests.""" - import tempfile import os - - with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.txt') as temp_file: + import tempfile + + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: temp_file.write("This is test content for submission testing.") temp_file_path = temp_file.name - + try: file = File( path=temp_file_path, name=f"test_file_{uuid.uuid4()}.txt", - parent_id=test_project.id + parent_id=test_project.id, ).store(synapse_client=syn) schedule_for_cleanup(file.id) return file @@ -490,7 +504,10 @@ async def test_to_synapse_request_missing_entity_id(self): submission = Submission(evaluation_id="456", name="Test") # THEN it should raise a ValueError - with pytest.raises(ValueError, match="Your submission object is missing the 'entity_id' attribute"): + with pytest.raises( + ValueError, + match="Your submission object is missing the 'entity_id' attribute", + ): submission.to_synapse_request() async def test_to_synapse_request_missing_evaluation_id(self): @@ -607,4 +624,3 @@ async def test_fill_from_dict_minimal_data(self): assert submission.entity_bundle_json is None assert submission.docker_repository_name is None assert submission.docker_digest is None - diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index d02c0e713..8a0ca25ac 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -88,14 +88,14 @@ def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:older123def456", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - } - ] + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], } def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: @@ -106,24 +106,24 @@ def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:version1", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v3.0", + "tag": "v3.0", "digest": "sha256:version3", - "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:version2", - "createdOn": "2024-06-01T15:30:00.000Z" + "createdOn": "2024-06-01T15:30:00.000Z", }, { "tag": "v1.5", "digest": "sha256:version1_5", - "createdOn": "2024-03-15T08:45:00.000Z" - } - ] + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], } def test_fill_from_dict_complete_data_async(self) -> None: @@ -210,9 +210,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_example_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information with latest docker tag info @@ -223,11 +223,11 @@ def side_effect(url): assert entity_info["tag"] == "v2.0" assert entity_info["digest"] == "sha256:latest456abc789" assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" - + # Verify both API calls were made expected_calls = [ call(f"/entity/{ENTITY_ID}"), - call(f"/entity/{ENTITY_ID}/dockerTag") + call(f"/entity/{ENTITY_ID}/dockerTag"), ] mock_rest_get.assert_has_calls(expected_calls) @@ -248,9 +248,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return {"totalNumberOfResults": 0, "results": []} - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information without docker tag info @@ -279,9 +279,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_complex_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should select the tag with the latest createdOn timestamp (v3.0) @@ -309,13 +309,12 @@ async def test_store_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should fetch entity information, create the submission, and fill the object mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify the submission is filled with response data assert stored_submission.id == SUBMISSION_ID assert stored_submission.entity_id == ENTITY_ID @@ -332,12 +331,14 @@ async def test_store_async_docker_repository_success(self) -> None: # WHEN I call store_async with mocked Docker repository entity docker_entity_with_tag = self.get_example_docker_entity_response() - docker_entity_with_tag.update({ - "tag": "v2.0", - "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - }) - + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + with patch.object( submission, "_fetch_latest_entity", @@ -348,13 +349,12 @@ async def test_store_async_docker_repository_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should handle Docker repository specific logic mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify Docker repository attributes are set correctly assert stored_submission.version_number == 1 # Docker repos get version 1 assert stored_submission.docker_repository_name == "test/repository" @@ -382,13 +382,12 @@ async def test_store_async_with_team_data_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should preserve team information in the stored submission mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify team data is preserved assert stored_submission.team_id == TEAM_ID assert stored_submission.contributors == CONTRIBUTORS @@ -407,7 +406,6 @@ async def test_get_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_get_submission: - retrieved_submission = await submission.get_async(synapse_client=self.syn) # THEN it should call the API and fill the object @@ -463,7 +461,9 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( @@ -496,7 +496,6 @@ async def test_get_evaluation_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, @@ -534,7 +533,6 @@ async def test_get_user_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, @@ -567,7 +565,6 @@ async def test_get_submission_count_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_count: - response = await Submission.get_submission_count_async( evaluation_id=evaluation_id, status=status, diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 77820d3a6..038045f11 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -88,14 +88,14 @@ def get_example_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:older123def456", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - } - ] + "createdOn": "2024-06-01T15:30:00.000Z", + }, + ], } def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: @@ -106,24 +106,24 @@ def get_complex_docker_tag_response(self) -> Dict[str, Union[str, int, List]]: { "tag": "v1.0", "digest": "sha256:version1", - "createdOn": "2024-01-01T10:00:00.000Z" + "createdOn": "2024-01-01T10:00:00.000Z", }, { - "tag": "v3.0", + "tag": "v3.0", "digest": "sha256:version3", - "createdOn": "2024-08-15T12:00:00.000Z" # This should be selected (latest) + "createdOn": "2024-08-15T12:00:00.000Z", # This should be selected (latest) }, { - "tag": "v2.0", + "tag": "v2.0", "digest": "sha256:version2", - "createdOn": "2024-06-01T15:30:00.000Z" + "createdOn": "2024-06-01T15:30:00.000Z", }, { "tag": "v1.5", "digest": "sha256:version1_5", - "createdOn": "2024-03-15T08:45:00.000Z" - } - ] + "createdOn": "2024-03-15T08:45:00.000Z", + }, + ], } def test_fill_from_dict_complete_data(self) -> None: @@ -270,9 +270,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_example_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information with latest docker tag info @@ -283,11 +283,11 @@ def side_effect(url): assert entity_info["tag"] == "v2.0" assert entity_info["digest"] == "sha256:latest456abc789" assert entity_info["createdOn"] == "2024-06-01T15:30:00.000Z" - + # Verify both API calls were made expected_calls = [ call(f"/entity/{ENTITY_ID}"), - call(f"/entity/{ENTITY_ID}/dockerTag") + call(f"/entity/{ENTITY_ID}/dockerTag"), ] mock_rest_get.assert_has_calls(expected_calls) @@ -308,9 +308,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return {"totalNumberOfResults": 0, "results": []} - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should return the entity information without docker tag info @@ -339,9 +339,9 @@ def side_effect(url): return self.get_example_docker_entity_response() elif url == f"/entity/{ENTITY_ID}/dockerTag": return self.get_complex_docker_tag_response() - + mock_rest_get.side_effect = side_effect - + entity_info = await submission._fetch_latest_entity(synapse_client=self.syn) # THEN it should select the tag with the latest createdOn timestamp (v3.0) @@ -356,7 +356,9 @@ async def test_fetch_latest_entity_without_entity_id(self) -> None: # WHEN I call _fetch_latest_entity # THEN it should raise a ValueError - with pytest.raises(ValueError, match="entity_id must be set to fetch entity information"): + with pytest.raises( + ValueError, match="entity_id must be set to fetch entity information" + ): await submission._fetch_latest_entity(synapse_client=self.syn) @pytest.mark.asyncio @@ -372,7 +374,9 @@ async def test_fetch_latest_entity_api_error(self) -> None: side_effect=SynapseHTTPError("Entity not found"), ) as mock_rest_get: # THEN it should raise a ValueError with context about the original error - with pytest.raises(ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}"): + with pytest.raises( + ValueError, match=f"Unable to fetch entity information for {ENTITY_ID}" + ): await submission._fetch_latest_entity(synapse_client=self.syn) @pytest.mark.asyncio @@ -395,24 +399,23 @@ async def test_store_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should fetch entity information, create the submission, and fill the object mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Check the call arguments to create_submission call_args = mock_create_submission.call_args request_body = call_args[0][0] etag = call_args[0][1] - + assert request_body["entityId"] == ENTITY_ID assert request_body["evaluationId"] == EVALUATION_ID assert request_body["name"] == SUBMISSION_NAME assert request_body["versionNumber"] == VERSION_NUMBER assert etag == ETAG - + # Verify the submission is filled with response data assert stored_submission.id == SUBMISSION_ID assert stored_submission.entity_id == ENTITY_ID @@ -429,12 +432,14 @@ async def test_store_async_docker_repository_success(self) -> None: # WHEN I call store_async with mocked Docker repository entity docker_entity_with_tag = self.get_example_docker_entity_response() - docker_entity_with_tag.update({ - "tag": "v2.0", - "digest": "sha256:latest456abc789", - "createdOn": "2024-06-01T15:30:00.000Z" - }) - + docker_entity_with_tag.update( + { + "tag": "v2.0", + "digest": "sha256:latest456abc789", + "createdOn": "2024-06-01T15:30:00.000Z", + } + ) + with patch.object( submission, "_fetch_latest_entity", @@ -445,13 +450,12 @@ async def test_store_async_docker_repository_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should handle Docker repository specific logic mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify Docker repository attributes are set correctly assert submission.version_number == 1 # Docker repos get version 1 assert submission.docker_repository_name == "test/repository" @@ -479,13 +483,12 @@ async def test_store_async_with_team_data_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_create_submission: - stored_submission = await submission.store_async(synapse_client=self.syn) # THEN it should preserve team information in the stored submission mock_fetch_entity.assert_called_once_with(synapse_client=self.syn) mock_create_submission.assert_called_once() - + # Verify team data is preserved assert stored_submission.team_id == TEAM_ID assert stored_submission.contributors == CONTRIBUTORS @@ -500,7 +503,9 @@ async def test_store_async_missing_entity_id(self) -> None: # WHEN I call store_async # THEN it should raise a ValueError during to_synapse_request - with pytest.raises(ValueError, match="entity_id is required to create a submission"): + with pytest.raises( + ValueError, match="entity_id is required to create a submission" + ): await submission.store_async(synapse_client=self.syn) @pytest.mark.asyncio @@ -534,7 +539,6 @@ async def test_get_async_success(self) -> None: new_callable=AsyncMock, return_value=self.get_example_submission_response(), ) as mock_get_submission: - retrieved_submission = await submission.get_async(synapse_client=self.syn) # THEN it should call the API and fill the object @@ -610,7 +614,9 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async(synapse_client=self.syn) + cancelled_submission = await submission.cancel_async( + synapse_client=self.syn + ) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( @@ -653,7 +659,6 @@ async def test_get_evaluation_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, @@ -691,7 +696,6 @@ async def test_get_user_submissions_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, @@ -724,7 +728,6 @@ async def test_get_submission_count_async(self) -> None: new_callable=AsyncMock, return_value=expected_response, ) as mock_get_count: - response = await Submission.get_submission_count_async( evaluation_id=evaluation_id, status=status, From 27817976976b21a8437055ea0c1b2cd466192d6f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:16:50 -0500 Subject: [PATCH 15/46] bug fix for Statuses: updated to_synapse_request to follow same pattern as evaluations design --- synapseclient/models/submission_status.py | 50 ++++++++++++++++------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 740351504..132be79c2 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -8,7 +8,7 @@ from synapseclient.annotations import to_submission_status_annotations from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method -from synapseclient.core.utils import delete_none_keys, merge_dataclass_entities +from synapseclient.core.utils import merge_dataclass_entities from synapseclient.models import Annotations from synapseclient.models.mixins.access_control import AccessControllable @@ -377,21 +377,41 @@ def to_synapse_request(self) -> Dict: Returns: A dictionary containing the request body for updating a submission status. + + Raises: + ValueError: If any required attributes are missing. """ - # Prepare request body with basic fields - request_body = delete_none_keys( - { - "id": self.id, - "etag": self.etag, - "status": self.status, - "score": self.score, - "report": self.report, - "entityId": self.entity_id, - "versionNumber": self.version_number, - "canCancel": self.can_cancel, - "cancelRequested": self.cancel_requested, - } - ) + # These attributes are required for updating a submission status + required_attributes = ["id", "etag", "status_version"] + + for attribute in required_attributes: + if getattr(self, attribute) is None: + raise ValueError( + f"Your submission status object is missing the '{attribute}' attribute. This attribute is required to update a submission status" + ) + + # Build request body with required fields + request_body = { + "id": self.id, + "etag": self.etag, + "statusVersion": self.status_version, + } + + # Add optional fields only if they have values + if self.status is not None: + request_body["status"] = self.status + if self.score is not None: + request_body["score"] = self.score + if self.report is not None: + request_body["report"] = self.report + if self.entity_id is not None: + request_body["entityId"] = self.entity_id + if self.version_number is not None: + request_body["versionNumber"] = self.version_number + if self.can_cancel is not None: + request_body["canCancel"] = self.can_cancel + if self.cancel_requested is not None: + request_body["cancelRequested"] = self.cancel_requested # Add annotations if present if self.annotations and len(self.annotations) > 0: From 3230b1fd06c1b22daaebb80139f84f5f20a6cb4b Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:39:12 -0500 Subject: [PATCH 16/46] replace != with is not for full object comparison (not just keys) --- synapseclient/api/evaluation_services.py | 3 ++- synapseclient/models/submission_status.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 478976137..8cd34a942 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -678,7 +678,8 @@ async def update_submission_status( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/status" - + print("request body") + print(request_body) response = await client.rest_put_async(uri, body=json.dumps(request_body)) return response diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 132be79c2..b1dae7dee 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -323,7 +323,7 @@ class SubmissionStatus( def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( - not self._last_persistent_instance or self._last_persistent_instance != self + not self._last_persistent_instance or self._last_persistent_instance is not self ) def _set_last_persistent_instance(self) -> None: @@ -368,6 +368,7 @@ def fill_from_dict( self.submission_annotations = Annotations.from_dict( submission_annotations_dict ) + print(self.submission_annotations) return self From 2d43e5de7523a06172657be4321069d737eb3ece Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 11 Nov 2025 10:43:50 -0500 Subject: [PATCH 17/46] expose the is_private arg for to_submission_status_annotations ONLY FOR submission annotations --- synapseclient/models/submission_status.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index b1dae7dee..08815709c 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -286,6 +286,9 @@ class SubmissionStatus( ] = field(default_factory=dict, compare=False) """Annotations are additional key-value pair metadata that are associated with an object.""" + is_private: Optional[bool] = field(default=True, compare=False) + """Indicates whether the submission annotations are private (True) or public (False). Default is True.""" + entity_id: Optional[str] = None """ The Synapse ID of the Entity in this Submission. @@ -425,7 +428,7 @@ def to_synapse_request(self) -> Dict: if self.submission_annotations and len(self.submission_annotations) > 0: # Convert submission annotations to the format expected by the API request_body["submissionAnnotations"] = to_submission_status_annotations( - self.submission_annotations + self.submission_annotations, is_private=self.is_private ) return request_body From d4d63ba54559909bfbc29733dba94e5de727bc95 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Wed, 12 Nov 2025 10:53:40 -0500 Subject: [PATCH 18/46] fixed submission status/submission annotations --- synapseclient/annotations.py | 122 ++++++++++++++++++++++ synapseclient/models/submission_status.py | 40 ++++--- 2 files changed, 149 insertions(+), 13 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index f9d6ac124..62f4df379 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -228,6 +228,128 @@ def to_submission_status_annotations(annotations, is_private=True): return synapseAnnos +def to_submission_annotations( + id: typing.Union[str, int], + etag: str, + annotations: typing.Dict[str, typing.Any], + is_private: bool = True, + logger: typing.Optional[typing.Any] = None, +) -> typing.Dict[str, typing.Any]: + """ + Converts a normal dictionary to the format used for submission annotations, which is different from the format + used to annotate entities. + + This function creates the proper nested structure that includes id, etag, and annotations in the format + expected by the submissionAnnotations field of a SubmissionStatus request body. + + Arguments: + id: The unique ID of the submission being annotated. + etag: The etag of the submission status for optimistic concurrency control. + annotations: A normal Python dictionary comprised of the annotations to be added. + logger: An optional logger instance. If not provided, a default logger will be used. + + Returns: + A dictionary in the format expected by submissionAnnotations with nested structure containing + id, etag, and annotations object with type/value format. + + Example: Using this function + Converting annotations to submission format + + from synapseclient.annotations import to_submission_annotations + + # Input annotations + my_annotations = { + "score": 85, + "feedback": "Good work!" + } + + # Convert to submission annotations format + submission_annos = to_submission_annotations( + id="9999999", + etag="abc123", + annotations=my_annotations, + is_private=True + ) + + # Result: + # { + # "id": "9999999", + # "etag": "abc123", + # "annotations": { + # "score": {"type": "INTEGER", "value": [85]}, + # "feedback": {"type": "STRING", "value": ["Good work!"]}, + # } + # } + + Note: + This function is designed specifically for the submissionAnnotations field format, + which is part of the creation of a SubmissionStatus request body: + + + """ + # Create the base structure + submission_annos = {"id": str(id), "etag": str(etag), "annotations": {}} + + # Convert each annotation to the proper nested format + for key, value in annotations.items(): + # Ensure value is a list + if not isinstance(value, list): + value_list = [value] + else: + value_list = value + + # Warn about empty annotation values and skip them + if not value_list: + if logger: + logger.warning( + f"Annotation '{key}' has an empty value list and will be skipped" + ) + else: + from synapseclient import Synapse + + client = Synapse.get_client() + client.logger.warning( + f"Annotation '{key}' has an empty value list and will be skipped" + ) + continue + + # Determine type based on the first element + first_element = value_list[0] + + if isinstance(first_element, str): + submission_annos["annotations"][key] = { + "type": "STRING", + "value": value_list, + } + elif isinstance(first_element, bool): + # Convert booleans to lowercase strings + submission_annos["annotations"][key] = { + "type": "STRING", + "value": [str(v).lower() for v in value_list], + } + elif isinstance(first_element, int): + submission_annos["annotations"][key] = {"type": "LONG", "value": value_list} + elif isinstance(first_element, float): + submission_annos["annotations"][key] = { + "type": "DOUBLE", + "value": value_list, + } + elif is_date(first_element): + # Convert dates to unix timestamps + submission_annos["annotations"][key] = { + "type": "LONG", + "value": [to_unix_epoch_time(v) for v in value_list], + } + else: + # Default to string representation + submission_annos["annotations"][key] = { + "type": "STRING", + "value": [str(v) for v in value_list], + } + + return submission_annos + + # TODO: this should accept a status object and return its annotations or an empty dict if there are none def from_submission_status_annotations(annotations) -> dict: """ diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 08815709c..8fedbbde8 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,3 +1,4 @@ +from calendar import c from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union @@ -5,7 +6,10 @@ from typing_extensions import Self from synapseclient import Synapse -from synapseclient.annotations import to_submission_status_annotations +from synapseclient.annotations import ( + to_submission_annotations, + to_submission_status_annotations, +) from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync, otel_trace_method from synapseclient.core.utils import merge_dataclass_entities @@ -286,8 +290,8 @@ class SubmissionStatus( ] = field(default_factory=dict, compare=False) """Annotations are additional key-value pair metadata that are associated with an object.""" - is_private: Optional[bool] = field(default=True, compare=False) - """Indicates whether the submission annotations are private (True) or public (False). Default is True.""" + private_status_annotations: Optional[bool] = field(default=True, compare=False) + """Indicates whether the submission status annotations (NOT to be confused with submission annotations) are private (True) or public (False). Default is True.""" entity_id: Optional[str] = None """ @@ -326,7 +330,8 @@ class SubmissionStatus( def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" return ( - not self._last_persistent_instance or self._last_persistent_instance is not self + not self._last_persistent_instance + or self._last_persistent_instance is not self ) def _set_last_persistent_instance(self) -> None: @@ -375,16 +380,25 @@ def fill_from_dict( return self - def to_synapse_request(self) -> Dict: + def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: """ Creates a request body expected by the Synapse REST API for the SubmissionStatus model. + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + Returns: A dictionary containing the request body for updating a submission status. Raises: ValueError: If any required attributes are missing. """ + # Get the client for logging + client = Synapse.get_client(synapse_client=synapse_client) + logger = client.logger + # These attributes are required for updating a submission status required_attributes = ["id", "etag", "status_version"] @@ -417,18 +431,17 @@ def to_synapse_request(self) -> Dict: if self.cancel_requested is not None: request_body["cancelRequested"] = self.cancel_requested - # Add annotations if present if self.annotations and len(self.annotations) > 0: - # Convert annotations to the format expected by the API request_body["annotations"] = to_submission_status_annotations( - self.annotations + self.annotations, self.private_status_annotations ) - # Add submission annotations if present if self.submission_annotations and len(self.submission_annotations) > 0: - # Convert submission annotations to the format expected by the API - request_body["submissionAnnotations"] = to_submission_status_annotations( - self.submission_annotations, is_private=self.is_private + request_body["submissionAnnotations"] = to_submission_annotations( + id=self.id, + etag=self.etag, + annotations=self.submission_annotations, + logger=logger, ) return request_body @@ -473,7 +486,8 @@ async def get_async( response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) - + print("raw response") + print(response) self.fill_from_dict(response) self._set_last_persistent_instance() return self From 8b9f9c9c34d16f616f44df7fa005de8f453eca4c Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 10:53:15 -0500 Subject: [PATCH 19/46] add support for legacy annotations --- synapseclient/annotations.py | 1 - synapseclient/models/submission_status.py | 38 ++++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/synapseclient/annotations.py b/synapseclient/annotations.py index 62f4df379..5e4414d29 100644 --- a/synapseclient/annotations.py +++ b/synapseclient/annotations.py @@ -232,7 +232,6 @@ def to_submission_annotations( id: typing.Union[str, int], etag: str, annotations: typing.Dict[str, typing.Any], - is_private: bool = True, logger: typing.Optional[typing.Any] = None, ) -> typing.Dict[str, typing.Any]: """ diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 8fedbbde8..1dc8646fa 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -1,4 +1,3 @@ -from calendar import c from dataclasses import dataclass, field, replace from datetime import date, datetime from typing import Dict, List, Optional, Protocol, Union @@ -199,9 +198,15 @@ class SubmissionStatus( status: The possible states of a Synapse Submission (e.g., RECEIVED, VALIDATED, SCORED). score: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. report: This field is deprecated and should not be used. Use the 'submission_annotations' field instead. - annotations: Primary container object for Annotations on a Synapse object. - submission_annotations: Annotations are additional key-value pair metadata that are associated with an object. + annotations: Primary container object for Annotations on a Synapse object. These annotations use the legacy + format and do not show up in a submission view. The visibility is controlled by private_status_annotations. + submission_annotations: Submission Annotations are additional key-value pair metadata that are associated with an object. + These annotations use the modern nested format and show up in a submission view. + private_status_annotations: Indicates whether the annotations (not to be confused with submission annotations) are private (True) or public (False). + Default is True. This controls the visibility of the 'annotations' field. entity_id: The Synapse ID of the Entity in this Submission. + evaluation_id: The ID of the Evaluation to which this Submission belongs. This field is automatically + populated when retrieving a SubmissionStatus via get() and is required when updating annotations. version_number: The version number of the Entity in this Submission. status_version: A version of the status, auto-generated and auto-incremented by the system and read-only to the client. can_cancel: Can this submission be cancelled? By default, this will be set to False. Users can read this value. @@ -298,6 +303,11 @@ class SubmissionStatus( The Synapse ID of the Entity in this Submission. """ + evaluation_id: Optional[str] = None + """ + The ID of the Evaluation to which this Submission belongs. + """ + version_number: Optional[int] = field(default=None, compare=False) """ The version number of the Entity in this Submission. @@ -432,8 +442,20 @@ def to_synapse_request(self, synapse_client: Optional[Synapse] = None) -> Dict: request_body["cancelRequested"] = self.cancel_requested if self.annotations and len(self.annotations) > 0: + # evaluation_id is required when annotations are provided for scopeId + if self.evaluation_id is None: + raise ValueError( + "Your submission status object is missing the 'evaluation_id' attribute. This attribute is required when submissions are updated with annotations. Please retrieve your submission status with .get() to populate this field." + ) + + # Add required objectId and scopeId to annotations dict as per Synapse API requirements + # https://rest-docs.synapse.org/rest/org/sagebionetworks/repo/model/annotation/Annotations.html + annotations_with_metadata = self.annotations.copy() + annotations_with_metadata["objectId"] = self.id + annotations_with_metadata["scopeId"] = self.evaluation_id + request_body["annotations"] = to_submission_status_annotations( - self.annotations, self.private_status_annotations + annotations_with_metadata, self.private_status_annotations ) if self.submission_annotations and len(self.submission_annotations) > 0: @@ -489,6 +511,14 @@ async def get_async( print("raw response") print(response) self.fill_from_dict(response) + + # Fetch evaluation_id from the associated submission since it's not in the SubmissionStatus response + if not self.evaluation_id: + submission_response = await evaluation_services.get_submission( + submission_id=self.id, synapse_client=synapse_client + ) + self.evaluation_id = submission_response.get("evaluationId", None) + self._set_last_persistent_instance() return self From 6aa914aeb8ac327b90534032840d06e6ac4c64de Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 13:59:42 -0500 Subject: [PATCH 20/46] remove debug prints --- synapseclient/api/evaluation_services.py | 3 +-- synapseclient/models/submission_status.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 8cd34a942..478976137 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -678,8 +678,7 @@ async def update_submission_status( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/submission/{submission_id}/status" - print("request body") - print(request_body) + response = await client.rest_put_async(uri, body=json.dumps(request_body)) return response diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 1dc8646fa..402c330be 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -508,8 +508,6 @@ async def get_async( response = await evaluation_services.get_submission_status( submission_id=self.id, synapse_client=synapse_client ) - print("raw response") - print(response) self.fill_from_dict(response) # Fetch evaluation_id from the associated submission since it's not in the SubmissionStatus response From 2d67fcca16e43569883811451dba66f30bceeca8 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 14:12:59 -0500 Subject: [PATCH 21/46] get_all_submission_statuses now returns a list of substat objects --- synapseclient/models/submission_status.py | 54 +++++++++++++++-------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 402c330be..6585f8a68 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -89,7 +89,7 @@ def get_all_submission_statuses( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> List["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -105,8 +105,7 @@ def get_all_submission_statuses( instance from the Synapse class constructor. Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. + A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation ```python @@ -116,15 +115,17 @@ def get_all_submission_statuses( syn = Synapse() syn.login() - response = SubmissionStatus.get_all_submission_statuses( + statuses = SubmissionStatus.get_all_submission_statuses( evaluation_id="9614543", status="SCORED", limit=50 ) - print(f"Found {len(response['results'])} submission statuses") + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") ``` """ - return {} + return [] @staticmethod def batch_update_submission_statuses( @@ -386,7 +387,6 @@ def fill_from_dict( self.submission_annotations = Annotations.from_dict( submission_annotations_dict ) - print(self.submission_annotations) return self @@ -611,7 +611,7 @@ async def get_all_submission_statuses_async( offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> List["SubmissionStatus"]: """ Gets a collection of SubmissionStatuses to a specified Evaluation. @@ -627,8 +627,7 @@ async def get_all_submission_statuses_async( instance from the Synapse class constructor. Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission statuses for the evaluation queue. + A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation ```python @@ -638,15 +637,17 @@ async def get_all_submission_statuses_async( syn = Synapse() syn.login() - response = await SubmissionStatus.get_all_submission_statuses_async( + statuses = await SubmissionStatus.get_all_submission_statuses_async( evaluation_id="9614543", status="SCORED", limit=50 ) - print(f"Found {len(response['results'])} submission statuses") + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") ``` """ - return await evaluation_services.get_all_submission_statuses( + response = await evaluation_services.get_all_submission_statuses( evaluation_id=evaluation_id, status=status, limit=limit, @@ -654,6 +655,18 @@ async def get_all_submission_statuses_async( synapse_client=synapse_client, ) + # Convert each result to a SubmissionStatus object + submission_statuses = [] + for status_dict in response.get("results", []): + submission_status = SubmissionStatus() + submission_status.fill_from_dict(status_dict) + # Manually set evaluation_id since it's not part of the response + submission_status.evaluation_id = evaluation_id + submission_status._set_last_persistent_instance() + submission_statuses.append(submission_status) + + return submission_statuses + @staticmethod async def batch_update_submission_statuses_async( evaluation_id: str, @@ -689,12 +702,17 @@ async def batch_update_submission_statuses_async( syn = Synapse() syn.login() - # Prepare list of status updates - statuses = [ - SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), - SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) - ] + # Retrieve existing statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="RECEIVED" + ) + + # Modify statuses as needed + for status in statuses: + status.status = "SCORED" + # Update statuses in batch response = await SubmissionStatus.batch_update_submission_statuses_async( evaluation_id="9614543", statuses=statuses, From c368e49f515c8420d0f9caec4e8fef4e6a31c567 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 13 Nov 2025 14:24:11 -0500 Subject: [PATCH 22/46] docstring updates --- synapseclient/models/submission_status.py | 59 +++++++++++++++++------ 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 6585f8a68..4d4360ae9 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -35,7 +35,11 @@ def get( Returns: The SubmissionStatus instance retrieved from Synapse. - Example: Retrieving a submission status by ID. + Raises: + ValueError: If the submission status does not have an ID to get. + + Example: Retrieving a submission status by ID +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -43,7 +47,7 @@ def get( syn = Synapse() syn.login() - status = SubmissionStatus(id="syn1234").get() + status = SubmissionStatus(id="9999999").get() print(status) ``` """ @@ -63,9 +67,14 @@ def store( instance from the Synapse class constructor. Returns: - The updated SubmissionStatus instance. + The updated SubmissionStatus object. - Example: Update a submission status. + Raises: + ValueError: If the submission status is missing required fields. + + Example: Update a submission status +   + Update an existing submission status by first retrieving it, then modifying fields and storing the changes. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -73,10 +82,17 @@ def store( syn = Synapse() syn.login() - status = SubmissionStatus(id="syn1234").get() + # Get existing status + status = SubmissionStatus(id="9999999").get() + + # Update fields status.status = "SCORED" + status.submission_annotations = {"score": [85.5]} + + # Store the update status = status.store() - print("Updated SubmissionStatus.") + print(f"Updated status:") + print(status) ``` """ return self @@ -108,6 +124,8 @@ def get_all_submission_statuses( A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation +   + Retrieve a list of submission statuses for a specific evaluation, optionally filtered by status. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -155,6 +173,8 @@ def batch_update_submission_statuses( and other response information. Example: Batch update submission statuses +   + Update multiple submission statuses in a single batch operation for efficiency. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -162,12 +182,17 @@ def batch_update_submission_statuses( syn = Synapse() syn.login() - # Prepare list of status updates - statuses = [ - SubmissionStatus(id="syn1", status="SCORED", submission_annotations={"score": [90.0]}), - SubmissionStatus(id="syn2", status="SCORED", submission_annotations={"score": [85.0]}) - ] + # Retrieve existing statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9614543", + status="RECEIVED" + ) + + # Modify statuses as needed + for status in statuses: + status.status = "SCORED" + # Update statuses in batch response = SubmissionStatus.batch_update_submission_statuses( evaluation_id="9614543", statuses=statuses, @@ -216,6 +241,8 @@ class SubmissionStatus( Submission owner can read and request to change this value. Example: Retrieve and update a SubmissionStatus. +   + This example demonstrates the basic workflow of retrieving an existing submission status, updating its fields, and storing the changes back to Synapse. ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -224,7 +251,7 @@ class SubmissionStatus( syn.login() # Get a submission status - status = SubmissionStatus(id="syn123456").get() + status = SubmissionStatus(id="9999999").get() # Update the status status.status = "SCORED" @@ -491,6 +518,7 @@ async def get_async( ValueError: If the submission status does not have an ID to get. Example: Retrieving a submission status by ID +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -498,7 +526,7 @@ async def get_async( syn = Synapse() syn.login() - status = await SubmissionStatus(id="syn1234").get_async() + status = await SubmissionStatus(id="9999999").get_async() print(status) ``` """ @@ -543,6 +571,7 @@ async def store_async( ValueError: If the submission status is missing required fields. Example: Update a submission status +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -551,7 +580,7 @@ async def store_async( syn.login() # Get existing status - status = await SubmissionStatus(id="syn1234").get_async() + status = await SubmissionStatus(id="9999999").get_async() # Update fields status.status = "SCORED" @@ -630,6 +659,7 @@ async def get_all_submission_statuses_async( A list of SubmissionStatus objects for the evaluation queue. Example: Getting all submission statuses for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus @@ -695,6 +725,7 @@ async def batch_update_submission_statuses_async( and other response information. Example: Batch update submission statuses +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionStatus From fbd43f6898a0932f3ed0330431cf7ab41832cc1f Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 10:22:54 -0500 Subject: [PATCH 23/46] update submissionbundle docstrings, add more examples --- synapseclient/models/submission_bundle.py | 56 +++++++++++++++-------- synapseclient/models/submission_status.py | 54 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index ce4c736de..2bf1e8430 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,10 +1,9 @@ -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union from synapseclient import Synapse from synapseclient.api import evaluation_services from synapseclient.core.async_utils import async_to_sync -from synapseclient.models.mixins.access_control import AccessControllable if TYPE_CHECKING: from synapseclient.models.submission import Submission @@ -41,7 +40,11 @@ def get_evaluation_submission_bundles( A list of SubmissionBundle objects containing the submission bundles for the evaluation queue. + Note: + The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. + Example: Getting submission bundles for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -87,6 +90,7 @@ def get_user_submission_bundles( submission bundles for the evaluation queue. Example: Getting user submission bundles +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -95,12 +99,12 @@ def get_user_submission_bundles( syn.login() bundles = SubmissionBundle.get_user_submission_bundles( - evaluation_id="9614543", + evaluation_id="9999999", limit=25 ) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ return [] @@ -108,13 +112,11 @@ def get_user_submission_bundles( @dataclass @async_to_sync -class SubmissionBundle( - SubmissionBundleSynchronousProtocol, - AccessControllable, -): +class SubmissionBundle(SubmissionBundleSynchronousProtocol): """A `SubmissionBundle` object represents a bundle containing a Synapse Submission and its accompanying SubmissionStatus. This bundle provides convenient access to both the submission data and its current status in a single object. + Attributes: @@ -124,6 +126,7 @@ class SubmissionBundle( This object should be used to contain scoring data about the Submission. Example: Retrieve submission bundles for an evaluation. +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -141,6 +144,26 @@ class SubmissionBundle( print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") print(f"Status: {bundle.submission_status.status if bundle.submission_status else 'N/A'}") ``` + + Example: Retrieve user submission bundles for an evaluation. +   + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionBundle + + syn = Synapse() + syn.login() + + # Get current user's submission bundles for an evaluation + user_bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id="9999999", + limit=25 + ) + + for bundle in user_bundles: + print(f"Submission ID: {bundle.submission.id}") + print(f"Status: {bundle.submission_status.status}") + ``` """ submission: Optional["Submission"] = None @@ -155,12 +178,6 @@ class SubmissionBundle( This object should be used to contain scoring data about the Submission. """ - _last_persistent_instance: Optional["SubmissionBundle"] = field( - default=None, repr=False, compare=False - ) - """The last persistent instance of this object. This is used to determine if the - object has been changed and needs to be updated in Synapse.""" - def fill_from_dict( self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] ) -> "SubmissionBundle": @@ -223,6 +240,7 @@ async def get_evaluation_submission_bundles_async( The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. Example: Getting submission bundles for an evaluation +   ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -231,13 +249,13 @@ async def get_evaluation_submission_bundles_async( syn.login() bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id="9614543", + evaluation_id="9999999", status="SCORED", limit=50 ) print(f"Found {len(bundles)} submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ response = await evaluation_services.get_evaluation_submission_bundles( @@ -281,6 +299,8 @@ async def get_user_submission_bundles_async( submission bundles for the evaluation queue. Example: Getting user submission bundles +   + ```python from synapseclient import Synapse from synapseclient.models import SubmissionBundle @@ -289,12 +309,12 @@ async def get_user_submission_bundles_async( syn.login() bundles = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id="9614543", + evaluation_id="9999999", limit=25 ) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: - print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") + print(f"Submission ID: {bundle.submission.id}") ``` """ response = await evaluation_services.get_user_submission_bundles( diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 4d4360ae9..93cc9e516 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -259,6 +259,60 @@ class SubmissionStatus( status = status.store() print(status) ``` + + Example: Get all submission statuses for an evaluation. +   + Retrieve multiple submission statuses for an evaluation queue with optional filtering. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Get all RECEIVED statuses for an evaluation + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9999999", + status="RECEIVED", + limit=100 + ) + + print(f"Found {len(statuses)} submission statuses") + for status in statuses: + print(f"Status ID: {status.id}, Status: {status.status}") + ``` + + Example: Batch update multiple submission statuses. +   + Efficiently update multiple submission statuses in a single operation. + ```python + from synapseclient import Synapse + from synapseclient.models import SubmissionStatus + + syn = Synapse() + syn.login() + + # Retrieve statuses to update + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id="9999999", + status="RECEIVED" + ) + + # Update each status + for status in statuses: + status.status = "SCORED" + status.submission_annotations = { + "validation_score": 95.0, + "comments": "Passed validation" + } + + # Batch update all statuses + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id="9614543", + statuses=statuses + ) + print(f"Batch update completed: {response}") + ``` """ id: Optional[str] = None From d5fac7eeb85840c18acb16c699e1f9f6d73715e0 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 11:27:54 -0500 Subject: [PATCH 24/46] initial sync test for status. moved evaluation_id retrieval to fill_from_dict for submissionstatus calls --- .../synchronous/test_submission_status.py | 748 ++++++++++++++++++ 1 file changed, 748 insertions(+) create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission_status.py diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py new file mode 100644 index 000000000..30b70e4bc --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -0,0 +1,748 @@ +"""Integration tests for the synapseclient.models.SubmissionStatus class.""" + +import os +import tempfile +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus + + +class TestSubmissionStatusRetrieval: + """Tests for retrieving SubmissionStatus objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_status_by_id( + self, test_submission: Submission, test_evaluation: Evaluation + ): + """Test retrieving a submission status by ID.""" + # WHEN I get a submission status by ID + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + + # THEN the submission status should be retrieved correctly + assert submission_status.id == test_submission.id + assert submission_status.entity_id == test_submission.entity_id + assert submission_status.evaluation_id == test_evaluation.id + assert submission_status.status is not None # Should have some status (e.g., "RECEIVED") + assert submission_status.etag is not None + assert submission_status.status_version is not None + assert submission_status.modified_on is not None + + async def test_get_submission_status_without_id(self): + """Test that getting a submission status without ID raises ValueError.""" + # WHEN I try to get a submission status without an ID + submission_status = SubmissionStatus() + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + submission_status.get(synapse_client=self.syn) + + async def test_get_submission_status_with_invalid_id(self): + """Test that getting a submission status with invalid ID raises exception.""" + # WHEN I try to get a submission status with an invalid ID + submission_status = SubmissionStatus(id="syn999999999999") + + # THEN it should raise a SynapseHTTPError (404) + with pytest.raises(SynapseHTTPError): + submission_status.get(synapse_client=self.syn) + + +class TestSubmissionStatusUpdates: + """Tests for updating SubmissionStatus objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + @pytest.fixture(scope="function") + async def test_submission_status( + self, test_submission: Submission + ) -> SubmissionStatus: + """Create a test submission status by getting the existing one.""" + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + return submission_status + + async def test_store_submission_status_with_status_change( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with a status change.""" + # GIVEN a submission status that exists + original_status = test_submission_status.status + original_etag = test_submission_status.etag + + # WHEN I update the status + test_submission_status.status = "VALIDATED" + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the submission status should be updated + assert updated_status.id == test_submission_status.id + assert updated_status.status == "VALIDATED" + assert updated_status.status != original_status + assert updated_status.etag != original_etag # etag should change + assert updated_status.status_version > test_submission_status.status_version + + async def test_store_submission_status_with_submission_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with submission annotations.""" + # WHEN I add submission annotations and store + test_submission_status.submission_annotations = { + "score": 85.5, + "validation_passed": True, + "feedback": "Good work!", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the submission annotations should be saved + assert updated_status.submission_annotations is not None + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == 85.5 + assert updated_status.submission_annotations["validation_passed"] == True + assert updated_status.submission_annotations["feedback"] == "Good work!" + + async def test_store_submission_status_with_legacy_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with legacy annotations.""" + # WHEN I add legacy annotations and store + test_submission_status.annotations = { + "internal_score": 92.3, + "reviewer_notes": "Excellent submission", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN the legacy annotations should be saved + assert updated_status.annotations is not None + assert "internal_score" in updated_status.annotations + assert updated_status.annotations["internal_score"] == 92.3 + assert updated_status.annotations["reviewer_notes"] == "Excellent submission" + + async def test_store_submission_status_with_combined_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with both types of annotations.""" + # WHEN I add both submission and legacy annotations + test_submission_status.submission_annotations = { + "public_score": 78.0, + "category": "Bronze", + } + test_submission_status.annotations = { + "internal_review": True, + "notes": "Needs minor improvements", + } + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN both types of annotations should be saved + assert updated_status.submission_annotations is not None + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == 78.0 + + assert updated_status.annotations is not None + assert "internal_review" in updated_status.annotations + assert updated_status.annotations["internal_review"] == True + + async def test_store_submission_status_with_private_annotations_false( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to False.""" + # WHEN I add legacy annotations with private_status_annotations set to False + test_submission_status.annotations = { + "public_internal_score": 88.5, + "public_notes": "This should be visible", + } + test_submission_status.private_status_annotations = False + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the annotations should be marked as not private in the request + assert "annotations" in request_body + annotations_data = request_body["annotations"] + assert "isPrivate" in annotations_data + assert annotations_data["isPrivate"] is False + + async def test_store_submission_status_with_private_annotations_true( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to True (default).""" + # WHEN I add legacy annotations with private_status_annotations set to True (default) + test_submission_status.annotations = { + "private_internal_score": 95.0, + "private_notes": "This should be private", + } + test_submission_status.private_status_annotations = True + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the annotations should be marked as private in the request + assert "annotations" in request_body + annotations_data = request_body["annotations"] + assert "isPrivate" in annotations_data + assert annotations_data["isPrivate"] is True + + async def test_store_submission_status_without_id(self): + """Test that storing a submission status without ID raises ValueError.""" + # WHEN I try to store a submission status without an ID + submission_status = SubmissionStatus(status="SCORED") + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to update"): + submission_status.store(synapse_client=self.syn) + + async def test_store_submission_status_without_changes( + self, test_submission_status: SubmissionStatus + ): + """Test that storing a submission status without changes shows warning.""" + # GIVEN a submission status that hasn't been modified + # (it already has _last_persistent_instance set from get()) + + # WHEN I try to store it without making changes + result = test_submission_status.store(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is test_submission_status + + async def test_store_submission_status_change_tracking( + self, test_submission_status: SubmissionStatus + ): + """Test that change tracking works correctly.""" + # GIVEN a submission status that was retrieved (has_changed should be False) + assert not test_submission_status.has_changed + + # WHEN I make a change + test_submission_status.status = "SCORED" + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store the changes + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN has_changed should be False again + assert not updated_status.has_changed + + +class TestSubmissionStatusBulkOperations: + """Tests for bulk SubmissionStatus operations.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_files( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[File]: + """Create multiple test files for submission status tests.""" + files = [] + for i in range(3): + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write(f"This is test content {i} for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{i}_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + files.append(file) + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + return files + + @pytest.fixture(scope="function") + async def test_submissions( + self, + test_evaluation: Evaluation, + test_files: list[File], + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple test submissions for status tests.""" + submissions = [] + for i, file in enumerate(test_files): + submission = Submission( + entity_id=file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {i} {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + submissions.append(created_submission) + return submissions + + async def test_get_all_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting all submission statuses for an evaluation.""" + # WHEN I get all submission statuses for the evaluation + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should get submission statuses for all submissions + assert len(statuses) >= len(test_submissions) + status_ids = [status.id for status in statuses] + + # AND all test submissions should have their statuses in the results + for submission in test_submissions: + assert submission.id in status_ids + + # AND each status should have proper attributes + for status in statuses: + assert status.id is not None + assert status.evaluation_id == test_evaluation.id + assert status.status is not None + assert status.etag is not None + + async def test_get_all_submission_statuses_with_status_filter( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with status filter.""" + # WHEN I get submission statuses filtered by status + statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should only get statuses with the specified status + for status in statuses: + assert status.status == "RECEIVED" + assert status.evaluation_id == test_evaluation.id + + async def test_get_all_submission_statuses_with_pagination( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with pagination.""" + # WHEN I get submission statuses with pagination + statuses_page1 = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 statuses + assert len(statuses_page1) <= 2 + + # WHEN I get the next page + statuses_page2 = SubmissionStatus.get_all_submission_statuses( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN the results should be different (assuming more than 2 submissions exist) + if len(statuses_page1) == 2 and len(statuses_page2) > 0: + page1_ids = {status.id for status in statuses_page1} + page2_ids = {status.id for status in statuses_page2} + assert page1_ids != page2_ids # Should be different sets + + async def test_batch_update_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating multiple submission statuses.""" + # GIVEN multiple submission statuses + statuses = [] + for submission in test_submissions: + status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) + # Update each status + status.status = "VALIDATED" + status.submission_annotations = { + "batch_score": 90.0 + (len(statuses) * 2), + "batch_processed": True, + } + statuses.append(status) + + # WHEN I batch update the statuses + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the batch update should succeed + assert response is not None + assert "batchToken" in response or response == {} # Response format may vary + + # AND I should be able to verify the updates by retrieving the statuses + for original_status in statuses: + updated_status = SubmissionStatus(id=original_status.id).get( + synapse_client=self.syn + ) + assert updated_status.status == "VALIDATED" + assert "batch_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["batch_processed"] == True + + async def test_batch_update_submission_statuses_large_batch( + self, test_evaluation: Evaluation + ): + """Test batch update behavior with larger batch (approaching limits).""" + # Note: This test demonstrates the pattern but doesn't create 500 submissions + # as that would be too expensive for regular test runs + + # GIVEN I have a list of statuses (simulated for this test) + statuses = [] + + # WHEN I try to batch update (even with empty list) + response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the operation should complete without error + assert response is not None + + async def test_batch_update_submission_statuses_with_batch_tokens( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating with batch tokens for multi-batch operations.""" + if len(test_submissions) < 2: + pytest.skip("Need at least 2 submissions for batch token test") + + # GIVEN multiple statuses split into batches + all_statuses = [] + for submission in test_submissions: + status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) + status.status = "SCORED" + all_statuses.append(status) + + # Split into batches + batch1 = all_statuses[:1] + batch2 = all_statuses[1:] + + # WHEN I update the first batch + response1 = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=batch1, + is_first_batch=True, + is_last_batch=len(batch2) == 0, + synapse_client=self.syn, + ) + + # THEN I should get a response (possibly with batch token) + assert response1 is not None + + # IF there's a second batch and we got a batch token + if len(batch2) > 0: + batch_token = response1.get("batchToken") if isinstance(response1, dict) else None + + # WHEN I update the second batch with the token + response2 = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=test_evaluation.id, + statuses=batch2, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the second batch should also succeed + assert response2 is not None + + +class TestSubmissionStatusValidation: + """Tests for SubmissionStatus validation and error handling.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_to_synapse_request_missing_required_attributes(self): + """Test that to_synapse_request validates required attributes.""" + # WHEN I try to create a request with missing required attributes + submission_status = SubmissionStatus(id="123") # Missing etag, status_version + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + # WHEN I add etag but still missing status_version + submission_status.etag = "some-etag" + + # THEN it should raise a ValueError for status_version + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): + """Test that annotations require evaluation_id.""" + # WHEN I try to create a request with annotations but no evaluation_id + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + annotations={"test": "value"} + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_valid_attributes(self): + """Test that to_synapse_request works with valid attributes.""" + # WHEN I create a request with all required attributes + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + status="SCORED", + evaluation_id="eval123", + submission_annotations={"score": 85.5} + ) + + # THEN it should create a valid request body + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # AND the request should have the required fields + assert request_body["id"] == "123" + assert request_body["etag"] == "some-etag" + assert request_body["statusVersion"] == 1 + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + + async def test_fill_from_dict_with_complete_response(self): + """Test filling a SubmissionStatus from a complete API response.""" + # GIVEN a complete API response + api_response = { + "id": "123456", + "etag": "abcd-1234", + "modifiedOn": "2023-01-01T00:00:00.000Z", + "status": "SCORED", + "entityId": "syn789", + "versionNumber": 1, + "statusVersion": 2, + "canCancel": False, + "cancelRequested": False, + "annotations": { + "stringAnnos": { + "internal_note": ["This is internal"] + }, + "doubleAnnos": {}, + "longAnnos": {} + }, + "submissionAnnotations": { + "stringAnnos": { + "feedback": ["Great work!"] + }, + "doubleAnnos": { + "score": [92.5] + }, + "longAnnos": {} + } + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN all fields should be populated correctly + assert result.id == "123456" + assert result.etag == "abcd-1234" + assert result.modified_on == "2023-01-01T00:00:00.000Z" + assert result.status == "SCORED" + assert result.entity_id == "syn789" + assert result.version_number == 1 + assert result.status_version == 2 + assert result.can_cancel is False + assert result.cancel_requested is False + assert "internal_note" in result.annotations + assert result.annotations["internal_note"] == "This is internal" + assert "feedback" in result.submission_annotations + assert "score" in result.submission_annotations + assert result.submission_annotations["score"] == 92.5 + + async def test_fill_from_dict_with_minimal_response(self): + """Test filling a SubmissionStatus from a minimal API response.""" + # GIVEN a minimal API response + api_response = { + "id": "123456", + "status": "RECEIVED" + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN basic fields should be populated + assert result.id == "123456" + assert result.status == "RECEIVED" + # AND optional fields should have default values + assert result.etag is None + assert result.can_cancel is False + assert result.cancel_requested is False From 47797c0af7155f8a1acf61831e92c9f2052952e2 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 12:29:56 -0500 Subject: [PATCH 25/46] update submissionBundle submissionstatus with evaluation_id --- synapseclient/models/submission_bundle.py | 16 ++-- synapseclient/models/submission_status.py | 5 +- .../synchronous/test_submission_status.py | 90 ++++++++++--------- 3 files changed, 59 insertions(+), 52 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 2bf1e8430..89f4fc50c 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -179,7 +179,8 @@ class SubmissionBundle(SubmissionBundleSynchronousProtocol): """ def fill_from_dict( - self, synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]] + self, + synapse_submission_bundle: Dict[str, Union[bool, str, int, Dict]], ) -> "SubmissionBundle": """ Converts a response from the REST API into this dataclass. @@ -201,9 +202,10 @@ def fill_from_dict( submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict( - submission_status_dict - ) + self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + # Manually set evaluation_id from the submission data if available + if self.submission_status and self.submission and self.submission.evaluation_id: + self.submission_status.evaluation_id = self.submission.evaluation_id else: self.submission_status = None @@ -268,7 +270,8 @@ async def get_evaluation_submission_bundles_async( bundles = [] for bundle_dict in response.get("results", []): - bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundle = SubmissionBundle() + bundle.fill_from_dict(bundle_dict) bundles.append(bundle) return bundles @@ -327,7 +330,8 @@ async def get_user_submission_bundles_async( # Convert response to list of SubmissionBundle objects bundles = [] for bundle_dict in response.get("results", []): - bundle = SubmissionBundle().fill_from_dict(bundle_dict) + bundle = SubmissionBundle() + bundle.fill_from_dict(bundle_dict) bundles.append(bundle) return bundles diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 93cc9e516..d19ac9494 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -432,7 +432,8 @@ def _set_last_persistent_instance(self) -> None: self._last_persistent_instance = replace(self) def fill_from_dict( - self, synapse_submission_status: Dict[str, Union[bool, str, int, float, List]] + self, + synapse_submission_status: Dict[str, Union[bool, str, int, float, List]], ) -> "SubmissionStatus": """ Converts a response from the REST API into this dataclass. @@ -744,7 +745,7 @@ async def get_all_submission_statuses_async( for status_dict in response.get("results", []): submission_status = SubmissionStatus() submission_status.fill_from_dict(status_dict) - # Manually set evaluation_id since it's not part of the response + # Manually set evaluation_id since it's not in the SubmissionStatus response submission_status.evaluation_id = evaluation_id submission_status._set_last_persistent_instance() submission_statuses.append(submission_status) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index 30b70e4bc..e68574f51 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -97,7 +97,9 @@ async def test_get_submission_status_by_id( assert submission_status.id == test_submission.id assert submission_status.entity_id == test_submission.entity_id assert submission_status.evaluation_id == test_evaluation.id - assert submission_status.status is not None # Should have some status (e.g., "RECEIVED") + assert ( + submission_status.status is not None + ) # Should have some status (e.g., "RECEIVED") assert submission_status.etag is not None assert submission_status.status_version is not None assert submission_status.modified_on is not None @@ -108,7 +110,9 @@ async def test_get_submission_status_without_id(self): submission_status = SubmissionStatus() # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): submission_status.get(synapse_client=self.syn) async def test_get_submission_status_with_invalid_id(self): @@ -229,7 +233,7 @@ async def test_store_submission_status_with_submission_annotations( # WHEN I add submission annotations and store test_submission_status.submission_annotations = { "score": 85.5, - "validation_passed": True, + "validation_passed": [True], "feedback": "Good work!", } updated_status = test_submission_status.store(synapse_client=self.syn) @@ -237,9 +241,9 @@ async def test_store_submission_status_with_submission_annotations( # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None assert "score" in updated_status.submission_annotations - assert updated_status.submission_annotations["score"] == 85.5 - assert updated_status.submission_annotations["validation_passed"] == True - assert updated_status.submission_annotations["feedback"] == "Good work!" + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["validation_passed"] == [True] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -255,8 +259,8 @@ async def test_store_submission_status_with_legacy_annotations( # THEN the legacy annotations should be saved assert updated_status.annotations is not None assert "internal_score" in updated_status.annotations - assert updated_status.annotations["internal_score"] == 92.3 - assert updated_status.annotations["reviewer_notes"] == "Excellent submission" + assert updated_status.annotations["internal_score"] == [92.3] + assert updated_status.annotations["reviewer_notes"] == ["Excellent submission"] async def test_store_submission_status_with_combined_annotations( self, test_submission_status: SubmissionStatus @@ -276,11 +280,11 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None assert "public_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["public_score"] == 78.0 + assert updated_status.submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None assert "internal_review" in updated_status.annotations - assert updated_status.annotations["internal_review"] == True + assert updated_status.annotations["internal_review"] == [True] async def test_store_submission_status_with_private_annotations_false( self, test_submission_status: SubmissionStatus @@ -292,10 +296,12 @@ async def test_store_submission_status_with_private_annotations_false( "public_notes": "This should be visible", } test_submission_status.private_status_annotations = False - + # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) - + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + # THEN the annotations should be marked as not private in the request assert "annotations" in request_body annotations_data = request_body["annotations"] @@ -312,10 +318,12 @@ async def test_store_submission_status_with_private_annotations_true( "private_notes": "This should be private", } test_submission_status.private_status_annotations = True - + # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request(synapse_client=self.syn) - + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + # THEN the annotations should be marked as private in the request assert "annotations" in request_body annotations_data = request_body["annotations"] @@ -328,7 +336,9 @@ async def test_store_submission_status_without_id(self): submission_status = SubmissionStatus(status="SCORED") # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to update"): + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): submission_status.store(synapse_client=self.syn) async def test_store_submission_status_without_changes( @@ -405,7 +415,9 @@ async def test_files( with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".txt" ) as temp_file: - temp_file.write(f"This is test content {i} for submission status testing.") + temp_file.write( + f"This is test content {i} for submission status testing." + ) temp_file_path = temp_file.name try: @@ -546,7 +558,7 @@ async def test_batch_update_submission_statuses( ) assert updated_status.status == "VALIDATED" assert "batch_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["batch_processed"] == True + assert updated_status.submission_annotations["batch_processed"] == [True] async def test_batch_update_submission_statuses_large_batch( self, test_evaluation: Evaluation @@ -554,10 +566,10 @@ async def test_batch_update_submission_statuses_large_batch( """Test batch update behavior with larger batch (approaching limits).""" # Note: This test demonstrates the pattern but doesn't create 500 submissions # as that would be too expensive for regular test runs - + # GIVEN I have a list of statuses (simulated for this test) statuses = [] - + # WHEN I try to batch update (even with empty list) response = SubmissionStatus.batch_update_submission_statuses( evaluation_id=test_evaluation.id, @@ -600,8 +612,10 @@ async def test_batch_update_submission_statuses_with_batch_tokens( # IF there's a second batch and we got a batch token if len(batch2) > 0: - batch_token = response1.get("batchToken") if isinstance(response1, dict) else None - + batch_token = ( + response1.get("batchToken") if isinstance(response1, dict) else None + ) + # WHEN I update the second batch with the token response2 = SubmissionStatus.batch_update_submission_statuses( evaluation_id=test_evaluation.id, @@ -644,10 +658,7 @@ async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): """Test that annotations require evaluation_id.""" # WHEN I try to create a request with annotations but no evaluation_id submission_status = SubmissionStatus( - id="123", - etag="some-etag", - status_version=1, - annotations={"test": "value"} + id="123", etag="some-etag", status_version=1, annotations={"test": "value"} ) # THEN it should raise a ValueError @@ -663,7 +674,7 @@ async def test_to_synapse_request_valid_attributes(self): status_version=1, status="SCORED", evaluation_id="eval123", - submission_annotations={"score": 85.5} + submission_annotations={"score": 85.5}, ) # THEN it should create a valid request body @@ -690,21 +701,15 @@ async def test_fill_from_dict_with_complete_response(self): "canCancel": False, "cancelRequested": False, "annotations": { - "stringAnnos": { - "internal_note": ["This is internal"] - }, + "stringAnnos": {"internal_note": ["This is internal"]}, "doubleAnnos": {}, - "longAnnos": {} + "longAnnos": {}, }, "submissionAnnotations": { - "stringAnnos": { - "feedback": ["Great work!"] - }, - "doubleAnnos": { - "score": [92.5] - }, - "longAnnos": {} - } + "stringAnnos": {"feedback": ["Great work!"]}, + "doubleAnnos": {"score": [92.5]}, + "longAnnos": {}, + }, } # WHEN I fill a SubmissionStatus from the response @@ -730,10 +735,7 @@ async def test_fill_from_dict_with_complete_response(self): async def test_fill_from_dict_with_minimal_response(self): """Test filling a SubmissionStatus from a minimal API response.""" # GIVEN a minimal API response - api_response = { - "id": "123456", - "status": "RECEIVED" - } + api_response = {"id": "123456", "status": "RECEIVED"} # WHEN I fill a SubmissionStatus from the response submission_status = SubmissionStatus() From e8e5361b0fe0c45f2d3c5c83b0812a127a8bb1c0 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 14 Nov 2025 16:22:50 -0500 Subject: [PATCH 26/46] patch sync substatus integ tests. style. --- synapseclient/models/submission_bundle.py | 10 +++- synapseclient/models/submission_status.py | 1 - .../synchronous/test_submission_status.py | 48 ++++++++++++------- 3 files changed, 40 insertions(+), 19 deletions(-) diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 89f4fc50c..6c7ae42e4 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -202,9 +202,15 @@ def fill_from_dict( submission_status_dict = synapse_submission_bundle.get("submissionStatus", None) if submission_status_dict: - self.submission_status = SubmissionStatus().fill_from_dict(submission_status_dict) + self.submission_status = SubmissionStatus().fill_from_dict( + submission_status_dict + ) # Manually set evaluation_id from the submission data if available - if self.submission_status and self.submission and self.submission.evaluation_id: + if ( + self.submission_status + and self.submission + and self.submission.evaluation_id + ): self.submission_status.evaluation_id = self.submission.evaluation_id else: self.submission_status = None diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index d19ac9494..f50e099fc 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -665,7 +665,6 @@ async def store_async( "modified_on", "entity_id", "version_number", - "status_version", ], logger=logger, ) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index e68574f51..ccc3c4083 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -8,6 +8,7 @@ import pytest from synapseclient import Synapse +from synapseclient.annotations import from_submission_status_annotations from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus @@ -214,6 +215,7 @@ async def test_store_submission_status_with_status_change( # GIVEN a submission status that exists original_status = test_submission_status.status original_etag = test_submission_status.etag + original_status_version = test_submission_status.status_version # WHEN I update the status test_submission_status.status = "VALIDATED" @@ -224,7 +226,7 @@ async def test_store_submission_status_with_status_change( assert updated_status.status == "VALIDATED" assert updated_status.status != original_status assert updated_status.etag != original_etag # etag should change - assert updated_status.status_version > test_submission_status.status_version + assert updated_status.status_version > original_status_version async def test_store_submission_status_with_submission_annotations( self, test_submission_status: SubmissionStatus @@ -233,17 +235,18 @@ async def test_store_submission_status_with_submission_annotations( # WHEN I add submission annotations and store test_submission_status.submission_annotations = { "score": 85.5, - "validation_passed": [True], "feedback": "Good work!", } updated_status = test_submission_status.store(synapse_client=self.syn) # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None - assert "score" in updated_status.submission_annotations - assert updated_status.submission_annotations["score"] == [85.5] - assert updated_status.submission_annotations["validation_passed"] == [True] - assert updated_status.submission_annotations["feedback"] == ["Good work!"] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "score" in converted_submission_annotations + assert converted_submission_annotations["score"] == [85.5] + assert converted_submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -255,12 +258,16 @@ async def test_store_submission_status_with_legacy_annotations( "reviewer_notes": "Excellent submission", } updated_status = test_submission_status.store(synapse_client=self.syn) + assert updated_status.annotations is not None + + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) # THEN the legacy annotations should be saved - assert updated_status.annotations is not None - assert "internal_score" in updated_status.annotations - assert updated_status.annotations["internal_score"] == [92.3] - assert updated_status.annotations["reviewer_notes"] == ["Excellent submission"] + assert "internal_score" in converted_annotations + assert converted_annotations["internal_score"] == 92.3 + assert converted_annotations["reviewer_notes"] == "Excellent submission" async def test_store_submission_status_with_combined_annotations( self, test_submission_status: SubmissionStatus @@ -279,12 +286,18 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None - assert "public_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["public_score"] == [78.0] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "public_score" in converted_submission_annotations + assert converted_submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None - assert "internal_review" in updated_status.annotations - assert updated_status.annotations["internal_review"] == [True] + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "internal_review" in converted_annotations + assert converted_annotations["internal_review"] == "true" async def test_store_submission_status_with_private_annotations_false( self, test_submission_status: SubmissionStatus @@ -557,8 +570,11 @@ async def test_batch_update_submission_statuses( synapse_client=self.syn ) assert updated_status.status == "VALIDATED" - assert "batch_score" in updated_status.submission_annotations - assert updated_status.submission_annotations["batch_processed"] == [True] + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "batch_score" in converted_submission_annotations + assert converted_submission_annotations["batch_processed"] == ["true"] async def test_batch_update_submission_statuses_large_batch( self, test_evaluation: Evaluation From 58ffe4276da71cf6b156960b725ea8d40a5e05b9 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 17 Nov 2025 16:18:45 -0500 Subject: [PATCH 27/46] fix submissionStatus integ tests and has_changed attribute --- synapseclient/models/submission_status.py | 17 +- .../synchronous/test_submission_status.py | 224 ++++++++++-------- 2 files changed, 135 insertions(+), 106 deletions(-) diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index f50e099fc..1e46d61b7 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -421,9 +421,22 @@ class SubmissionStatus( @property def has_changed(self) -> bool: """Determines if the object has been newly created OR changed since last retrieval, and needs to be updated in Synapse.""" + if not self._last_persistent_instance: + return True + + model_attributes_changed = self._last_persistent_instance != self + annotations_changed = ( + self._last_persistent_instance.annotations != self.annotations + ) + submission_annotations_changed = ( + self._last_persistent_instance.submission_annotations + != self.submission_annotations + ) + return ( - not self._last_persistent_instance - or self._last_persistent_instance is not self + model_attributes_changed + or annotations_changed + or submission_annotations_changed ) def _set_last_persistent_instance(self) -> None: diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index ccc3c4083..b2c4bd65c 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -2,6 +2,7 @@ import os import tempfile +import test import uuid from typing import Callable @@ -241,12 +242,9 @@ async def test_store_submission_status_with_submission_annotations( # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None - converted_submission_annotations = from_submission_status_annotations( - updated_status.submission_annotations - ) - assert "score" in converted_submission_annotations - assert converted_submission_annotations["score"] == [85.5] - assert converted_submission_annotations["feedback"] == ["Good work!"] + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] async def test_store_submission_status_with_legacy_annotations( self, test_submission_status: SubmissionStatus @@ -286,11 +284,8 @@ async def test_store_submission_status_with_combined_annotations( # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None - converted_submission_annotations = from_submission_status_annotations( - updated_status.submission_annotations - ) - assert "public_score" in converted_submission_annotations - assert converted_submission_annotations["public_score"] == [78.0] + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == [78.0] assert updated_status.annotations is not None converted_annotations = from_submission_status_annotations( @@ -310,16 +305,22 @@ async def test_store_submission_status_with_private_annotations_false( } test_submission_status.private_status_annotations = False - # AND I create the request body to inspect it - request_body = test_submission_status.to_synapse_request( - synapse_client=self.syn + # WHEN I store the submission status + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations ) + assert "public_internal_score" in converted_annotations + assert converted_annotations["public_internal_score"] == 88.5 + assert converted_annotations["public_notes"] == "This should be visible" - # THEN the annotations should be marked as not private in the request - assert "annotations" in request_body - annotations_data = request_body["annotations"] - assert "isPrivate" in annotations_data - assert annotations_data["isPrivate"] is False + # AND the annotations should be marked as not private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + assert all(not anno["isPrivate"] for anno in annotations) async def test_store_submission_status_with_private_annotations_true( self, test_submission_status: SubmissionStatus @@ -337,11 +338,23 @@ async def test_store_submission_status_with_private_annotations_true( synapse_client=self.syn ) - # THEN the annotations should be marked as private in the request - assert "annotations" in request_body - annotations_data = request_body["annotations"] - assert "isPrivate" in annotations_data - assert annotations_data["isPrivate"] is True + # WHEN I store the submission status + updated_status = test_submission_status.store(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "private_internal_score" in converted_annotations + assert converted_annotations["private_internal_score"] == 95.0 + assert converted_annotations["private_notes"] == "This should be private" + + # AND the annotations should be marked as private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + print(annotations) + assert all(anno["isPrivate"] for anno in annotations) async def test_store_submission_status_without_id(self): """Test that storing a submission status without ID raises ValueError.""" @@ -386,6 +399,57 @@ async def test_store_submission_status_change_tracking( # THEN has_changed should be False again assert not updated_status.has_changed + async def test_has_changed_property_edge_cases( + self, test_submission_status: SubmissionStatus + ): + """Test the has_changed property with various edge cases and detailed scenarios.""" + # GIVEN a submission status that was just retrieved + assert not test_submission_status.has_changed + original_annotations = ( + test_submission_status.annotations.copy() + if test_submission_status.annotations + else {} + ) + + # WHEN I modify only annotations (not submission_annotations) + test_submission_status.annotations = {"test_key": "test_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I reset annotations to the original value (should be the same as the persistent instance) + test_submission_status.annotations = original_annotations + + # THEN has_changed should be False (same as original) + assert not test_submission_status.has_changed + + # WHEN I add a different annotation value + test_submission_status.annotations = {"different_key": "different_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store and get a fresh copy + updated_status = test_submission_status.store(synapse_client=self.syn) + fresh_status = SubmissionStatus(id=updated_status.id).get( + synapse_client=self.syn + ) + + # THEN the fresh copy should not have changes + assert not fresh_status.has_changed + + # WHEN I modify only submission_annotations + fresh_status.submission_annotations = {"new_key": ["new_value"]} + + # THEN has_changed should be True + assert fresh_status.has_changed + + # WHEN I modify a scalar field + fresh_status.status = "VALIDATED" + + # THEN has_changed should still be True + assert fresh_status.has_changed + class TestSubmissionStatusBulkOperations: """Tests for bulk SubmissionStatus operations.""" @@ -576,75 +640,6 @@ async def test_batch_update_submission_statuses( assert "batch_score" in converted_submission_annotations assert converted_submission_annotations["batch_processed"] == ["true"] - async def test_batch_update_submission_statuses_large_batch( - self, test_evaluation: Evaluation - ): - """Test batch update behavior with larger batch (approaching limits).""" - # Note: This test demonstrates the pattern but doesn't create 500 submissions - # as that would be too expensive for regular test runs - - # GIVEN I have a list of statuses (simulated for this test) - statuses = [] - - # WHEN I try to batch update (even with empty list) - response = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=statuses, - synapse_client=self.syn, - ) - - # THEN the operation should complete without error - assert response is not None - - async def test_batch_update_submission_statuses_with_batch_tokens( - self, test_evaluation: Evaluation, test_submissions: list[Submission] - ): - """Test batch updating with batch tokens for multi-batch operations.""" - if len(test_submissions) < 2: - pytest.skip("Need at least 2 submissions for batch token test") - - # GIVEN multiple statuses split into batches - all_statuses = [] - for submission in test_submissions: - status = SubmissionStatus(id=submission.id).get(synapse_client=self.syn) - status.status = "SCORED" - all_statuses.append(status) - - # Split into batches - batch1 = all_statuses[:1] - batch2 = all_statuses[1:] - - # WHEN I update the first batch - response1 = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=batch1, - is_first_batch=True, - is_last_batch=len(batch2) == 0, - synapse_client=self.syn, - ) - - # THEN I should get a response (possibly with batch token) - assert response1 is not None - - # IF there's a second batch and we got a batch token - if len(batch2) > 0: - batch_token = ( - response1.get("batchToken") if isinstance(response1, dict) else None - ) - - # WHEN I update the second batch with the token - response2 = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=test_evaluation.id, - statuses=batch2, - is_first_batch=False, - is_last_batch=True, - batch_token=batch_token, - synapse_client=self.syn, - ) - - # THEN the second batch should also succeed - assert response2 is not None - class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling.""" @@ -717,15 +712,26 @@ async def test_fill_from_dict_with_complete_response(self): "canCancel": False, "cancelRequested": False, "annotations": { - "stringAnnos": {"internal_note": ["This is internal"]}, - "doubleAnnos": {}, - "longAnnos": {}, - }, - "submissionAnnotations": { - "stringAnnos": {"feedback": ["Great work!"]}, - "doubleAnnos": {"score": [92.5]}, - "longAnnos": {}, + "objectId": "123456", + "scopeId": "9617645", + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + }, + { + "key": "reviewer_notes", + "isPrivate": True, + "value": "Excellent work", + }, + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, } # WHEN I fill a SubmissionStatus from the response @@ -742,11 +748,21 @@ async def test_fill_from_dict_with_complete_response(self): assert result.status_version == 2 assert result.can_cancel is False assert result.cancel_requested is False - assert "internal_note" in result.annotations - assert result.annotations["internal_note"] == "This is internal" + + # The annotations field should contain the raw submission status format + assert result.annotations is not None + assert "objectId" in result.annotations + assert "scopeId" in result.annotations + assert "stringAnnos" in result.annotations + assert "doubleAnnos" in result.annotations + assert len(result.annotations["stringAnnos"]) == 2 + assert len(result.annotations["doubleAnnos"]) == 1 + + # The submission_annotations should be in simple key-value format assert "feedback" in result.submission_annotations assert "score" in result.submission_annotations - assert result.submission_annotations["score"] == 92.5 + assert result.submission_annotations["feedback"] == ["Great work!"] + assert result.submission_annotations["score"] == [92.5] async def test_fill_from_dict_with_minimal_response(self): """Test filling a SubmissionStatus from a minimal API response.""" From f34667dd4a87f14a420cf4160172b4ff4c5a7ddc Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 10:57:45 -0500 Subject: [PATCH 28/46] new substatus async integ tests. can_cancel can now be modified by an organizer on the client. cancel request returns no response body. --- synapseclient/models/submission.py | 8 +- synapseclient/models/submission_status.py | 6 +- .../async/test_submission_status_async.py | 785 ++++++++++++++++++ 3 files changed, 790 insertions(+), 9 deletions(-) create mode 100644 tests/integration/synapseclient/models/async/test_submission_status_async.py diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 5d7390050..7096201c9 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -699,7 +699,7 @@ async def cancel_submission_example(): if not self.id: raise ValueError("The submission must have an ID to cancel.") - response = await evaluation_services.cancel_submission( + await evaluation_services.cancel_submission( submission_id=self.id, synapse_client=synapse_client ) @@ -707,8 +707,4 @@ async def cancel_submission_example(): client = Synapse.get_client(synapse_client=synapse_client) logger = client.logger - logger.info(f"Submission {self.id} has successfully been cancelled.") - - # Update this object with the response - self.fill_from_dict(response) - return self + logger.info(f"A request to cancel Submission {self.id} has been submitted.") diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 1e46d61b7..60787be32 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -400,10 +400,10 @@ class SubmissionStatus( A version of the status, auto-generated and auto-incremented by the system and read-only to the client. """ - can_cancel: Optional[bool] = field(default=False, compare=False) + can_cancel: Optional[bool] = field(default=False) """ - Can this submission be cancelled? By default, this will be set to False. Users can read this value. - Only the queue's scoring application can change this value. + Can this submission be cancelled? By default, this will be set to False. Submission owner can read this value. + Only the queue's organizers can change this value. """ cancel_requested: Optional[bool] = field(default=False, compare=False) diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py new file mode 100644 index 000000000..f46aea107 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -0,0 +1,785 @@ +"""Integration tests for the synapseclient.models.SubmissionStatus class async methods.""" + +import os +import tempfile +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.annotations import from_submission_status_annotations +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionStatus + + +class TestSubmissionStatusRetrieval: + """Tests for retrieving SubmissionStatus objects async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_get_submission_status_by_id( + self, test_submission: Submission, test_evaluation: Evaluation + ): + """Test retrieving a submission status by ID async.""" + # WHEN I get a submission status by ID + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + + # THEN the submission status should be retrieved correctly + assert submission_status.id == test_submission.id + assert submission_status.entity_id == test_submission.entity_id + assert submission_status.evaluation_id == test_evaluation.id + assert ( + submission_status.status is not None + ) # Should have some status (e.g., "RECEIVED") + assert submission_status.etag is not None + assert submission_status.status_version is not None + assert submission_status.modified_on is not None + + async def test_get_submission_status_without_id(self): + """Test that getting a submission status without ID raises ValueError async.""" + # WHEN I try to get a submission status without an ID + submission_status = SubmissionStatus() + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): + await submission_status.get_async(synapse_client=self.syn) + + async def test_get_submission_status_with_invalid_id(self): + """Test that getting a submission status with invalid ID raises exception async.""" + # WHEN I try to get a submission status with an invalid ID + submission_status = SubmissionStatus(id="syn999999999999") + + # THEN it should raise a SynapseHTTPError (404) + with pytest.raises(SynapseHTTPError): + await submission_status.get_async(synapse_client=self.syn) + + +class TestSubmissionStatusUpdates: + """Tests for updating SubmissionStatus objects async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + @pytest.fixture(scope="function") + async def test_submission_status( + self, test_submission: Submission + ) -> SubmissionStatus: + """Create a test submission status by getting the existing one.""" + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + return submission_status + + async def test_store_submission_status_with_status_change( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with a status change async.""" + # GIVEN a submission status that exists + original_status = test_submission_status.status + original_etag = test_submission_status.etag + original_status_version = test_submission_status.status_version + + # WHEN I update the status + test_submission_status.status = "VALIDATED" + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN the submission status should be updated + assert updated_status.id == test_submission_status.id + assert updated_status.status == "VALIDATED" + assert updated_status.status != original_status + assert updated_status.etag != original_etag # etag should change + assert updated_status.status_version > original_status_version + + async def test_store_submission_status_with_submission_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with submission annotations async.""" + # WHEN I add submission annotations and store + test_submission_status.submission_annotations = { + "score": 85.5, + "feedback": "Good work!", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN the submission annotations should be saved + assert updated_status.submission_annotations is not None + assert "score" in updated_status.submission_annotations + assert updated_status.submission_annotations["score"] == [85.5] + assert updated_status.submission_annotations["feedback"] == ["Good work!"] + + async def test_store_submission_status_with_legacy_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with legacy annotations async.""" + # WHEN I add legacy annotations and store + test_submission_status.annotations = { + "internal_score": 92.3, + "reviewer_notes": "Excellent submission", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + assert updated_status.annotations is not None + + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + + # THEN the legacy annotations should be saved + assert "internal_score" in converted_annotations + assert converted_annotations["internal_score"] == 92.3 + assert converted_annotations["reviewer_notes"] == "Excellent submission" + + async def test_store_submission_status_with_combined_annotations( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with both types of annotations async.""" + # WHEN I add both submission and legacy annotations + test_submission_status.submission_annotations = { + "public_score": 78.0, + "category": "Bronze", + } + test_submission_status.annotations = { + "internal_review": True, + "notes": "Needs minor improvements", + } + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN both types of annotations should be saved + assert updated_status.submission_annotations is not None + assert "public_score" in updated_status.submission_annotations + assert updated_status.submission_annotations["public_score"] == [78.0] + + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "internal_review" in converted_annotations + assert converted_annotations["internal_review"] == "true" + + async def test_store_submission_status_with_private_annotations_false( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to False async.""" + # WHEN I add legacy annotations with private_status_annotations set to False + test_submission_status.annotations = { + "public_internal_score": 88.5, + "public_notes": "This should be visible", + } + test_submission_status.private_status_annotations = False + + # WHEN I store the submission status + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "public_internal_score" in converted_annotations + assert converted_annotations["public_internal_score"] == 88.5 + assert converted_annotations["public_notes"] == "This should be visible" + + # AND the annotations should be marked as not private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + assert all(not anno["isPrivate"] for anno in annotations) + + async def test_store_submission_status_with_private_annotations_true( + self, test_submission_status: SubmissionStatus + ): + """Test updating a submission status with private_status_annotations set to True (default) async.""" + # WHEN I add legacy annotations with private_status_annotations set to True (default) + test_submission_status.annotations = { + "private_internal_score": 95.0, + "private_notes": "This should be private", + } + test_submission_status.private_status_annotations = True + + # AND I create the request body to inspect it + request_body = test_submission_status.to_synapse_request( + synapse_client=self.syn + ) + + # WHEN I store the submission status + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN they should be properly stored + assert updated_status.annotations is not None + converted_annotations = from_submission_status_annotations( + updated_status.annotations + ) + assert "private_internal_score" in converted_annotations + assert converted_annotations["private_internal_score"] == 95.0 + assert converted_annotations["private_notes"] == "This should be private" + + # AND the annotations should be marked as private + for annos_type in ["stringAnnos", "doubleAnnos"]: + annotations = updated_status.annotations[annos_type] + print(annotations) + assert all(anno["isPrivate"] for anno in annotations) + + async def test_store_submission_status_without_id(self): + """Test that storing a submission status without ID raises ValueError async.""" + # WHEN I try to store a submission status without an ID + submission_status = SubmissionStatus(status="SCORED") + + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + await submission_status.store_async(synapse_client=self.syn) + + async def test_store_submission_status_without_changes( + self, test_submission_status: SubmissionStatus + ): + """Test that storing a submission status without changes shows warning async.""" + # GIVEN a submission status that hasn't been modified + # (it already has _last_persistent_instance set from get()) + + # WHEN I try to store it without making changes + result = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is test_submission_status + + async def test_store_submission_status_change_tracking( + self, test_submission_status: SubmissionStatus + ): + """Test that change tracking works correctly async.""" + # GIVEN a submission status that was retrieved (has_changed should be False) + assert not test_submission_status.has_changed + + # WHEN I make a change + test_submission_status.status = "SCORED" + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store the changes + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + + # THEN has_changed should be False again + assert not updated_status.has_changed + + async def test_has_changed_property_edge_cases( + self, test_submission_status: SubmissionStatus + ): + """Test the has_changed property with various edge cases and detailed scenarios async.""" + # GIVEN a submission status that was just retrieved + assert not test_submission_status.has_changed + original_annotations = ( + test_submission_status.annotations.copy() + if test_submission_status.annotations + else {} + ) + + # WHEN I modify only annotations (not submission_annotations) + test_submission_status.annotations = {"test_key": "test_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I reset annotations to the original value (should be the same as the persistent instance) + test_submission_status.annotations = original_annotations + + # THEN has_changed should be False (same as original) + assert not test_submission_status.has_changed + + # WHEN I add a different annotation value + test_submission_status.annotations = {"different_key": "different_value"} + + # THEN has_changed should be True + assert test_submission_status.has_changed + + # WHEN I store and get a fresh copy + updated_status = await test_submission_status.store_async(synapse_client=self.syn) + fresh_status = await SubmissionStatus(id=updated_status.id).get_async( + synapse_client=self.syn + ) + + # THEN the fresh copy should not have changes + assert not fresh_status.has_changed + + # WHEN I modify only submission_annotations + fresh_status.submission_annotations = {"new_key": ["new_value"]} + + # THEN has_changed should be True + assert fresh_status.has_changed + + # WHEN I modify a scalar field + fresh_status.status = "VALIDATED" + + # THEN has_changed should still be True + assert fresh_status.has_changed + + +class TestSubmissionStatusBulkOperations: + """Tests for bulk SubmissionStatus operations async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_files( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[File]: + """Create multiple test files for submission status tests.""" + files = [] + for i in range(3): + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write( + f"This is test content {i} for submission status testing." + ) + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{i}_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + files.append(stored_file) + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + return files + + @pytest.fixture(scope="function") + async def test_submissions( + self, + test_evaluation: Evaluation, + test_files: list[File], + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple test submissions for status tests.""" + submissions = [] + for i, file in enumerate(test_files): + submission = Submission( + entity_id=file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {i} {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + submissions.append(created_submission) + return submissions + + async def test_get_all_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting all submission statuses for an evaluation async.""" + # WHEN I get all submission statuses for the evaluation + statuses = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should get submission statuses for all submissions + assert len(statuses) >= len(test_submissions) + status_ids = [status.id for status in statuses] + + # AND all test submissions should have their statuses in the results + for submission in test_submissions: + assert submission.id in status_ids + + # AND each status should have proper attributes + for status in statuses: + assert status.id is not None + assert status.evaluation_id == test_evaluation.id + assert status.status is not None + assert status.etag is not None + + async def test_get_all_submission_statuses_with_status_filter( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with status filter async.""" + # WHEN I get submission statuses filtered by status + statuses = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN I should only get statuses with the specified status + for status in statuses: + assert status.status == "RECEIVED" + assert status.evaluation_id == test_evaluation.id + + async def test_get_all_submission_statuses_with_pagination( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test getting submission statuses with pagination async.""" + # WHEN I get submission statuses with pagination + statuses_page1 = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 statuses + assert len(statuses_page1) <= 2 + + # WHEN I get the next page + statuses_page2 = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN the results should be different (assuming more than 2 submissions exist) + if len(statuses_page1) == 2 and len(statuses_page2) > 0: + page1_ids = {status.id for status in statuses_page1} + page2_ids = {status.id for status in statuses_page2} + assert page1_ids != page2_ids # Should be different sets + + async def test_batch_update_submission_statuses( + self, test_evaluation: Evaluation, test_submissions: list[Submission] + ): + """Test batch updating multiple submission statuses async.""" + # GIVEN multiple submission statuses + statuses = [] + for submission in test_submissions: + status = await SubmissionStatus(id=submission.id).get_async(synapse_client=self.syn) + # Update each status + status.status = "VALIDATED" + status.submission_annotations = { + "batch_score": 90.0 + (len(statuses) * 2), + "batch_processed": True, + } + statuses.append(status) + + # WHEN I batch update the statuses + response = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=test_evaluation.id, + statuses=statuses, + synapse_client=self.syn, + ) + + # THEN the batch update should succeed + assert response is not None + assert "batchToken" in response or response == {} # Response format may vary + + # AND I should be able to verify the updates by retrieving the statuses + for original_status in statuses: + updated_status = await SubmissionStatus(id=original_status.id).get_async( + synapse_client=self.syn + ) + assert updated_status.status == "VALIDATED" + converted_submission_annotations = from_submission_status_annotations( + updated_status.submission_annotations + ) + assert "batch_score" in converted_submission_annotations + assert converted_submission_annotations["batch_processed"] == ["true"] + + +class TestSubmissionStatusValidation: + """Tests for SubmissionStatus validation and error handling async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + async def test_to_synapse_request_missing_required_attributes(self): + """Test that to_synapse_request validates required attributes async.""" + # WHEN I try to create a request with missing required attributes + submission_status = SubmissionStatus(id="123") # Missing etag, status_version + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + # WHEN I add etag but still missing status_version + submission_status.etag = "some-etag" + + # THEN it should raise a ValueError for status_version + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_with_annotations_missing_evaluation_id(self): + """Test that annotations require evaluation_id async.""" + # WHEN I try to create a request with annotations but no evaluation_id + submission_status = SubmissionStatus( + id="123", etag="some-etag", status_version=1, annotations={"test": "value"} + ) + + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + async def test_to_synapse_request_valid_attributes(self): + """Test that to_synapse_request works with valid attributes async.""" + # WHEN I create a request with all required attributes + submission_status = SubmissionStatus( + id="123", + etag="some-etag", + status_version=1, + status="SCORED", + evaluation_id="eval123", + submission_annotations={"score": 85.5}, + ) + + # THEN it should create a valid request body + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # AND the request should have the required fields + assert request_body["id"] == "123" + assert request_body["etag"] == "some-etag" + assert request_body["statusVersion"] == 1 + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + + async def test_fill_from_dict_with_complete_response(self): + """Test filling a SubmissionStatus from a complete API response async.""" + # GIVEN a complete API response + api_response = { + "id": "123456", + "etag": "abcd-1234", + "modifiedOn": "2023-01-01T00:00:00.000Z", + "status": "SCORED", + "entityId": "syn789", + "versionNumber": 1, + "statusVersion": 2, + "canCancel": False, + "cancelRequested": False, + "annotations": { + "objectId": "123456", + "scopeId": "9617645", + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + }, + { + "key": "reviewer_notes", + "isPrivate": True, + "value": "Excellent work", + }, + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN all fields should be populated correctly + assert result.id == "123456" + assert result.etag == "abcd-1234" + assert result.modified_on == "2023-01-01T00:00:00.000Z" + assert result.status == "SCORED" + assert result.entity_id == "syn789" + assert result.version_number == 1 + assert result.status_version == 2 + assert result.can_cancel is False + assert result.cancel_requested is False + + # The annotations field should contain the raw submission status format + assert result.annotations is not None + assert "objectId" in result.annotations + assert "scopeId" in result.annotations + assert "stringAnnos" in result.annotations + assert "doubleAnnos" in result.annotations + assert len(result.annotations["stringAnnos"]) == 2 + assert len(result.annotations["doubleAnnos"]) == 1 + + # The submission_annotations should be in simple key-value format + assert "feedback" in result.submission_annotations + assert "score" in result.submission_annotations + assert result.submission_annotations["feedback"] == ["Great work!"] + assert result.submission_annotations["score"] == [92.5] + + async def test_fill_from_dict_with_minimal_response(self): + """Test filling a SubmissionStatus from a minimal API response async.""" + # GIVEN a minimal API response + api_response = {"id": "123456", "status": "RECEIVED"} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus() + result = submission_status.fill_from_dict(api_response) + + # THEN basic fields should be populated + assert result.id == "123456" + assert result.status == "RECEIVED" + # AND optional fields should have default values + assert result.etag is None + assert result.can_cancel is False + assert result.cancel_requested is False From 5e762f095ccf363e9d0229e01bf76eeb9db9deb2 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 11:08:04 -0500 Subject: [PATCH 29/46] new test class for submission cancel functionality --- .../async/test_submission_status_async.py | 104 ++++++++++++++++++ .../synchronous/test_submission_status.py | 103 +++++++++++++++++ 2 files changed, 207 insertions(+) diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index f46aea107..b558d7f08 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -644,6 +644,110 @@ async def test_batch_update_submission_statuses( assert converted_submission_annotations["batch_processed"] == ["true"] +class TestSubmissionStatusCancellation: + """Tests for SubmissionStatus cancellation functionality async.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = await evaluation.store_async(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ) + stored_file = await file.store_async(synapse_client=syn) + schedule_for_cleanup(stored_file.id) + return stored_file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = await submission.store_async(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_submission_cancellation_workflow( + self, test_submission: Submission + ): + """Test the complete submission cancellation workflow async.""" + # GIVEN a submission that exists + submission_id = test_submission.id + + # WHEN I get the initial submission status + initial_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + + # THEN initially it should not be cancellable or cancelled + assert initial_status.can_cancel is False + assert initial_status.cancel_requested is False + + # WHEN I update the submission status to allow cancellation + initial_status.can_cancel = True + updated_status = await initial_status.store_async(synapse_client=self.syn) + + # THEN the submission should be marked as cancellable + assert updated_status.can_cancel is True + assert updated_status.cancel_requested is False + + # WHEN I cancel the submission + await test_submission.cancel_async() + + # THEN I should be able to retrieve the updated status showing cancellation was requested + final_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + assert final_status.can_cancel is True + assert final_status.cancel_requested is True + + class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling async.""" diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index b2c4bd65c..b720a6ba3 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -641,6 +641,109 @@ async def test_batch_update_submission_statuses( assert converted_submission_annotations["batch_processed"] == ["true"] +class TestSubmissionStatusCancellation: + """Tests for SubmissionStatus cancellation functionality.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + """Create a test evaluation for submission status tests.""" + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="A test evaluation for submission status tests", + content_source=project_model.id, + submission_instructions_message="Please submit your results", + submission_receipt_message="Thank you!", + ) + created_evaluation = evaluation.store(synapse_client=syn) + schedule_for_cleanup(created_evaluation.id) + return created_evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + project_model: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + """Create a test file for submission status tests.""" + # Create a temporary file + with tempfile.NamedTemporaryFile( + mode="w", delete=False, suffix=".txt" + ) as temp_file: + temp_file.write("This is test content for submission status testing.") + temp_file_path = temp_file.name + + try: + file = File( + path=temp_file_path, + name=f"test_file_{uuid.uuid4()}.txt", + parent_id=project_model.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file.id) + return file + finally: + # Clean up the temporary file + os.unlink(temp_file_path) + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + """Create a test submission for status tests.""" + submission = Submission( + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + name=f"Test Submission {uuid.uuid4()}", + ) + created_submission = submission.store(synapse_client=syn) + schedule_for_cleanup(created_submission.id) + return created_submission + + async def test_submission_cancellation_workflow( + self, test_submission: Submission + ): + """Test the complete submission cancellation workflow.""" + # GIVEN a submission that exists + submission_id = test_submission.id + + # WHEN I get the initial submission status + initial_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) + + # THEN initially it should not be cancellable or cancelled + assert initial_status.can_cancel is False + assert initial_status.cancel_requested is False + + # WHEN I update the submission status to allow cancellation + initial_status.can_cancel = True + updated_status = initial_status.store(synapse_client=self.syn) + + # THEN the submission should be marked as cancellable + assert updated_status.can_cancel is True + assert updated_status.cancel_requested is False + + # WHEN I cancel the submission + test_submission.cancel() + + # THEN I should be able to retrieve the updated status showing cancellation was requested + final_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) + assert final_status.can_cancel is True + assert final_status.cancel_requested is True + + class TestSubmissionStatusValidation: """Tests for SubmissionStatus validation and error handling.""" From 1bd7f4b1435f423dc63af07638d35ee12870fdf1 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 11:19:37 -0500 Subject: [PATCH 30/46] substatus async unit tests --- .../unit_test_submission_status_async.py | 593 ++++++++++++++++++ 1 file changed, 593 insertions(+) create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_status_async.py diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py new file mode 100644 index 000000000..3ca7d906d --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py @@ -0,0 +1,593 @@ +"""Unit tests for the synapseclient.models.SubmissionStatus class.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import SubmissionStatus + +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" +SCORE = 85.5 +REPORT = "Test report" +VERSION_NUMBER = 1 +STATUS_VERSION = 1 +CAN_CANCEL = False +CANCEL_REQUESTED = False +PRIVATE_STATUS_ANNOTATIONS = True + + +class TestSubmissionStatus: + """Tests for the synapseclient.models.SubmissionStatus class.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "score": SCORE, + "report": REPORT, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "statusVersion": STATUS_VERSION, + "canCancel": CAN_CANCEL, + "cancelRequested": CANCEL_REQUESTED, + "annotations": { + "objectId": SUBMISSION_STATUS_ID, + "scopeId": EVALUATION_ID, + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + } + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + def get_example_submission_dict(self) -> Dict[str, str]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "evaluationId": EVALUATION_ID, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "userId": "123456", + "submitterAlias": "test_user", + "createdOn": "2023-01-01T00:00:00.000Z", + } + + def test_init_submission_status(self) -> None: + """Test creating a SubmissionStatus with basic attributes.""" + # WHEN I create a SubmissionStatus object + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + + # THEN the SubmissionStatus should have the expected attributes + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + assert submission_status.entity_id == ENTITY_ID + assert submission_status.evaluation_id == EVALUATION_ID + assert submission_status.can_cancel is False # default value + assert submission_status.cancel_requested is False # default value + assert submission_status.private_status_annotations is True # default value + + def test_fill_from_dict(self) -> None: + """Test filling a SubmissionStatus from a REST API response.""" + # GIVEN an example submission status response + submission_status_data = self.get_example_submission_status_dict() + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(submission_status_data) + + # THEN all fields should be populated correctly + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.etag == ETAG + assert submission_status.modified_on == MODIFIED_ON + assert submission_status.status == STATUS + assert submission_status.score == SCORE + assert submission_status.report == REPORT + assert submission_status.entity_id == ENTITY_ID + assert submission_status.version_number == VERSION_NUMBER + assert submission_status.status_version == STATUS_VERSION + assert submission_status.can_cancel is CAN_CANCEL + assert submission_status.cancel_requested is CANCEL_REQUESTED + + # Check annotations + assert submission_status.annotations is not None + assert "objectId" in submission_status.annotations + assert "scopeId" in submission_status.annotations + assert "stringAnnos" in submission_status.annotations + assert "doubleAnnos" in submission_status.annotations + + # Check submission annotations + assert "feedback" in submission_status.submission_annotations + assert "score" in submission_status.submission_annotations + assert submission_status.submission_annotations["feedback"] == ["Great work!"] + assert submission_status.submission_annotations["score"] == [92.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionStatus from minimal REST API response.""" + # GIVEN a minimal submission status response + minimal_data = {"id": SUBMISSION_STATUS_ID, "status": STATUS} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(minimal_data) + + # THEN basic fields should be populated + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + # AND optional fields should have default values + assert submission_status.etag is None + assert submission_status.can_cancel is False + assert submission_status.cancel_requested is False + + async def test_get_async(self) -> None: + """Test retrieving a SubmissionStatus by ID.""" + # GIVEN a SubmissionStatus with an ID + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call get_async + with patch( + "synapseclient.api.evaluation_services.get_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_get_status, patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_dict(), + ) as mock_get_submission: + result = await submission_status.get_async(synapse_client=self.syn) + + # THEN the submission status should be retrieved + mock_get_status.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + + # AND the result should have the expected data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS + assert result.evaluation_id == EVALUATION_ID + assert result._last_persistent_instance is not None + + async def test_get_async_without_id(self) -> None: + """Test that getting a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call get_async + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + await submission_status.get_async(synapse_client=self.syn) + + async def test_store_async(self) -> None: + """Test storing a SubmissionStatus.""" + # GIVEN a SubmissionStatus with required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + ) + submission_status._set_last_persistent_instance() + + # AND I modify the status + submission_status.status = "VALIDATED" + + # WHEN I call store_async + with patch( + "synapseclient.api.evaluation_services.update_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_update: + result = await submission_status.store_async(synapse_client=self.syn) + + # THEN the submission status should be updated + mock_update.assert_called_once() + call_args = mock_update.call_args + assert call_args.kwargs["submission_id"] == SUBMISSION_STATUS_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # AND the result should have updated data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS # from mock response + assert result._last_persistent_instance is not None + + async def test_store_async_without_id(self) -> None: + """Test that storing a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus(status="SCORED") + + # WHEN I call store_async + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + await submission_status.store_async(synapse_client=self.syn) + + async def test_store_async_without_changes(self) -> None: + """Test storing a SubmissionStatus without changes.""" + # GIVEN a SubmissionStatus that hasn't been modified + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status=STATUS, + ) + submission_status._set_last_persistent_instance() + + # WHEN I call store_async without making changes + result = await submission_status.store_async(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is submission_status + + def test_to_synapse_request_missing_id(self) -> None: + """Test to_synapse_request with missing ID.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_etag(self) -> None: + """Test to_synapse_request with missing etag.""" + # GIVEN a SubmissionStatus with ID but no etag + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_status_version(self) -> None: + """Test to_synapse_request with missing status_version.""" + # GIVEN a SubmissionStatus with ID and etag but no status_version + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, etag=ETAG) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: + """Test to_synapse_request with annotations but missing evaluation_id.""" + # GIVEN a SubmissionStatus with annotations but no evaluation_id + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + annotations={"test": "value"}, + ) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_valid(self) -> None: + """Test to_synapse_request with valid attributes.""" + # GIVEN a SubmissionStatus with all required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + submission_annotations={"score": 85.5}, + annotations={"internal_note": "test"}, + ) + + # WHEN I call to_synapse_request + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the request should have the required fields + assert request_body["id"] == SUBMISSION_STATUS_ID + assert request_body["etag"] == ETAG + assert request_body["statusVersion"] == STATUS_VERSION + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + assert "annotations" in request_body + + def test_has_changed_property_new_instance(self) -> None: + """Test has_changed property for a new instance.""" + # GIVEN a new SubmissionStatus instance + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # THEN has_changed should be True (no persistent instance) + assert submission_status.has_changed is True + + def test_has_changed_property_after_get(self) -> None: + """Test has_changed property after retrieving from Synapse.""" + # GIVEN a SubmissionStatus that was retrieved (has persistent instance) + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False + assert submission_status.has_changed is False + + # WHEN I modify a field + submission_status.status = "VALIDATED" + + # THEN has_changed should be True + assert submission_status.has_changed is True + + def test_has_changed_property_annotations(self) -> None: + """Test has_changed property with annotation changes.""" + # GIVEN a SubmissionStatus with annotations + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + annotations={"original": "value"}, + submission_annotations={"score": 85.0}, + ) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False initially + assert submission_status.has_changed is False + + # WHEN I modify annotations + submission_status.annotations = {"modified": "value"} + + # THEN has_changed should be True + assert submission_status.has_changed is True + + # WHEN I reset annotations to original and modify submission_annotations + submission_status.annotations = {"original": "value"} + submission_status.submission_annotations = {"score": 90.0} + + # THEN has_changed should still be True + assert submission_status.has_changed is True + + async def test_get_all_submission_statuses_async(self) -> None: + """Test getting all submission statuses for an evaluation.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + ] + } + + # WHEN I call get_all_submission_statuses_async + with patch( + "synapseclient.api.evaluation_services.get_all_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_all: + result = await SubmissionStatus.get_all_submission_statuses_async( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_all.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionStatus objects + assert len(result) == 2 + assert all(isinstance(status, SubmissionStatus) for status in result) + assert result[0].id == "123" + assert result[0].status == "RECEIVED" + assert result[1].id == "456" + assert result[1].status == "SCORED" + + async def test_batch_update_submission_statuses_async(self) -> None: + """Test batch updating submission statuses.""" + # GIVEN a list of SubmissionStatus objects + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ), + SubmissionStatus( + id="456", + etag="etag2", + status_version=1, + status="SCORED", + evaluation_id=EVALUATION_ID, + ), + ] + + # AND mock response + mock_response = {"batchToken": "token123"} + + # WHEN I call batch_update_submission_statuses_async + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_batch_update: + result = await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=True, + is_last_batch=True, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_batch_update.assert_called_once() + call_args = mock_batch_update.call_args + assert call_args.kwargs["evaluation_id"] == EVALUATION_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # Check request body structure + request_body = call_args.kwargs["request_body"] + assert request_body["isFirstBatch"] is True + assert request_body["isLastBatch"] is True + assert "statuses" in request_body + assert len(request_body["statuses"]) == 2 + + # AND the result should be the mock response + assert result == mock_response + + async def test_batch_update_with_batch_token(self) -> None: + """Test batch update with batch token for subsequent batches.""" + # GIVEN a list of SubmissionStatus objects and a batch token + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ) + ] + batch_token = "previous_batch_token" + + # WHEN I call batch_update_submission_statuses_async with a batch token + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value={}, + ) as mock_batch_update: + await SubmissionStatus.batch_update_submission_statuses_async( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the batch token should be included in the request + call_args = mock_batch_update.call_args + request_body = call_args.kwargs["request_body"] + assert request_body["batchToken"] == batch_token + assert request_body["isFirstBatch"] is False + + def test_set_last_persistent_instance(self) -> None: + """Test setting the last persistent instance.""" + # GIVEN a SubmissionStatus + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + annotations={"test": "value"}, + ) + + # WHEN I set the last persistent instance + submission_status._set_last_persistent_instance() + + # THEN the persistent instance should be set + assert submission_status._last_persistent_instance is not None + assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID + assert submission_status._last_persistent_instance.status == STATUS + assert submission_status._last_persistent_instance.annotations == {"test": "value"} + + # AND modifying the current instance shouldn't affect the persistent one + submission_status.status = "MODIFIED" + assert submission_status._last_persistent_instance.status == STATUS + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionStatus objects with the same data + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # THEN they should be equal + assert status1 == status2 + + # WHEN I modify one of them + status2.status = "DIFFERENT" + + # THEN they should not be equal + assert status1 != status2 + + def test_dataclass_fields_excluded_from_comparison(self) -> None: + """Test that certain fields are excluded from comparison.""" + # GIVEN two SubmissionStatus objects that differ only in comparison-excluded fields + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag1", + modified_on="2023-01-01", + cancel_requested=False, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag2", # different etag + modified_on="2023-01-02", # different modified_on + cancel_requested=True, # different cancel_requested + ) + + # THEN they should still be equal (these fields are excluded from comparison) + assert status1 == status2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionStatus.""" + # GIVEN a SubmissionStatus with some data + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I get the string representation + repr_str = repr(submission_status) + str_str = str(submission_status) + + # THEN it should contain the relevant information + assert SUBMISSION_STATUS_ID in repr_str + assert STATUS in repr_str + assert ENTITY_ID in repr_str + assert "SubmissionStatus" in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str From 4c8904dc6bf3bd5379f1a246d1ae3ec0538333dd Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 13:59:14 -0500 Subject: [PATCH 31/46] remove compare=false for some attributes. update sync unit tests --- synapseclient/models/submission_status.py | 6 +- .../unit_test_submission_status.py | 593 ++++++++++++++++++ 2 files changed, 596 insertions(+), 3 deletions(-) create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py diff --git a/synapseclient/models/submission_status.py b/synapseclient/models/submission_status.py index 60787be32..d8be2c913 100644 --- a/synapseclient/models/submission_status.py +++ b/synapseclient/models/submission_status.py @@ -359,7 +359,7 @@ class SubmissionStatus( List[datetime], ], ] - ] = field(default_factory=dict, compare=False) + ] = field(default_factory=dict) """Primary container object for Annotations on a Synapse object.""" submission_annotations: Optional[ @@ -374,10 +374,10 @@ class SubmissionStatus( List[datetime], ], ] - ] = field(default_factory=dict, compare=False) + ] = field(default_factory=dict) """Annotations are additional key-value pair metadata that are associated with an object.""" - private_status_annotations: Optional[bool] = field(default=True, compare=False) + private_status_annotations: Optional[bool] = field(default=True) """Indicates whether the submission status annotations (NOT to be confused with submission annotations) are private (True) or public (False). Default is True.""" entity_id: Optional[str] = None diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py new file mode 100644 index 000000000..76b00641f --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -0,0 +1,593 @@ +"""Unit tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" + +import uuid +from typing import Dict, List, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import SubmissionStatus + +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" +SCORE = 85.5 +REPORT = "Test report" +VERSION_NUMBER = 1 +STATUS_VERSION = 1 +CAN_CANCEL = False +CANCEL_REQUESTED = False +PRIVATE_STATUS_ANNOTATIONS = True + + +class TestSubmissionStatusSync: + """Tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "score": SCORE, + "report": REPORT, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "statusVersion": STATUS_VERSION, + "canCancel": CAN_CANCEL, + "cancelRequested": CANCEL_REQUESTED, + "annotations": { + "objectId": SUBMISSION_STATUS_ID, + "scopeId": EVALUATION_ID, + "stringAnnos": [ + { + "key": "internal_note", + "isPrivate": True, + "value": "This is internal", + } + ], + "doubleAnnos": [ + {"key": "validation_score", "isPrivate": True, "value": 95.0} + ], + "longAnnos": [], + }, + "submissionAnnotations": {"feedback": ["Great work!"], "score": [92.5]}, + } + + def get_example_submission_dict(self) -> Dict[str, str]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "evaluationId": EVALUATION_ID, + "entityId": ENTITY_ID, + "versionNumber": VERSION_NUMBER, + "userId": "123456", + "submitterAlias": "test_user", + "createdOn": "2023-01-01T00:00:00.000Z", + } + + def test_init_submission_status(self) -> None: + """Test creating a SubmissionStatus with basic attributes.""" + # WHEN I create a SubmissionStatus object + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + + # THEN the SubmissionStatus should have the expected attributes + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + assert submission_status.entity_id == ENTITY_ID + assert submission_status.evaluation_id == EVALUATION_ID + assert submission_status.can_cancel is False # default value + assert submission_status.cancel_requested is False # default value + assert submission_status.private_status_annotations is True # default value + + def test_fill_from_dict(self) -> None: + """Test filling a SubmissionStatus from a REST API response.""" + # GIVEN an example submission status response + submission_status_data = self.get_example_submission_status_dict() + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(submission_status_data) + + # THEN all fields should be populated correctly + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.etag == ETAG + assert submission_status.modified_on == MODIFIED_ON + assert submission_status.status == STATUS + assert submission_status.score == SCORE + assert submission_status.report == REPORT + assert submission_status.entity_id == ENTITY_ID + assert submission_status.version_number == VERSION_NUMBER + assert submission_status.status_version == STATUS_VERSION + assert submission_status.can_cancel is CAN_CANCEL + assert submission_status.cancel_requested is CANCEL_REQUESTED + + # Check annotations + assert submission_status.annotations is not None + assert "objectId" in submission_status.annotations + assert "scopeId" in submission_status.annotations + assert "stringAnnos" in submission_status.annotations + assert "doubleAnnos" in submission_status.annotations + + # Check submission annotations + assert "feedback" in submission_status.submission_annotations + assert "score" in submission_status.submission_annotations + assert submission_status.submission_annotations["feedback"] == ["Great work!"] + assert submission_status.submission_annotations["score"] == [92.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionStatus from minimal REST API response.""" + # GIVEN a minimal submission status response + minimal_data = {"id": SUBMISSION_STATUS_ID, "status": STATUS} + + # WHEN I fill a SubmissionStatus from the response + submission_status = SubmissionStatus().fill_from_dict(minimal_data) + + # THEN basic fields should be populated + assert submission_status.id == SUBMISSION_STATUS_ID + assert submission_status.status == STATUS + # AND optional fields should have default values + assert submission_status.etag is None + assert submission_status.can_cancel is False + assert submission_status.cancel_requested is False + + def test_get(self) -> None: + """Test retrieving a SubmissionStatus by ID using sync method.""" + # GIVEN a SubmissionStatus with an ID + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call get (sync method) + with patch( + "synapseclient.api.evaluation_services.get_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_get_status, patch( + "synapseclient.api.evaluation_services.get_submission", + new_callable=AsyncMock, + return_value=self.get_example_submission_dict(), + ) as mock_get_submission: + result = submission_status.get(synapse_client=self.syn) + + # THEN the submission status should be retrieved + mock_get_status.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + mock_get_submission.assert_called_once_with( + submission_id=SUBMISSION_STATUS_ID, synapse_client=self.syn + ) + + # AND the result should have the expected data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS + assert result.evaluation_id == EVALUATION_ID + + def test_get_without_id(self) -> None: + """Test that getting a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call get + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="The submission status must have an ID to get"): + submission_status.get(synapse_client=self.syn) + + def test_store(self) -> None: + """Test storing a SubmissionStatus using sync method.""" + # GIVEN a SubmissionStatus with required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + ) + submission_status._set_last_persistent_instance() + + # AND I modify the status + submission_status.status = "VALIDATED" + + # WHEN I call store (sync method) + with patch( + "synapseclient.api.evaluation_services.update_submission_status", + new_callable=AsyncMock, + return_value=self.get_example_submission_status_dict(), + ) as mock_update: + result = submission_status.store(synapse_client=self.syn) + + # THEN the submission status should be updated + mock_update.assert_called_once() + call_args = mock_update.call_args + assert call_args.kwargs["submission_id"] == SUBMISSION_STATUS_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # AND the result should have updated data + assert result.id == SUBMISSION_STATUS_ID + assert result.status == STATUS # from mock response + + def test_store_without_id(self) -> None: + """Test that storing a SubmissionStatus without ID raises ValueError.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus(status="SCORED") + + # WHEN I call store + # THEN it should raise a ValueError + with pytest.raises( + ValueError, match="The submission status must have an ID to update" + ): + submission_status.store(synapse_client=self.syn) + + def test_store_without_changes(self) -> None: + """Test storing a SubmissionStatus without changes.""" + # GIVEN a SubmissionStatus that hasn't been modified + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status=STATUS, + ) + submission_status._set_last_persistent_instance() + + # WHEN I call store without making changes + result = submission_status.store(synapse_client=self.syn) + + # THEN it should return the same instance (no update sent to Synapse) + assert result is submission_status + + def test_to_synapse_request_missing_id(self) -> None: + """Test to_synapse_request with missing ID.""" + # GIVEN a SubmissionStatus without an ID + submission_status = SubmissionStatus() + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_etag(self) -> None: + """Test to_synapse_request with missing etag.""" + # GIVEN a SubmissionStatus with ID but no etag + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'etag' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_status_version(self) -> None: + """Test to_synapse_request with missing status_version.""" + # GIVEN a SubmissionStatus with ID and etag but no status_version + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, etag=ETAG) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'status_version' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_missing_evaluation_id_with_annotations(self) -> None: + """Test to_synapse_request with annotations but missing evaluation_id.""" + # GIVEN a SubmissionStatus with annotations but no evaluation_id + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + annotations={"test": "value"}, + ) + + # WHEN I call to_synapse_request + # THEN it should raise a ValueError + with pytest.raises(ValueError, match="missing the 'evaluation_id' attribute"): + submission_status.to_synapse_request(synapse_client=self.syn) + + def test_to_synapse_request_valid(self) -> None: + """Test to_synapse_request with valid attributes.""" + # GIVEN a SubmissionStatus with all required attributes + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + etag=ETAG, + status_version=STATUS_VERSION, + status="SCORED", + evaluation_id=EVALUATION_ID, + submission_annotations={"score": 85.5}, + annotations={"internal_note": "test"}, + ) + + # WHEN I call to_synapse_request + request_body = submission_status.to_synapse_request(synapse_client=self.syn) + + # THEN the request should have the required fields + assert request_body["id"] == SUBMISSION_STATUS_ID + assert request_body["etag"] == ETAG + assert request_body["statusVersion"] == STATUS_VERSION + assert request_body["status"] == "SCORED" + assert "submissionAnnotations" in request_body + assert "annotations" in request_body + + def test_has_changed_property_new_instance(self) -> None: + """Test has_changed property for a new instance.""" + # GIVEN a new SubmissionStatus instance + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID) + + # THEN has_changed should be True (no persistent instance) + assert submission_status.has_changed is True + + def test_has_changed_property_after_get(self) -> None: + """Test has_changed property after retrieving from Synapse.""" + # GIVEN a SubmissionStatus that was retrieved (has persistent instance) + submission_status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False + assert submission_status.has_changed is False + + # WHEN I modify a field + submission_status.status = "VALIDATED" + + # THEN has_changed should be True + assert submission_status.has_changed is True + + def test_has_changed_property_annotations(self) -> None: + """Test has_changed property with annotation changes.""" + # GIVEN a SubmissionStatus with annotations + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + annotations={"original": "value"}, + submission_annotations={"score": 85.0}, + ) + submission_status._set_last_persistent_instance() + + # THEN has_changed should be False initially + assert submission_status.has_changed is False + + # WHEN I modify annotations + submission_status.annotations = {"modified": "value"} + + # THEN has_changed should be True + assert submission_status.has_changed is True + + # WHEN I reset annotations to original and modify submission_annotations + submission_status.annotations = {"original": "value"} + submission_status.submission_annotations = {"score": 90.0} + + # THEN has_changed should still be True + assert submission_status.has_changed is True + + def test_get_all_submission_statuses(self) -> None: + """Test getting all submission statuses using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + ] + } + + # WHEN I call get_all_submission_statuses (sync method) + with patch( + "synapseclient.api.evaluation_services.get_all_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_all: + result = SubmissionStatus.get_all_submission_statuses( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_all.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionStatus objects + assert len(result) == 2 + assert all(isinstance(status, SubmissionStatus) for status in result) + assert result[0].id == "123" + assert result[0].status == "RECEIVED" + assert result[1].id == "456" + assert result[1].status == "SCORED" + + def test_batch_update_submission_statuses(self) -> None: + """Test batch updating submission statuses using sync method.""" + # GIVEN a list of SubmissionStatus objects + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ), + SubmissionStatus( + id="456", + etag="etag2", + status_version=1, + status="SCORED", + evaluation_id=EVALUATION_ID, + ), + ] + + # AND mock response + mock_response = {"batchToken": "token123"} + + # WHEN I call batch_update_submission_statuses (sync method) + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_batch_update: + result = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=True, + is_last_batch=True, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_batch_update.assert_called_once() + call_args = mock_batch_update.call_args + assert call_args.kwargs["evaluation_id"] == EVALUATION_ID + assert call_args.kwargs["synapse_client"] == self.syn + + # Check request body structure + request_body = call_args.kwargs["request_body"] + assert request_body["isFirstBatch"] is True + assert request_body["isLastBatch"] is True + assert "statuses" in request_body + assert len(request_body["statuses"]) == 2 + + # AND the result should be the mock response + assert result == mock_response + + def test_batch_update_with_batch_token(self) -> None: + """Test batch update with batch token for subsequent batches.""" + # GIVEN a list of SubmissionStatus objects and a batch token + statuses = [ + SubmissionStatus( + id="123", + etag="etag1", + status_version=1, + status="VALIDATED", + evaluation_id=EVALUATION_ID, + ) + ] + batch_token = "previous_batch_token" + + # WHEN I call batch_update_submission_statuses with a batch token + with patch( + "synapseclient.api.evaluation_services.batch_update_submission_statuses", + new_callable=AsyncMock, + return_value={}, + ) as mock_batch_update: + SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses, + is_first_batch=False, + is_last_batch=True, + batch_token=batch_token, + synapse_client=self.syn, + ) + + # THEN the batch token should be included in the request + call_args = mock_batch_update.call_args + request_body = call_args.kwargs["request_body"] + assert request_body["batchToken"] == batch_token + assert request_body["isFirstBatch"] is False + + def test_set_last_persistent_instance(self) -> None: + """Test setting the last persistent instance.""" + # GIVEN a SubmissionStatus + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + annotations={"test": "value"}, + ) + + # WHEN I set the last persistent instance + submission_status._set_last_persistent_instance() + + # THEN the persistent instance should be set + assert submission_status._last_persistent_instance is not None + assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID + assert submission_status._last_persistent_instance.status == STATUS + assert submission_status._last_persistent_instance.annotations == {"test": "value"} + + # AND modifying the current instance shouldn't affect the persistent one + submission_status.status = "MODIFIED" + assert submission_status._last_persistent_instance.status == STATUS + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionStatus objects with the same data + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # THEN they should be equal + assert status1 == status2 + + # WHEN I modify one of them + status2.status = "DIFFERENT" + + # THEN they should not be equal + assert status1 != status2 + + def test_dataclass_fields_excluded_from_comparison(self) -> None: + """Test that certain fields are excluded from comparison.""" + # GIVEN two SubmissionStatus objects that differ only in comparison-excluded fields + status1 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag1", + modified_on="2023-01-01", + cancel_requested=False, + ) + status2 = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + etag="etag2", # different etag + modified_on="2023-01-02", # different modified_on + cancel_requested=True, # different cancel_requested + ) + + # THEN they should still be equal (these fields are excluded from comparison) + assert status1 == status2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionStatus.""" + # GIVEN a SubmissionStatus with some data + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I get the string representation + repr_str = repr(submission_status) + str_str = str(submission_status) + + # THEN it should contain the relevant information + assert SUBMISSION_STATUS_ID in repr_str + assert STATUS in repr_str + assert ENTITY_ID in repr_str + assert "SubmissionStatus" in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str From a2233201306b710f09b9cd55d9f3beda551f561a Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:29:24 -0500 Subject: [PATCH 32/46] add submissionBundle integration tests --- .../async/test_submission_bundle_async.py | 596 ++++++++++++++++++ .../synchronous/test_submission_bundle.py | 560 ++++++++++++++++ 2 files changed, 1156 insertions(+) create mode 100644 tests/integration/synapseclient/models/async/test_submission_bundle_async.py create mode 100644 tests/integration/synapseclient/models/synchronous/test_submission_bundle.py diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py new file mode 100644 index 000000000..b9162b726 --- /dev/null +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -0,0 +1,596 @@ +"""Integration tests for the synapseclient.models.SubmissionBundle class async methods.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus + + +class TestSubmissionBundleRetrievalAsync: + """Tests for retrieving SubmissionBundle objects using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for SubmissionBundle async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for submission bundle async tests {uuid.uuid4()}" + with open("test_file_for_submission_bundle_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_submission_bundle_async.txt", + name=f"test_submission_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_bundle_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + @pytest.fixture(scope="function") + async def multiple_submissions( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple submissions for testing pagination and filtering.""" + submissions = [] + for i in range(3): + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}_{i}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias=f"test_user_async_{i}", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + submissions.append(submission) + return submissions + + async def test_get_evaluation_submission_bundles_basic_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles for an evaluation using async methods.""" + # WHEN I get submission bundles for an evaluation + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in bundles" + + async def test_get_evaluation_submission_bundles_with_status_filter_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles filtered by status using async methods.""" + # WHEN I get submission bundles filtered by "RECEIVED" status + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + + # AND all bundles should have RECEIVED status (if any exist) + for bundle in bundles: + if bundle.submission_status: + assert bundle.submission_status.status == "RECEIVED" + + # WHEN I attempt to get submission bundles with an invalid status + with pytest.raises(SynapseHTTPError) as exc_info: + await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + # THEN it should raise a SynapseHTTPError (400 for invalid enum) + assert exc_info.value.response.status_code == 400 + assert "No enum constant" in str(exc_info.value) + assert "NONEXISTENT_STATUS" in str(exc_info.value) + + async def test_get_evaluation_submission_bundles_with_pagination_async( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting submission bundles using async methods.""" + # WHEN I get submission bundles with a limit + bundles_page1 = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): + """Test getting submission bundles for invalid evaluation ID using async methods.""" + # WHEN I try to get submission bundles for a non-existent evaluation + with pytest.raises(SynapseHTTPError) as exc_info: + await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + + # THEN it should raise a SynapseHTTPError (likely 403 or 404) + assert exc_info.value.response.status_code in [403, 404] + + async def test_get_user_submission_bundles_basic_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting user submission bundles for an evaluation using async methods.""" + # WHEN I get user submission bundles for an evaluation + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in user bundles" + + async def test_get_user_submission_bundles_with_pagination_async( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting user submission bundles using async methods.""" + # WHEN I get user submission bundles with a limit + bundles_page1 = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + +class TestSubmissionBundleDataIntegrityAsync: + """Tests for data integrity and relationships in SubmissionBundle objects using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for data integrity async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for data integrity async tests {uuid.uuid4()}" + with open("test_file_for_data_integrity_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_data_integrity_async.txt", + name=f"test_integrity_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_integrity_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_submission_bundle_data_consistency_async( + self, test_evaluation: Evaluation, test_submission: Submission, test_file: File + ): + """Test that submission bundles maintain data consistency between submission and status using async methods.""" + # WHEN I get submission bundles for the evaluation + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should find our test submission + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None, "Test submission bundle should be found" + + # AND the submission data should be consistent + assert test_bundle.submission.id == test_submission.id + assert test_bundle.submission.entity_id == test_file.id + assert test_bundle.submission.evaluation_id == test_evaluation.id + assert test_bundle.submission.name == test_submission.name + + # AND if there's a submission status, it should reference the same entities + if test_bundle.submission_status: + assert test_bundle.submission_status.id == test_submission.id + assert test_bundle.submission_status.entity_id == test_file.id + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + async def test_submission_bundle_status_updates_reflected_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that submission status updates are reflected in bundles using async methods.""" + # GIVEN a submission status that I can update + submission_status = await SubmissionStatus(id=test_submission.id).get_async( + synapse_client=self.syn + ) + original_status = submission_status.status + + # WHEN I update the submission status + submission_status.status = "VALIDATED" + submission_status.submission_annotations = { + "test_score": 95.5, + "test_feedback": "Excellent work!", + } + updated_status = await submission_status.store_async(synapse_client=self.syn) + + # AND I get submission bundles again + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundle should reflect the updated status + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + assert test_bundle.submission_status is not None + assert test_bundle.submission_status.status == "VALIDATED" + assert test_bundle.submission_status.submission_annotations is not None + assert "test_score" in test_bundle.submission_status.submission_annotations + assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + + # CLEANUP: Reset the status back to original + submission_status.status = original_status + submission_status.submission_annotations = {} + await submission_status.store_async(synapse_client=self.syn) + + async def test_submission_bundle_evaluation_id_propagation_async( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that evaluation_id is properly propagated from submission to status using async methods.""" + # WHEN I get submission bundles + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN find our test bundle + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + + # AND both submission and status should have the correct evaluation_id + assert test_bundle.submission.evaluation_id == test_evaluation.id + if test_bundle.submission_status: + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + +class TestSubmissionBundleEdgeCasesAsync: + """Tests for edge cases and error handling in SubmissionBundle operations using async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = await Project(name=f"test_project_{uuid.uuid4()}").store_async( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = await Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for edge case async testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store_async(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for edge case async tests {uuid.uuid4()}" + with open("test_file_for_edge_case_async.txt", "w") as f: + f.write(file_content) + + file_entity = await File( + path="test_file_for_edge_case_async.txt", + name=f"test_edge_case_file_async_{uuid.uuid4()}", + parent_id=test_project.id, + ).store_async(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = await Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_edge_case_async", + ).store_async(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_get_evaluation_submission_bundles_empty_evaluation_async( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles from an evaluation with no submissions using async methods.""" + # WHEN I get submission bundles from an evaluation with no submissions + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_user_submission_bundles_empty_evaluation_async( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles from an evaluation with no submissions using async methods.""" + # WHEN I get user submission bundles from an evaluation with no submissions + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_evaluation_submission_bundles_large_limit_async( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles with a very large limit using async methods.""" + # WHEN I request bundles with a large limit (within API bounds) + bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + limit=100, # Maximum allowed by API + synapse_client=self.syn, + ) + + # THEN it should work without error + assert bundles is not None + assert isinstance(bundles, list) + # The actual count doesn't matter since the evaluation is empty + + async def test_get_user_submission_bundles_large_offset_async( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles with a large offset using async methods.""" + # WHEN I request bundles with a large offset (beyond available data) + bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + offset=1000, # Large offset beyond any real data + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_submission_bundles_with_default_parameters_async( + self, test_evaluation: Evaluation + ): + """Test that default parameters work correctly using async methods.""" + # WHEN I call methods without optional parameters + eval_bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + user_bundles = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN both should work with default values + assert eval_bundles is not None + assert user_bundles is not None + assert isinstance(eval_bundles, list) + assert isinstance(user_bundles, list) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py new file mode 100644 index 000000000..6e32a3cad --- /dev/null +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -0,0 +1,560 @@ +"""Integration tests for the synapseclient.models.SubmissionBundle class.""" + +import uuid +from typing import Callable + +import pytest + +from synapseclient import Synapse +from synapseclient.core.exceptions import SynapseHTTPError +from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus + + +class TestSubmissionBundleRetrieval: + """Tests for retrieving SubmissionBundle objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for SubmissionBundle testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for submission bundle tests {uuid.uuid4()}" + with open("test_file_for_submission_bundle.txt", "w") as f: + f.write(file_content) + + file_entity = File( + path="test_file_for_submission_bundle.txt", + name=f"test_submission_file_{uuid.uuid4()}", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_bundle", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + @pytest.fixture(scope="function") + async def multiple_submissions( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> list[Submission]: + """Create multiple submissions for testing pagination and filtering.""" + submissions = [] + for i in range(3): + submission = Submission( + name=f"test_submission_{uuid.uuid4()}_{i}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias=f"test_user_{i}", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + submissions.append(submission) + return submissions + + async def test_get_evaluation_submission_bundles_basic( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles for an evaluation.""" + # WHEN I get submission bundles for an evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in bundles" + + async def test_get_evaluation_submission_bundles_with_status_filter( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting submission bundles filtered by status.""" + # WHEN I get submission bundles filtered by "RECEIVED" status + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + + # AND all bundles should have RECEIVED status (if any exist) + for bundle in bundles: + if bundle.submission_status: + assert bundle.submission_status.status == "RECEIVED" + + # WHEN I attempt to get submission bundles with an invalid status + with pytest.raises(SynapseHTTPError) as exc_info: + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + # THEN it should raise a SynapseHTTPError (400 for invalid enum) + assert exc_info.value.response.status_code == 400 + assert "No enum constant" in str(exc_info.value) + assert "NONEXISTENT_STATUS" in str(exc_info.value) + + async def test_get_evaluation_submission_bundles_with_pagination( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting submission bundles.""" + # WHEN I get submission bundles with a limit + bundles_page1 = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + async def test_get_evaluation_submission_bundles_invalid_evaluation(self): + """Test getting submission bundles for invalid evaluation ID.""" + # WHEN I try to get submission bundles for a non-existent evaluation + with pytest.raises(SynapseHTTPError) as exc_info: + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + + # THEN it should raise a SynapseHTTPError (likely 403 or 404) + assert exc_info.value.response.status_code in [403, 404] + + async def test_get_user_submission_bundles_basic( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test getting user submission bundles for an evaluation.""" + # WHEN I get user submission bundles for an evaluation + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundles should be retrieved + assert bundles is not None + assert len(bundles) >= 1 # At least our test submission + + # AND each bundle should have proper structure + found_test_bundle = False + for bundle in bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.id is not None + assert bundle.submission.evaluation_id == test_evaluation.id + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name + + # AND our test submission should be found + assert found_test_bundle, "Test submission should be found in user bundles" + + async def test_get_user_submission_bundles_with_pagination( + self, test_evaluation: Evaluation, multiple_submissions: list[Submission] + ): + """Test pagination when getting user submission bundles.""" + # WHEN I get user submission bundles with a limit + bundles_page1 = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=0, + synapse_client=self.syn, + ) + + # THEN I should get at most 2 bundles + assert bundles_page1 is not None + assert len(bundles_page1) <= 2 + + # WHEN I get the next page + bundles_page2 = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + limit=2, + offset=2, + synapse_client=self.syn, + ) + + # THEN I should get different bundles (if there are more than 2 total) + assert bundles_page2 is not None + + # AND the bundle IDs should not overlap if we have enough submissions + if len(bundles_page1) == 2 and len(bundles_page2) > 0: + page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} + page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} + assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + + +class TestSubmissionBundleDataIntegrity: + """Tests for data integrity and relationships in SubmissionBundle objects.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for data integrity testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + @pytest.fixture(scope="function") + async def test_file( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> File: + file_content = f"Test file content for data integrity tests {uuid.uuid4()}" + with open("test_file_for_data_integrity.txt", "w") as f: + f.write(file_content) + + file_entity = File( + path="test_file_for_data_integrity.txt", + name=f"test_integrity_file_{uuid.uuid4()}", + parent_id=test_project.id, + ).store(synapse_client=syn) + schedule_for_cleanup(file_entity.id) + return file_entity + + @pytest.fixture(scope="function") + async def test_submission( + self, + test_evaluation: Evaluation, + test_file: File, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Submission: + submission = Submission( + name=f"test_submission_{uuid.uuid4()}", + entity_id=test_file.id, + evaluation_id=test_evaluation.id, + submitter_alias="test_user_integrity", + ).store(synapse_client=syn) + schedule_for_cleanup(submission.id) + return submission + + async def test_submission_bundle_data_consistency( + self, test_evaluation: Evaluation, test_submission: Submission, test_file: File + ): + """Test that submission bundles maintain data consistency between submission and status.""" + # WHEN I get submission bundles for the evaluation + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should find our test submission + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None, "Test submission bundle should be found" + + # AND the submission data should be consistent + assert test_bundle.submission.id == test_submission.id + assert test_bundle.submission.entity_id == test_file.id + assert test_bundle.submission.evaluation_id == test_evaluation.id + assert test_bundle.submission.name == test_submission.name + + # AND if there's a submission status, it should reference the same entities + if test_bundle.submission_status: + assert test_bundle.submission_status.id == test_submission.id + assert test_bundle.submission_status.entity_id == test_file.id + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + async def test_submission_bundle_status_updates_reflected( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that submission status updates are reflected in bundles.""" + # GIVEN a submission status that I can update + submission_status = SubmissionStatus(id=test_submission.id).get( + synapse_client=self.syn + ) + original_status = submission_status.status + + # WHEN I update the submission status + submission_status.status = "VALIDATED" + submission_status.submission_annotations = { + "test_score": 95.5, + "test_feedback": "Excellent work!", + } + updated_status = submission_status.store(synapse_client=self.syn) + + # AND I get submission bundles again + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN the bundle should reflect the updated status + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + assert test_bundle.submission_status is not None + assert test_bundle.submission_status.status == "VALIDATED" + assert test_bundle.submission_status.submission_annotations is not None + assert "test_score" in test_bundle.submission_status.submission_annotations + assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + + # CLEANUP: Reset the status back to original + submission_status.status = original_status + submission_status.submission_annotations = {} + submission_status.store(synapse_client=self.syn) + + async def test_submission_bundle_evaluation_id_propagation( + self, test_evaluation: Evaluation, test_submission: Submission + ): + """Test that evaluation_id is properly propagated from submission to status.""" + # WHEN I get submission bundles + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN find our test bundle + test_bundle = None + for bundle in bundles: + if bundle.submission and bundle.submission.id == test_submission.id: + test_bundle = bundle + break + + assert test_bundle is not None + + # AND both submission and status should have the correct evaluation_id + assert test_bundle.submission.evaluation_id == test_evaluation.id + if test_bundle.submission_status: + assert test_bundle.submission_status.evaluation_id == test_evaluation.id + + +class TestSubmissionBundleEdgeCases: + """Tests for edge cases and error handling in SubmissionBundle operations.""" + + @pytest.fixture(autouse=True, scope="function") + def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: + self.syn = syn + self.schedule_for_cleanup = schedule_for_cleanup + + @pytest.fixture(scope="function") + async def test_project( + self, syn: Synapse, schedule_for_cleanup: Callable[..., None] + ) -> Project: + project = Project(name=f"test_project_{uuid.uuid4()}").store( + synapse_client=syn + ) + schedule_for_cleanup(project.id) + return project + + @pytest.fixture(scope="function") + async def test_evaluation( + self, + test_project: Project, + syn: Synapse, + schedule_for_cleanup: Callable[..., None], + ) -> Evaluation: + evaluation = Evaluation( + name=f"test_evaluation_{uuid.uuid4()}", + description="Test evaluation for edge case testing", + content_source=test_project.id, + submission_instructions_message="Submit your files here", + submission_receipt_message="Thank you for your submission!", + ).store(synapse_client=syn) + schedule_for_cleanup(evaluation.id) + return evaluation + + async def test_get_evaluation_submission_bundles_empty_evaluation( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles from an evaluation with no submissions.""" + # WHEN I get submission bundles from an evaluation with no submissions + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_user_submission_bundles_empty_evaluation( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles from an evaluation with no submissions.""" + # WHEN I get user submission bundles from an evaluation with no submissions + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not None or error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_evaluation_submission_bundles_large_limit( + self, test_evaluation: Evaluation + ): + """Test getting submission bundles with a very large limit.""" + # WHEN I request bundles with a large limit (within API bounds) + bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + limit=100, # Maximum allowed by API + synapse_client=self.syn, + ) + + # THEN it should work without error + assert bundles is not None + assert isinstance(bundles, list) + # The actual count doesn't matter since the evaluation is empty + + async def test_get_user_submission_bundles_large_offset( + self, test_evaluation: Evaluation + ): + """Test getting user submission bundles with a large offset.""" + # WHEN I request bundles with a large offset (beyond available data) + bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + offset=1000, # Large offset beyond any real data + synapse_client=self.syn, + ) + + # THEN it should return an empty list (not error) + assert bundles is not None + assert isinstance(bundles, list) + assert len(bundles) == 0 + + async def test_get_submission_bundles_with_default_parameters( + self, test_evaluation: Evaluation + ): + """Test that default parameters work correctly.""" + # WHEN I call methods without optional parameters + eval_bundles = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + user_bundles = SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN both should work with default values + assert eval_bundles is not None + assert user_bundles is not None + assert isinstance(eval_bundles, list) + assert isinstance(user_bundles, list) From f059751959814e1827174a1e59ff4d95bc4a6b90 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:29:53 -0500 Subject: [PATCH 33/46] add submissionBundle unit tests --- .../unit_test_submission_bundle_async.py | 452 +++++++++++++++++ .../unit_test_submission_bundle.py | 462 ++++++++++++++++++ 2 files changed, 914 insertions(+) create mode 100644 tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py create mode 100644 tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py new file mode 100644 index 000000000..1e2df93d7 --- /dev/null +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -0,0 +1,452 @@ +"""Unit tests for the synapseclient.models.SubmissionBundle class async methods.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus + +SUBMISSION_ID = "9999999" +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +USER_ID = "123456" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +CREATED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" + + +class TestSubmissionBundle: + """Tests for the synapseclient.models.SubmissionBundle class async methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": "test_user", + "entityId": ENTITY_ID, + "versionNumber": 1, + "name": "Test Submission", + "createdOn": CREATED_ON, + "evaluationId": EVALUATION_ID, + "entityBundle": { + "entity": { + "id": ENTITY_ID, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + }, + "entityType": "org.sagebionetworks.repo.model.FileEntity", + }, + } + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "entityId": ENTITY_ID, + "versionNumber": 1, + "statusVersion": 1, + "canCancel": False, + "cancelRequested": False, + "submissionAnnotations": {"score": [85.5], "feedback": ["Good work!"]}, + } + + def get_example_submission_bundle_dict(self) -> Dict[str, Dict]: + """Return example submission bundle data from REST API.""" + return { + "submission": self.get_example_submission_dict(), + "submissionStatus": self.get_example_submission_status_dict(), + } + + def get_example_submission_bundle_minimal_dict(self) -> Dict[str, Dict]: + """Return example minimal submission bundle data from REST API.""" + return { + "submission": { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + "submissionStatus": None, + } + + def test_init_submission_bundle(self) -> None: + """Test creating a SubmissionBundle with basic attributes.""" + # GIVEN submission and submission status objects + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I create a SubmissionBundle object + bundle = SubmissionBundle( + submission=submission, + submission_status=submission_status, + ) + + # THEN the SubmissionBundle should have the expected attributes + assert bundle.submission == submission + assert bundle.submission_status == submission_status + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + + def test_init_submission_bundle_empty(self) -> None: + """Test creating an empty SubmissionBundle.""" + # WHEN I create an empty SubmissionBundle object + bundle = SubmissionBundle() + + # THEN the SubmissionBundle should have None attributes + assert bundle.submission is None + assert bundle.submission_status is None + + def test_fill_from_dict_complete(self) -> None: + """Test filling a SubmissionBundle from complete REST API response.""" + # GIVEN a complete submission bundle response + bundle_data = self.get_example_submission_bundle_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN all fields should be populated correctly + assert bundle.submission is not None + assert bundle.submission_status is not None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission.user_id == USER_ID + + # Check submission status fields + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + assert bundle.submission_status.entity_id == ENTITY_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check submission annotations + assert "score" in bundle.submission_status.submission_annotations + assert bundle.submission_status.submission_annotations["score"] == [85.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionBundle from minimal REST API response.""" + # GIVEN a minimal submission bundle response + bundle_data = self.get_example_submission_bundle_minimal_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be populated but submission_status should be None + assert bundle.submission is not None + assert bundle.submission_status is None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + + def test_fill_from_dict_no_submission(self) -> None: + """Test filling a SubmissionBundle with no submission data.""" + # GIVEN a bundle response with no submission + bundle_data = { + "submission": None, + "submissionStatus": self.get_example_submission_status_dict(), + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be None but submission_status should be populated + assert bundle.submission is None + assert bundle.submission_status is not None + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + + def test_fill_from_dict_evaluation_id_setting(self) -> None: + """Test that evaluation_id is properly set from submission to submission_status.""" + # GIVEN a bundle response where submission_status doesn't have evaluation_id + submission_dict = self.get_example_submission_dict() + status_dict = self.get_example_submission_status_dict() + # Remove evaluation_id from status_dict to simulate API response + status_dict.pop("evaluationId", None) + + bundle_data = { + "submission": submission_dict, + "submissionStatus": status_dict, + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission_status should get evaluation_id from submission + assert bundle.submission is not None + assert bundle.submission_status is not None + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID + + async def test_get_evaluation_submission_bundles_async(self) -> None: + """Test getting submission bundles for an evaluation.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "123", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + }, + }, + { + "submission": { + "id": "456", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_evaluation_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 2 + assert all(isinstance(bundle, SubmissionBundle) for bundle in result) + + # Check first bundle + assert result[0].submission is not None + assert result[0].submission.id == "123" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "123" + assert result[0].submission_status.status == "RECEIVED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check second bundle + assert result[1].submission is not None + assert result[1].submission.id == "456" + assert result[1].submission_status is not None + assert result[1].submission_status.id == "456" + assert result[1].submission_status.status == "SCORED" + + async def test_get_evaluation_submission_bundles_async_empty_response(self) -> None: + """Test getting submission bundles with empty response.""" + # GIVEN empty mock response + mock_response = {"results": []} + + # WHEN I call get_evaluation_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = await SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status=None, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be an empty list + assert len(result) == 0 + + async def test_get_user_submission_bundles_async(self) -> None: + """Test getting user submission bundles.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "789", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + "name": "User Submission 1", + }, + "submissionStatus": { + "id": "789", + "status": "VALIDATED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_user_submission_bundles_async + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 1 + assert isinstance(result[0], SubmissionBundle) + + # Check bundle contents + assert result[0].submission is not None + assert result[0].submission.id == "789" + assert result[0].submission.name == "User Submission 1" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "789" + assert result[0].submission_status.status == "VALIDATED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID + + async def test_get_user_submission_bundles_async_default_params(self) -> None: + """Test getting user submission bundles with default parameters.""" + # GIVEN mock response + mock_response = {"results": []} + + # WHEN I call get_user_submission_bundles_async with defaults + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = await SubmissionBundle.get_user_submission_bundles_async( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called with default parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be empty + assert len(result) == 0 + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionBundle objects with the same data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + + bundle1 = SubmissionBundle(submission=submission, submission_status=status) + bundle2 = SubmissionBundle(submission=submission, submission_status=status) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I modify one of them + bundle2.submission_status = SubmissionStatus(id="different", status="DIFFERENT") + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_dataclass_equality_with_none(self) -> None: + """Test dataclass equality with None values.""" + # GIVEN two SubmissionBundle objects with None values + bundle1 = SubmissionBundle(submission=None, submission_status=None) + bundle2 = SubmissionBundle(submission=None, submission_status=None) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I add a submission to one + bundle2.submission = Submission(id=SUBMISSION_ID) + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionBundle.""" + # GIVEN a SubmissionBundle with some data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + bundle = SubmissionBundle(submission=submission, submission_status=status) + + # WHEN I get the string representation + repr_str = repr(bundle) + str_str = str(bundle) + + # THEN it should contain relevant information + assert "SubmissionBundle" in repr_str + assert SUBMISSION_ID in repr_str + assert SUBMISSION_STATUS_ID in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str + + def test_repr_with_none_values(self) -> None: + """Test string representation with None values.""" + # GIVEN a SubmissionBundle with None values + bundle = SubmissionBundle(submission=None, submission_status=None) + + # WHEN I get the string representation + repr_str = repr(bundle) + + # THEN it should show None values + assert "SubmissionBundle" in repr_str + assert "submission=None" in repr_str + assert "submission_status=None" in repr_str \ No newline at end of file diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py new file mode 100644 index 000000000..22287c528 --- /dev/null +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -0,0 +1,462 @@ +"""Unit tests for the synapseclient.models.SubmissionBundle class synchronous methods.""" + +from typing import Dict, Union +from unittest.mock import AsyncMock, patch + +import pytest + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus + +SUBMISSION_ID = "9999999" +SUBMISSION_STATUS_ID = "9999999" +ENTITY_ID = "syn123456" +EVALUATION_ID = "9614543" +USER_ID = "123456" +ETAG = "etag_value" +MODIFIED_ON = "2023-01-01T00:00:00.000Z" +CREATED_ON = "2023-01-01T00:00:00.000Z" +STATUS = "RECEIVED" + + +class TestSubmissionBundleSync: + """Tests for the synapseclient.models.SubmissionBundle class synchronous methods.""" + + @pytest.fixture(autouse=True, scope="function") + def init_syn(self, syn: Synapse) -> None: + self.syn = syn + + def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: + """Return example submission data from REST API.""" + return { + "id": SUBMISSION_ID, + "userId": USER_ID, + "submitterAlias": "test_user", + "entityId": ENTITY_ID, + "versionNumber": 1, + "name": "Test Submission", + "createdOn": CREATED_ON, + "evaluationId": EVALUATION_ID, + "entityBundle": { + "entity": { + "id": ENTITY_ID, + "name": "test_entity", + "concreteType": "org.sagebionetworks.repo.model.FileEntity", + }, + "entityType": "org.sagebionetworks.repo.model.FileEntity", + }, + } + + def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + """Return example submission status data from REST API.""" + return { + "id": SUBMISSION_STATUS_ID, + "etag": ETAG, + "modifiedOn": MODIFIED_ON, + "status": STATUS, + "entityId": ENTITY_ID, + "versionNumber": 1, + "statusVersion": 1, + "canCancel": False, + "cancelRequested": False, + "submissionAnnotations": {"score": [85.5], "feedback": ["Good work!"]}, + } + + def get_example_submission_bundle_dict(self) -> Dict[str, Dict]: + """Return example submission bundle data from REST API.""" + return { + "submission": self.get_example_submission_dict(), + "submissionStatus": self.get_example_submission_status_dict(), + } + + def get_example_submission_bundle_minimal_dict(self) -> Dict[str, Dict]: + """Return example minimal submission bundle data from REST API.""" + return { + "submission": { + "id": SUBMISSION_ID, + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + }, + "submissionStatus": None, + } + + def test_init_submission_bundle(self) -> None: + """Test creating a SubmissionBundle with basic attributes.""" + # GIVEN submission and submission status objects + submission = Submission( + id=SUBMISSION_ID, + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + ) + submission_status = SubmissionStatus( + id=SUBMISSION_STATUS_ID, + status=STATUS, + entity_id=ENTITY_ID, + ) + + # WHEN I create a SubmissionBundle object + bundle = SubmissionBundle( + submission=submission, + submission_status=submission_status, + ) + + # THEN the SubmissionBundle should have the expected attributes + assert bundle.submission == submission + assert bundle.submission_status == submission_status + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + + def test_init_submission_bundle_empty(self) -> None: + """Test creating an empty SubmissionBundle.""" + # WHEN I create an empty SubmissionBundle object + bundle = SubmissionBundle() + + # THEN the SubmissionBundle should have None attributes + assert bundle.submission is None + assert bundle.submission_status is None + + def test_fill_from_dict_complete(self) -> None: + """Test filling a SubmissionBundle from complete REST API response.""" + # GIVEN a complete submission bundle response + bundle_data = self.get_example_submission_bundle_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN all fields should be populated correctly + assert bundle.submission is not None + assert bundle.submission_status is not None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission.user_id == USER_ID + + # Check submission status fields + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + assert bundle.submission_status.entity_id == ENTITY_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check submission annotations + assert "score" in bundle.submission_status.submission_annotations + assert bundle.submission_status.submission_annotations["score"] == [85.5] + + def test_fill_from_dict_minimal(self) -> None: + """Test filling a SubmissionBundle from minimal REST API response.""" + # GIVEN a minimal submission bundle response + bundle_data = self.get_example_submission_bundle_minimal_dict() + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be populated but submission_status should be None + assert bundle.submission is not None + assert bundle.submission_status is None + + # Check submission fields + assert bundle.submission.id == SUBMISSION_ID + assert bundle.submission.entity_id == ENTITY_ID + assert bundle.submission.evaluation_id == EVALUATION_ID + + def test_fill_from_dict_no_submission(self) -> None: + """Test filling a SubmissionBundle with no submission data.""" + # GIVEN a bundle response with no submission + bundle_data = { + "submission": None, + "submissionStatus": self.get_example_submission_status_dict(), + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission should be None but submission_status should be populated + assert bundle.submission is None + assert bundle.submission_status is not None + assert bundle.submission_status.id == SUBMISSION_STATUS_ID + assert bundle.submission_status.status == STATUS + + def test_fill_from_dict_evaluation_id_setting(self) -> None: + """Test that evaluation_id is properly set from submission to submission_status.""" + # GIVEN a bundle response where submission_status doesn't have evaluation_id + submission_dict = self.get_example_submission_dict() + status_dict = self.get_example_submission_status_dict() + # Remove evaluation_id from status_dict to simulate API response + status_dict.pop("evaluationId", None) + + bundle_data = { + "submission": submission_dict, + "submissionStatus": status_dict, + } + + # WHEN I fill a SubmissionBundle from the response + bundle = SubmissionBundle().fill_from_dict(bundle_data) + + # THEN submission_status should get evaluation_id from submission + assert bundle.submission is not None + assert bundle.submission_status is not None + assert bundle.submission.evaluation_id == EVALUATION_ID + assert bundle.submission_status.evaluation_id == EVALUATION_ID + + def test_get_evaluation_submission_bundles(self) -> None: + """Test getting submission bundles for an evaluation using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "123", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "123", + "status": "RECEIVED", + "entityId": ENTITY_ID, + }, + }, + { + "submission": { + "id": "456", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + }, + "submissionStatus": { + "id": "456", + "status": "SCORED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_evaluation_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + limit=50, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 2 + assert all(isinstance(bundle, SubmissionBundle) for bundle in result) + + # Check first bundle + assert result[0].submission is not None + assert result[0].submission.id == "123" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "123" + assert result[0].submission_status.status == "RECEIVED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission + + # Check second bundle + assert result[1].submission is not None + assert result[1].submission.id == "456" + assert result[1].submission_status is not None + assert result[1].submission_status.id == "456" + assert result[1].submission_status.status == "SCORED" + + def test_get_evaluation_submission_bundles_empty_response(self) -> None: + """Test getting submission bundles with empty response using sync method.""" + # GIVEN empty mock response + mock_response = {"results": []} + + # WHEN I call get_evaluation_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_bundles: + result = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called + mock_get_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + status=None, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be an empty list + assert len(result) == 0 + + def test_get_user_submission_bundles(self) -> None: + """Test getting user submission bundles using sync method.""" + # GIVEN mock response data + mock_response = { + "results": [ + { + "submission": { + "id": "789", + "entityId": ENTITY_ID, + "evaluationId": EVALUATION_ID, + "userId": USER_ID, + "name": "User Submission 1", + }, + "submissionStatus": { + "id": "789", + "status": "VALIDATED", + "entityId": ENTITY_ID, + }, + }, + ] + } + + # WHEN I call get_user_submission_bundles (sync method) + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # THEN the service should be called with correct parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=25, + offset=5, + synapse_client=self.syn, + ) + + # AND the result should contain SubmissionBundle objects + assert len(result) == 1 + assert isinstance(result[0], SubmissionBundle) + + # Check bundle contents + assert result[0].submission is not None + assert result[0].submission.id == "789" + assert result[0].submission.name == "User Submission 1" + assert result[0].submission_status is not None + assert result[0].submission_status.id == "789" + assert result[0].submission_status.status == "VALIDATED" + assert result[0].submission_status.evaluation_id == EVALUATION_ID + + def test_get_user_submission_bundles_default_params(self) -> None: + """Test getting user submission bundles with default parameters using sync method.""" + # GIVEN mock response + mock_response = {"results": []} + + # WHEN I call get_user_submission_bundles with defaults (sync method) + with patch( + "synapseclient.api.evaluation_services.get_user_submission_bundles", + new_callable=AsyncMock, + return_value=mock_response, + ) as mock_get_user_bundles: + result = SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + + # THEN the service should be called with default parameters + mock_get_user_bundles.assert_called_once_with( + evaluation_id=EVALUATION_ID, + limit=10, + offset=0, + synapse_client=self.syn, + ) + + # AND the result should be empty + assert len(result) == 0 + + def test_dataclass_equality(self) -> None: + """Test dataclass equality comparison.""" + # GIVEN two SubmissionBundle objects with the same data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + + bundle1 = SubmissionBundle(submission=submission, submission_status=status) + bundle2 = SubmissionBundle(submission=submission, submission_status=status) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I modify one of them + bundle2.submission_status = SubmissionStatus(id="different", status="DIFFERENT") + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_dataclass_equality_with_none(self) -> None: + """Test dataclass equality with None values.""" + # GIVEN two SubmissionBundle objects with None values + bundle1 = SubmissionBundle(submission=None, submission_status=None) + bundle2 = SubmissionBundle(submission=None, submission_status=None) + + # THEN they should be equal + assert bundle1 == bundle2 + + # WHEN I add a submission to one + bundle2.submission = Submission(id=SUBMISSION_ID) + + # THEN they should not be equal + assert bundle1 != bundle2 + + def test_repr_and_str(self) -> None: + """Test string representation of SubmissionBundle.""" + # GIVEN a SubmissionBundle with some data + submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) + status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) + bundle = SubmissionBundle(submission=submission, submission_status=status) + + # WHEN I get the string representation + repr_str = repr(bundle) + str_str = str(bundle) + + # THEN it should contain relevant information + assert "SubmissionBundle" in repr_str + assert SUBMISSION_ID in repr_str + assert SUBMISSION_STATUS_ID in repr_str + + # AND str should be the same as repr for dataclasses + assert str_str == repr_str + + def test_repr_with_none_values(self) -> None: + """Test string representation with None values.""" + # GIVEN a SubmissionBundle with None values + bundle = SubmissionBundle(submission=None, submission_status=None) + + # WHEN I get the string representation + repr_str = repr(bundle) + + # THEN it should show None values + assert "SubmissionBundle" in repr_str + assert "submission=None" in repr_str + assert "submission_status=None" in repr_str + + def test_protocol_implementation(self) -> None: + """Test that SubmissionBundle implements the synchronous protocol correctly.""" + # THEN it should have all the required synchronous methods + assert hasattr(SubmissionBundle, "get_evaluation_submission_bundles") + assert hasattr(SubmissionBundle, "get_user_submission_bundles") + + # AND the methods should be callable + assert callable(SubmissionBundle.get_evaluation_submission_bundles) + assert callable(SubmissionBundle.get_user_submission_bundles) \ No newline at end of file From 3cb1915b8ee8dbb85f443f210518ea7c6c78db65 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Thu, 20 Nov 2025 14:30:40 -0500 Subject: [PATCH 34/46] remove unnecessary imports and add style --- .../async/test_submission_bundle_async.py | 45 +++++++++++--- .../async/test_submission_status_async.py | 62 ++++++++++++------- .../synchronous/test_submission_bundle.py | 49 ++++++++++----- .../synchronous/test_submission_status.py | 16 +++-- .../unit_test_submission_bundle_async.py | 28 +++++---- .../unit_test_submission_status_async.py | 12 +++- .../unit_test_submission_bundle.py | 28 +++++---- .../unit_test_submission_status.py | 16 +++-- 8 files changed, 168 insertions(+), 88 deletions(-) diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index b9162b726..eb30af938 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -7,7 +7,14 @@ from synapseclient import Synapse from synapseclient.core.exceptions import SynapseHTTPError -from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus +from synapseclient.models import ( + Evaluation, + File, + Project, + Submission, + SubmissionBundle, + SubmissionStatus, +) class TestSubmissionBundleRetrievalAsync: @@ -52,7 +59,9 @@ async def test_file( syn: Synapse, schedule_for_cleanup: Callable[..., None], ) -> File: - file_content = f"Test file content for submission bundle async tests {uuid.uuid4()}" + file_content = ( + f"Test file content for submission bundle async tests {uuid.uuid4()}" + ) with open("test_file_for_submission_bundle_async.txt", "w") as f: f.write(file_content) @@ -192,9 +201,15 @@ async def test_get_evaluation_submission_bundles_with_pagination_async( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): """Test getting submission bundles for invalid evaluation ID using async methods.""" @@ -267,9 +282,15 @@ async def test_get_user_submission_bundles_with_pagination_async( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" class TestSubmissionBundleDataIntegrityAsync: @@ -314,7 +335,9 @@ async def test_file( syn: Synapse, schedule_for_cleanup: Callable[..., None], ) -> File: - file_content = f"Test file content for data integrity async tests {uuid.uuid4()}" + file_content = ( + f"Test file content for data integrity async tests {uuid.uuid4()}" + ) with open("test_file_for_data_integrity_async.txt", "w") as f: f.write(file_content) @@ -410,7 +433,9 @@ async def test_submission_bundle_status_updates_reflected_async( assert test_bundle.submission_status.status == "VALIDATED" assert test_bundle.submission_status.submission_annotations is not None assert "test_score" in test_bundle.submission_status.submission_annotations - assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + assert test_bundle.submission_status.submission_annotations["test_score"] == [ + 95.5 + ] # CLEANUP: Reset the status back to original submission_status.status = original_status diff --git a/tests/integration/synapseclient/models/async/test_submission_status_async.py b/tests/integration/synapseclient/models/async/test_submission_status_async.py index b558d7f08..bf72a8158 100644 --- a/tests/integration/synapseclient/models/async/test_submission_status_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_status_async.py @@ -221,7 +221,9 @@ async def test_store_submission_status_with_status_change( # WHEN I update the status test_submission_status.status = "VALIDATED" - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN the submission status should be updated assert updated_status.id == test_submission_status.id @@ -239,7 +241,9 @@ async def test_store_submission_status_with_submission_annotations( "score": 85.5, "feedback": "Good work!", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN the submission annotations should be saved assert updated_status.submission_annotations is not None @@ -256,7 +260,9 @@ async def test_store_submission_status_with_legacy_annotations( "internal_score": 92.3, "reviewer_notes": "Excellent submission", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) assert updated_status.annotations is not None converted_annotations = from_submission_status_annotations( @@ -281,7 +287,9 @@ async def test_store_submission_status_with_combined_annotations( "internal_review": True, "notes": "Needs minor improvements", } - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN both types of annotations should be saved assert updated_status.submission_annotations is not None @@ -307,7 +315,9 @@ async def test_store_submission_status_with_private_annotations_false( test_submission_status.private_status_annotations = False # WHEN I store the submission status - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN they should be properly stored assert updated_status.annotations is not None @@ -340,7 +350,9 @@ async def test_store_submission_status_with_private_annotations_true( ) # WHEN I store the submission status - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN they should be properly stored assert updated_status.annotations is not None @@ -395,7 +407,9 @@ async def test_store_submission_status_change_tracking( assert test_submission_status.has_changed # WHEN I store the changes - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) # THEN has_changed should be False again assert not updated_status.has_changed @@ -431,7 +445,9 @@ async def test_has_changed_property_edge_cases( assert test_submission_status.has_changed # WHEN I store and get a fresh copy - updated_status = await test_submission_status.store_async(synapse_client=self.syn) + updated_status = await test_submission_status.store_async( + synapse_client=self.syn + ) fresh_status = await SubmissionStatus(id=updated_status.id).get_async( synapse_client=self.syn ) @@ -510,7 +526,7 @@ async def test_files( finally: # Clean up the temporary file os.unlink(temp_file_path) - + return files @pytest.fixture(scope="function") @@ -611,7 +627,9 @@ async def test_batch_update_submission_statuses( # GIVEN multiple submission statuses statuses = [] for submission in test_submissions: - status = await SubmissionStatus(id=submission.id).get_async(synapse_client=self.syn) + status = await SubmissionStatus(id=submission.id).get_async( + synapse_client=self.syn + ) # Update each status status.status = "VALIDATED" status.submission_annotations = { @@ -717,33 +735,35 @@ async def test_submission( schedule_for_cleanup(created_submission.id) return created_submission - async def test_submission_cancellation_workflow( - self, test_submission: Submission - ): + async def test_submission_cancellation_workflow(self, test_submission: Submission): """Test the complete submission cancellation workflow async.""" # GIVEN a submission that exists submission_id = test_submission.id - + # WHEN I get the initial submission status - initial_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) - + initial_status = await SubmissionStatus(id=submission_id).get_async( + synapse_client=self.syn + ) + # THEN initially it should not be cancellable or cancelled assert initial_status.can_cancel is False assert initial_status.cancel_requested is False - + # WHEN I update the submission status to allow cancellation initial_status.can_cancel = True updated_status = await initial_status.store_async(synapse_client=self.syn) - + # THEN the submission should be marked as cancellable assert updated_status.can_cancel is True assert updated_status.cancel_requested is False - + # WHEN I cancel the submission await test_submission.cancel_async() - + # THEN I should be able to retrieve the updated status showing cancellation was requested - final_status = await SubmissionStatus(id=submission_id).get_async(synapse_client=self.syn) + final_status = await SubmissionStatus(id=submission_id).get_async( + synapse_client=self.syn + ) assert final_status.can_cancel is True assert final_status.cancel_requested is True diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index 6e32a3cad..a9816765b 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -7,7 +7,14 @@ from synapseclient import Synapse from synapseclient.core.exceptions import SynapseHTTPError -from synapseclient.models import Evaluation, File, Project, Submission, SubmissionBundle, SubmissionStatus +from synapseclient.models import ( + Evaluation, + File, + Project, + Submission, + SubmissionBundle, + SubmissionStatus, +) class TestSubmissionBundleRetrieval: @@ -22,9 +29,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -192,9 +197,15 @@ async def test_get_evaluation_submission_bundles_with_pagination( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" @@ -267,9 +278,15 @@ async def test_get_user_submission_bundles_with_pagination( # AND the bundle IDs should not overlap if we have enough submissions if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = {bundle.submission.id for bundle in bundles_page1 if bundle.submission} - page2_ids = {bundle.submission.id for bundle in bundles_page2 if bundle.submission} - assert page1_ids.isdisjoint(page2_ids), "Pages should not have overlapping submissions" + page1_ids = { + bundle.submission.id for bundle in bundles_page1 if bundle.submission + } + page2_ids = { + bundle.submission.id for bundle in bundles_page2 if bundle.submission + } + assert page1_ids.isdisjoint( + page2_ids + ), "Pages should not have overlapping submissions" class TestSubmissionBundleDataIntegrity: @@ -284,9 +301,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project @@ -410,7 +425,9 @@ async def test_submission_bundle_status_updates_reflected( assert test_bundle.submission_status.status == "VALIDATED" assert test_bundle.submission_status.submission_annotations is not None assert "test_score" in test_bundle.submission_status.submission_annotations - assert test_bundle.submission_status.submission_annotations["test_score"] == [95.5] + assert test_bundle.submission_status.submission_annotations["test_score"] == [ + 95.5 + ] # CLEANUP: Reset the status back to original submission_status.status = original_status @@ -454,9 +471,7 @@ def init(self, syn: Synapse, schedule_for_cleanup: Callable[..., None]) -> None: async def test_project( self, syn: Synapse, schedule_for_cleanup: Callable[..., None] ) -> Project: - project = Project(name=f"test_project_{uuid.uuid4()}").store( - synapse_client=syn - ) + project = Project(name=f"test_project_{uuid.uuid4()}").store(synapse_client=syn) schedule_for_cleanup(project.id) return project diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_status.py b/tests/integration/synapseclient/models/synchronous/test_submission_status.py index b720a6ba3..b4ae45372 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_status.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_status.py @@ -713,31 +713,29 @@ async def test_submission( schedule_for_cleanup(created_submission.id) return created_submission - async def test_submission_cancellation_workflow( - self, test_submission: Submission - ): + async def test_submission_cancellation_workflow(self, test_submission: Submission): """Test the complete submission cancellation workflow.""" # GIVEN a submission that exists submission_id = test_submission.id - + # WHEN I get the initial submission status initial_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) - + # THEN initially it should not be cancellable or cancelled assert initial_status.can_cancel is False assert initial_status.cancel_requested is False - + # WHEN I update the submission status to allow cancellation initial_status.can_cancel = True updated_status = initial_status.store(synapse_client=self.syn) - + # THEN the submission should be marked as cancellable assert updated_status.can_cancel is True assert updated_status.cancel_requested is False - + # WHEN I cancel the submission test_submission.cancel() - + # THEN I should be able to retrieve the updated status showing cancellation was requested final_status = SubmissionStatus(id=submission_id).get(synapse_client=self.syn) assert final_status.can_cancel is True diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py index 1e2df93d7..1e2efca02 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -47,7 +47,9 @@ def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: }, } - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -126,7 +128,7 @@ def test_fill_from_dict_complete(self) -> None: # THEN all fields should be populated correctly assert bundle.submission is not None assert bundle.submission_status is not None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -137,7 +139,9 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + assert ( + bundle.submission_status.evaluation_id == EVALUATION_ID + ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -154,7 +158,7 @@ def test_fill_from_dict_minimal(self) -> None: # THEN submission should be populated but submission_status should be None assert bundle.submission is not None assert bundle.submission_status is None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -184,7 +188,7 @@ def test_fill_from_dict_evaluation_id_setting(self) -> None: status_dict = self.get_example_submission_status_dict() # Remove evaluation_id from status_dict to simulate API response status_dict.pop("evaluationId", None) - + bundle_data = { "submission": submission_dict, "submissionStatus": status_dict, @@ -259,15 +263,17 @@ async def test_get_evaluation_submission_bundles_async(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 2 assert all(isinstance(bundle, SubmissionBundle) for bundle in result) - + # Check first bundle assert result[0].submission is not None assert result[0].submission.id == "123" assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission - + assert ( + result[0].submission_status.evaluation_id == EVALUATION_ID + ) # set from submission + # Check second bundle assert result[1].submission is not None assert result[1].submission.id == "456" @@ -349,7 +355,7 @@ async def test_get_user_submission_bundles_async(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 1 assert isinstance(result[0], SubmissionBundle) - + # Check bundle contents assert result[0].submission is not None assert result[0].submission.id == "789" @@ -391,7 +397,7 @@ def test_dataclass_equality(self) -> None: # GIVEN two SubmissionBundle objects with the same data submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) - + bundle1 = SubmissionBundle(submission=submission, submission_status=status) bundle2 = SubmissionBundle(submission=submission, submission_status=status) @@ -449,4 +455,4 @@ def test_repr_with_none_values(self) -> None: # THEN it should show None values assert "SubmissionBundle" in repr_str assert "submission=None" in repr_str - assert "submission_status=None" in repr_str \ No newline at end of file + assert "submission_status=None" in repr_str diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py index 3ca7d906d..c48ad2e1e 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_status_async.py @@ -30,7 +30,9 @@ class TestSubmissionStatus: def init_syn(self, syn: Synapse) -> None: self.syn = syn - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -181,7 +183,9 @@ async def test_get_async_without_id(self) -> None: # WHEN I call get_async # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): await submission_status.get_async(synapse_client=self.syn) async def test_store_async(self) -> None: @@ -520,7 +524,9 @@ def test_set_last_persistent_instance(self) -> None: assert submission_status._last_persistent_instance is not None assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID assert submission_status._last_persistent_instance.status == STATUS - assert submission_status._last_persistent_instance.annotations == {"test": "value"} + assert submission_status._last_persistent_instance.annotations == { + "test": "value" + } # AND modifying the current instance shouldn't affect the persistent one submission_status.status = "MODIFIED" diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index 22287c528..9dc252286 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -47,7 +47,9 @@ def get_example_submission_dict(self) -> Dict[str, Union[str, int, Dict]]: }, } - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -126,7 +128,7 @@ def test_fill_from_dict_complete(self) -> None: # THEN all fields should be populated correctly assert bundle.submission is not None assert bundle.submission_status is not None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -137,7 +139,9 @@ def test_fill_from_dict_complete(self) -> None: assert bundle.submission_status.id == SUBMISSION_STATUS_ID assert bundle.submission_status.status == STATUS assert bundle.submission_status.entity_id == ENTITY_ID - assert bundle.submission_status.evaluation_id == EVALUATION_ID # set from submission + assert ( + bundle.submission_status.evaluation_id == EVALUATION_ID + ) # set from submission # Check submission annotations assert "score" in bundle.submission_status.submission_annotations @@ -154,7 +158,7 @@ def test_fill_from_dict_minimal(self) -> None: # THEN submission should be populated but submission_status should be None assert bundle.submission is not None assert bundle.submission_status is None - + # Check submission fields assert bundle.submission.id == SUBMISSION_ID assert bundle.submission.entity_id == ENTITY_ID @@ -184,7 +188,7 @@ def test_fill_from_dict_evaluation_id_setting(self) -> None: status_dict = self.get_example_submission_status_dict() # Remove evaluation_id from status_dict to simulate API response status_dict.pop("evaluationId", None) - + bundle_data = { "submission": submission_dict, "submissionStatus": status_dict, @@ -259,15 +263,17 @@ def test_get_evaluation_submission_bundles(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 2 assert all(isinstance(bundle, SubmissionBundle) for bundle in result) - + # Check first bundle assert result[0].submission is not None assert result[0].submission.id == "123" assert result[0].submission_status is not None assert result[0].submission_status.id == "123" assert result[0].submission_status.status == "RECEIVED" - assert result[0].submission_status.evaluation_id == EVALUATION_ID # set from submission - + assert ( + result[0].submission_status.evaluation_id == EVALUATION_ID + ) # set from submission + # Check second bundle assert result[1].submission is not None assert result[1].submission.id == "456" @@ -349,7 +355,7 @@ def test_get_user_submission_bundles(self) -> None: # AND the result should contain SubmissionBundle objects assert len(result) == 1 assert isinstance(result[0], SubmissionBundle) - + # Check bundle contents assert result[0].submission is not None assert result[0].submission.id == "789" @@ -391,7 +397,7 @@ def test_dataclass_equality(self) -> None: # GIVEN two SubmissionBundle objects with the same data submission = Submission(id=SUBMISSION_ID, entity_id=ENTITY_ID) status = SubmissionStatus(id=SUBMISSION_STATUS_ID, status=STATUS) - + bundle1 = SubmissionBundle(submission=submission, submission_status=status) bundle2 = SubmissionBundle(submission=submission, submission_status=status) @@ -459,4 +465,4 @@ def test_protocol_implementation(self) -> None: # AND the methods should be callable assert callable(SubmissionBundle.get_evaluation_submission_bundles) - assert callable(SubmissionBundle.get_user_submission_bundles) \ No newline at end of file + assert callable(SubmissionBundle.get_user_submission_bundles) diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py index 76b00641f..e724e64f0 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_status.py @@ -1,13 +1,11 @@ """Unit tests for the synapseclient.models.SubmissionStatus class synchronous methods.""" -import uuid -from typing import Dict, List, Union +from typing import Dict, Union from unittest.mock import AsyncMock, patch import pytest from synapseclient import Synapse -from synapseclient.core.exceptions import SynapseHTTPError from synapseclient.models import SubmissionStatus SUBMISSION_STATUS_ID = "9999999" @@ -32,7 +30,9 @@ class TestSubmissionStatusSync: def init_syn(self, syn: Synapse) -> None: self.syn = syn - def get_example_submission_status_dict(self) -> Dict[str, Union[str, int, bool, Dict]]: + def get_example_submission_status_dict( + self, + ) -> Dict[str, Union[str, int, bool, Dict]]: """Return example submission status data from REST API.""" return { "id": SUBMISSION_STATUS_ID, @@ -182,7 +182,9 @@ def test_get_without_id(self) -> None: # WHEN I call get # THEN it should raise a ValueError - with pytest.raises(ValueError, match="The submission status must have an ID to get"): + with pytest.raises( + ValueError, match="The submission status must have an ID to get" + ): submission_status.get(synapse_client=self.syn) def test_store(self) -> None: @@ -520,7 +522,9 @@ def test_set_last_persistent_instance(self) -> None: assert submission_status._last_persistent_instance is not None assert submission_status._last_persistent_instance.id == SUBMISSION_STATUS_ID assert submission_status._last_persistent_instance.status == STATUS - assert submission_status._last_persistent_instance.annotations == {"test": "value"} + assert submission_status._last_persistent_instance.annotations == { + "test": "value" + } # AND modifying the current instance shouldn't affect the persistent one submission_status.status = "MODIFIED" From 1fb0fbf8c83849c6eb09987f2168449b576d85a5 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 11:15:02 -0500 Subject: [PATCH 35/46] get_evaluation_submissions returns generator object --- synapseclient/api/evaluation_services.py | 28 +- synapseclient/models/submission.py | 255 ++++++++++++++++-- .../models/async/test_submission_async.py | 58 ++-- .../models/synchronous/test_submission.py | 50 ++-- .../async/unit_test_submission_async.py | 40 ++- .../synchronous/unit_test_submission.py | 42 ++- 6 files changed, 333 insertions(+), 140 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 478976137..19184e562 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -4,7 +4,9 @@ """ import json -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional + +from synapseclient.api.api_client import rest_get_paginated_async if TYPE_CHECKING: from synapseclient import Synapse @@ -448,12 +450,11 @@ async def get_submission( async def get_evaluation_submissions( evaluation_id: str, status: Optional[str] = None, - limit: int = 20, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Retrieves all Submissions for a specified Evaluation queue. + Generator to get all Submissions for a specified Evaluation queue. @@ -461,29 +462,26 @@ async def get_evaluation_submissions( evaluation_id: The ID of the evaluation queue. status: Optionally filter submissions by a submission status, such as SCORED, VALID, INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - # TODO: Support pagination in the return type. - A response JSON containing a paginated list of submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/all" - query_params = {"limit": limit, "offset": offset} + query_params = {} if status: query_params["status"] = status - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_user_submissions( diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 7096201c9..2e0860cb6 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -1,17 +1,55 @@ from dataclasses import dataclass, field -from typing import Dict, List, Optional, Protocol, Union +from typing import AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union from typing_extensions import Self from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, otel_trace_method +from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, otel_trace_method, wrap_async_generator_to_sync_generator from synapseclient.models.mixins.access_control import AccessControllable class SubmissionSynchronousProtocol(Protocol): """Protocol defining the synchronous interface for Submission operations.""" + def store( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Store the submission in Synapse. This creates a new submission in an evaluation queue. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The Submission object with the ID set. + + Raises: + ValueError: If the submission is missing required fields, or if unable to fetch entity etag. + + Example: Creating a submission +   + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission( + entity_id="syn123456", + evaluation_id="9614543", + name="My Submission" + ).store() + print(submission.id) + ``` + """ + return self + def get( self, *, @@ -21,9 +59,6 @@ def get( Retrieve a Submission from Synapse. Arguments: - include_activity: Whether to include the activity in the returned submission. - Defaults to False. Setting this to True will include the activity - record associated with this submission. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. @@ -31,9 +66,11 @@ def get( Returns: The Submission instance retrieved from Synapse. - Example: Retrieving a submission by ID. -   + Raises: + ValueError: If the submission does not have an ID to get. + Example: Retrieving a submission by ID +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -56,9 +93,11 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Example: Delete a submission. -   + Raises: + ValueError: If the submission does not have an ID to delete. + Example: Delete a submission +   ```python from synapseclient import Synapse from synapseclient.models import Submission @@ -73,6 +112,171 @@ def delete(self, *, synapse_client: Optional[Synapse] = None) -> None: """ pass + def cancel( + self, + *, + synapse_client: Optional[Synapse] = None, + ) -> "Self": + """ + Cancel a Submission. Only the user who created the Submission may cancel it. + + Arguments: + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + The updated Submission object. + + Raises: + ValueError: If the submission does not have an ID to cancel. + + Example: Cancel a submission +   + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submission = Submission(id="syn1234") + canceled_submission = submission.cancel() + ``` + """ + return self + + @classmethod + def get_evaluation_submissions( + cls, + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Generator["Submission", None, None]: + """ + Retrieves all Submissions for a specified Evaluation queue. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status. + Submission status can be one of + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + Submission objects as they are retrieved from the API. + + Example: Getting submissions for an evaluation +   + Get SCORED submissions from a specific evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + submissions = list(Submission.get_evaluation_submissions( + evaluation_id="9999999", + status="SCORED" + )) + print(f"Found {len(submissions)} submissions") + ``` + """ + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_evaluation_submissions_async, + evaluation_id=evaluation_id, + status=status, + synapse_client=synapse_client, + ) + + @staticmethod + def get_user_submissions( + evaluation_id: str, + user_id: Optional[str] = None, + limit: int = 20, + offset: int = 0, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Retrieves Submissions for a specified Evaluation queue and user. + If user_id is omitted, this returns the submissions of the caller. + + Arguments: + evaluation_id: The ID of the evaluation queue. + user_id: Optionally specify the ID of the user whose submissions will be returned. + If omitted, this returns the submissions of the caller. + limit: Limits the number of submissions in a single response. Default to 20. + offset: The offset index determines where this page will start from. + An index of 0 is the first submission. Default to 0. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing a paginated list of user submissions for the evaluation queue. + + Example: Getting user submissions + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = Submission.get_user_submissions( + evaluation_id="9999999", + user_id="123456", + limit=10 + ) + print(f"Found {len(response['results'])} user submissions") + ``` + """ + return {} + + @staticmethod + def get_submission_count( + evaluation_id: str, + status: Optional[str] = None, + *, + synapse_client: Optional[Synapse] = None, + ) -> Dict: + """ + Gets the number of Submissions for a specified Evaluation queue, optionally filtered by submission status. + + Arguments: + evaluation_id: The ID of the evaluation queue. + status: Optionally filter submissions by a submission status, such as SCORED, VALID, + INVALID, OPEN, CLOSED or EVALUATION_IN_PROGRESS. + synapse_client: If not passed in and caching was not disabled by + `Synapse.allow_client_caching(False)` this will use the last created + instance from the Synapse class constructor. + + Returns: + A response JSON containing the submission count. + + Example: Getting submission count +   + Get the total number of SCORED submissions from a specific evaluation. + ```python + from synapseclient import Synapse + from synapseclient.models import Submission + + syn = Synapse() + syn.login() + + response = Submission.get_submission_count( + evaluation_id="9999999", + status="SCORED" + ) + print(f"Found {response} submissions") + ``` + """ + return {} + @dataclass @async_to_sync @@ -448,31 +652,28 @@ async def get_submission_example(): return self # TODO: Have all staticmethods return generators for pagination - @staticmethod + @skip_async_to_sync + @classmethod async def get_evaluation_submissions_async( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> AsyncGenerator["Submission", None]: """ - Retrieves all Submissions for a specified Evaluation queue. + Generator to get all Submissions for a specified Evaluation queue. Arguments: evaluation_id: The ID of the evaluation queue. status: Optionally filter submissions by a submission status. Submission status can be one of - limit: Limits the number of submissions in a single response. Defaults to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Defaults to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. Example: Getting submissions for an evaluation   @@ -486,22 +687,24 @@ async def get_evaluation_submissions_async( syn.login() async def get_evaluation_submissions_example(): - response = await Submission.get_evaluation_submissions_async( + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id="9999999", status="SCORED" - ) - print(f"Found {len(response['results'])} submissions") + ): + submissions.append(submission) + print(f"Found {len(submissions)} submissions") asyncio.run(get_evaluation_submissions_example()) ``` """ - return await evaluation_services.get_evaluation_submissions( + async for submission_data in evaluation_services.get_evaluation_submissions( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) + ): + submission_object = cls().fill_from_dict(synapse_submission=submission_data) + yield submission_object @staticmethod async def get_user_submissions_async( diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 745a60960..622e4dce3 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -245,52 +245,54 @@ async def test_get_submission_by_id_async( async def test_get_evaluation_submissions_async( self, test_evaluation: Evaluation, test_submission: Submission ): - # WHEN I get all submissions for an evaluation using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get all submissions for an evaluation using async generator + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + ): + submissions.append(submission) - # THEN I should get a response with submissions - assert "results" in response - assert len(response["results"]) > 0 + # THEN I should get a list of submission objects + assert len(submissions) > 0 + assert all(isinstance(sub, Submission) for sub in submissions) - # AND the submission should be in the results - submission_ids = [sub.get("id") for sub in response["results"]] + # AND the test submission should be in the results + submission_ids = [sub.id for sub in submissions] assert test_submission.id in submission_ids async def test_get_evaluation_submissions_with_status_filter_async( self, test_evaluation: Evaluation, test_submission: Submission ): - # WHEN I get submissions filtered by status using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get submissions filtered by status using async generator + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN I should get submissions with the specified status - assert "results" in response - for submission in response["results"]: - if submission.get("id") == test_submission.id: - # The submission should be in RECEIVED status initially - break - else: - pytest.fail("Test submission not found in filtered results") - - async def test_get_evaluation_submissions_with_pagination_async( + # The test submission should be in the results (initially in RECEIVED status) + submission_ids = [sub.id for sub in submissions] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_async_generator_behavior( self, test_evaluation: Evaluation ): - # WHEN I get submissions with pagination parameters using async method - response = await Submission.get_evaluation_submissions_async( + # WHEN I get submissions using the async generator + submissions_generator = Submission.get_evaluation_submissions_async( evaluation_id=test_evaluation.id, - limit=5, - offset=0, synapse_client=self.syn, ) - # THEN the response should respect pagination - assert "results" in response - assert len(response["results"]) <= 5 + # THEN I should be able to iterate through the results + submissions = [] + async for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions_async(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user using async method diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index e6943a006..a16109b07 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -239,51 +239,49 @@ async def test_get_evaluation_submissions( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get all submissions for an evaluation - response = Submission.get_evaluation_submissions( + submissions = list(Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + )) - # THEN I should get a response with submissions - assert "results" in response - assert len(response["results"]) > 0 + # THEN I should get a list of submission objects + assert len(submissions) > 0 + assert all(isinstance(sub, Submission) for sub in submissions) - # AND the submission should be in the results - submission_ids = [sub.get("id") for sub in response["results"]] + # AND the test submission should be in the results + submission_ids = [sub.id for sub in submissions] assert test_submission.id in submission_ids async def test_get_evaluation_submissions_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get submissions filtered by status - response = Submission.get_evaluation_submissions( + submissions = list(Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + )) - # THEN I should get submissions with the specified status - assert "results" in response - for submission in response["results"]: - if submission.get("id") == test_submission.id: - # The submission should be in RECEIVED status initially - break - else: - pytest.fail("Test submission not found in filtered results") - - async def test_get_evaluation_submissions_with_pagination( + # The test submission should be in the results (initially in RECEIVED status) + submission_ids = [sub.id for sub in submissions] + assert test_submission.id in submission_ids + + async def test_get_evaluation_submissions_generator_behavior( self, test_evaluation: Evaluation ): - # WHEN I get submissions with pagination parameters - response = Submission.get_evaluation_submissions( + # WHEN I get submissions using the generator + submissions_generator = Submission.get_evaluation_submissions( evaluation_id=test_evaluation.id, - limit=5, - offset=0, synapse_client=self.syn, ) - # THEN the response should respect pagination - assert "results" in response - assert len(response["results"]) <= 5 + # THEN I should be able to iterate through the results + submissions = [] + for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index 8a0ca25ac..0d33095f2 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -471,48 +471,44 @@ async def test_cancel_async_success(self) -> None: synapse_client=self.syn, ) self.syn.logger.info.assert_called_once_with( - f"Submission {SUBMISSION_ID} has successfully been cancelled." + f"A request to cancel Submission {SUBMISSION_ID} has been submitted." ) - assert cancelled_submission.id == SUBMISSION_ID - assert cancelled_submission.entity_id == ENTITY_ID - assert cancelled_submission.evaluation_id == EVALUATION_ID @pytest.mark.asyncio async def test_get_evaluation_submissions_async(self) -> None: # GIVEN evaluation parameters evaluation_id = EVALUATION_ID status = "SCORED" - limit = 10 - offset = 5 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_evaluation_submissions_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_evaluation_submissions" ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_submissions.assert_called_once_with( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_user_submissions_async(self) -> None: diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 038045f11..31f00eb0f 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -614,7 +614,7 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - cancelled_submission = await submission.cancel_async( + await submission.cancel_async( synapse_client=self.syn ) @@ -624,11 +624,8 @@ async def test_cancel_async_success(self) -> None: synapse_client=self.syn, ) self.syn.logger.info.assert_called_once_with( - f"Submission {SUBMISSION_ID} has successfully been cancelled." + f"A request to cancel Submission {SUBMISSION_ID} has been submitted." ) - assert cancelled_submission.id == SUBMISSION_ID - assert cancelled_submission.entity_id == ENTITY_ID - assert cancelled_submission.evaluation_id == EVALUATION_ID @pytest.mark.asyncio async def test_cancel_async_without_id(self) -> None: @@ -645,37 +642,36 @@ async def test_get_evaluation_submissions_async(self) -> None: # GIVEN evaluation parameters evaluation_id = EVALUATION_ID status = "SCORED" - limit = 10 - offset = 5 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_evaluation_submissions_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_evaluation_submissions" ) as mock_get_submissions: - response = await Submission.get_evaluation_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_evaluation_submissions_async( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_submissions.assert_called_once_with( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_user_submissions_async(self) -> None: From 281ed3dfa09d6596c177539b3806d0e3f0a839f7 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 12:01:55 -0500 Subject: [PATCH 36/46] get_user_submissions returns generator object --- synapseclient/api/evaluation_services.py | 23 +++---- synapseclient/models/submission.py | 66 +++++++++---------- .../models/async/test_submission_async.py | 30 +++++++-- .../models/synchronous/test_submission.py | 25 ++++++- .../async/unit_test_submission_async.py | 35 +++++----- .../synchronous/unit_test_submission.py | 35 +++++----- 6 files changed, 123 insertions(+), 91 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index 19184e562..cb5784808 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -487,12 +487,11 @@ async def get_evaluation_submissions( async def get_user_submissions( evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Generator to get all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. @@ -501,28 +500,26 @@ async def get_user_submissions( evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission" - query_params = {"limit": limit, "offset": offset} + query_params = {} if user_id: query_params["userId"] = user_id - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_submission_count( diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 2e0860cb6..5ab8f3c64 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -192,32 +192,28 @@ def get_evaluation_submissions( synapse_client=synapse_client, ) - @staticmethod + @classmethod def get_user_submissions( + cls, evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> Generator["Submission", None, None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Retrieves all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. Arguments: evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Submission objects as they are retrieved from the API. Example: Getting user submissions ```python @@ -227,15 +223,19 @@ def get_user_submissions( syn = Synapse() syn.login() - response = Submission.get_user_submissions( + submissions = list(Submission.get_user_submissions( evaluation_id="9999999", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + user_id="123456" + )) + print(f"Found {len(submissions)} user submissions") ``` """ - return {} + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_user_submissions_async, + evaluation_id=evaluation_id, + user_id=user_id, + synapse_client=synapse_client, + ) @staticmethod def get_submission_count( @@ -706,32 +706,29 @@ async def get_evaluation_submissions_example(): submission_object = cls().fill_from_dict(synapse_submission=submission_data) yield submission_object - @staticmethod + @skip_async_to_sync + @classmethod async def get_user_submissions_async( + cls, evaluation_id: str, user_id: Optional[str] = None, - limit: int = 20, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> Dict: + ) -> AsyncGenerator["Submission", None]: """ - Retrieves Submissions for a specified Evaluation queue and user. + Generator to get all user Submissions for a specified Evaluation queue. If user_id is omitted, this returns the submissions of the caller. Arguments: evaluation_id: The ID of the evaluation queue. user_id: Optionally specify the ID of the user whose submissions will be returned. If omitted, this returns the submissions of the caller. - limit: Limits the number of submissions in a single response. Default to 20. - offset: The offset index determines where this page will start from. - An index of 0 is the first submission. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A response JSON containing a paginated list of user submissions for the evaluation queue. + Yields: + Individual Submission objects from each page of the response. Example: Getting user submissions ```python @@ -743,23 +740,24 @@ async def get_user_submissions_async( syn.login() async def get_user_submissions_example(): - response = await Submission.get_user_submissions_async( + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id="9999999", - user_id="123456", - limit=10 - ) - print(f"Found {len(response['results'])} user submissions") + user_id="123456" + ): + submissions.append(submission) + print(f"Found {len(submissions)} user submissions") asyncio.run(get_user_submissions_example()) ``` """ - return await evaluation_services.get_user_submissions( + async for submission_data in evaluation_services.get_user_submissions( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) + ): + submission_object = cls().fill_from_dict(synapse_submission=submission_data) + yield submission_object @staticmethod async def get_submission_count_async( diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 622e4dce3..5ef52ed4e 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -295,14 +295,34 @@ async def test_get_evaluation_submissions_async_generator_behavior( assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_user_submissions_async(self, test_evaluation: Evaluation): - # WHEN I get submissions for the current user using async method - response = await Submission.get_user_submissions_async( + # WHEN I get submissions for the current user using async generator + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=test_evaluation.id, synapse_client=self.syn - ) + ): + submissions.append(submission) - # THEN I should get a response with user submissions - assert "results" in response + # THEN all submissions should be valid Submission objects # Note: Could be empty if user hasn't made submissions to this evaluation + assert all(isinstance(sub, Submission) for sub in submissions) + + async def test_get_user_submissions_async_generator_behavior( + self, test_evaluation: Evaluation + ): + # WHEN I get user submissions using the async generator + submissions_generator = Submission.get_user_submissions_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + submissions = [] + async for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_submission_count_async(self, test_evaluation: Evaluation): # WHEN I get the submission count for an evaluation using async method diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index a16109b07..16fa52b1a 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -285,13 +285,32 @@ async def test_get_evaluation_submissions_generator_behavior( async def test_get_user_submissions(self, test_evaluation: Evaluation): # WHEN I get submissions for the current user - response = Submission.get_user_submissions( + submissions_generator = Submission.get_user_submissions( evaluation_id=test_evaluation.id, synapse_client=self.syn ) - # THEN I should get a response with user submissions - assert "results" in response + # THEN I should get a generator that yields Submission objects + submissions = list(submissions_generator) # Note: Could be empty if user hasn't made submissions to this evaluation + assert all(isinstance(sub, Submission) for sub in submissions) + + async def test_get_user_submissions_generator_behavior( + self, test_evaluation: Evaluation + ): + # WHEN I get user submissions using the generator + submissions_generator = Submission.get_user_submissions( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + submissions = [] + for submission in submissions_generator: + assert isinstance(submission, Submission) + submissions.append(submission) + + # AND all submissions should be valid Submission objects + assert all(isinstance(sub, Submission) for sub in submissions) async def test_get_submission_count(self, test_evaluation: Evaluation): # WHEN I get the submission count for an evaluation diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index 0d33095f2..a18f22fe8 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -515,37 +515,36 @@ async def test_get_user_submissions_async(self) -> None: # GIVEN user submission parameters evaluation_id = EVALUATION_ID user_id = USER_ID - limit = 15 - offset = 0 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_user_submissions_async with patch( - "synapseclient.api.evaluation_services.get_user_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_user_submissions" ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_user_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_user_submissions.assert_called_once_with( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_submission_count_async(self) -> None: diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index 31f00eb0f..c1d2eb2c0 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -678,37 +678,36 @@ async def test_get_user_submissions_async(self) -> None: # GIVEN user submission parameters evaluation_id = EVALUATION_ID user_id = USER_ID - limit = 15 - offset = 0 - - expected_response = { - "results": [self.get_example_submission_response()], - "totalNumberOfResults": 1, - } # WHEN I call get_user_submissions_async with patch( - "synapseclient.api.evaluation_services.get_user_submissions", - new_callable=AsyncMock, - return_value=expected_response, + "synapseclient.api.evaluation_services.get_user_submissions" ) as mock_get_user_submissions: - response = await Submission.get_user_submissions_async( + # Create an async generator function that yields submission data + async def mock_async_gen(*args, **kwargs): + submission_data = self.get_example_submission_response() + yield submission_data + + # Make the mock return our async generator when called + mock_get_user_submissions.side_effect = mock_async_gen + + submissions = [] + async for submission in Submission.get_user_submissions_async( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, - ) + ): + submissions.append(submission) - # THEN it should call the API with correct parameters + # THEN it should call the API with correct parameters and yield Submission objects mock_get_user_submissions.assert_called_once_with( evaluation_id=evaluation_id, user_id=user_id, - limit=limit, - offset=offset, synapse_client=self.syn, ) - assert response == expected_response + assert len(submissions) == 1 + assert isinstance(submissions[0], Submission) + assert submissions[0].id == SUBMISSION_ID @pytest.mark.asyncio async def test_get_submission_count_async(self) -> None: From 1168cae902dfbc725ef191dab82a8a4d5609aa32 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 13:08:35 -0500 Subject: [PATCH 37/46] submissionBundle methods return generators --- synapseclient/api/evaluation_services.py | 50 ++-- synapseclient/models/submission.py | 1 - synapseclient/models/submission_bundle.py | 176 ++++++------ .../async/test_submission_bundle_async.py | 250 ++++++++++-------- .../synchronous/test_submission_bundle.py | 198 +++++++------- .../unit_test_submission_bundle_async.py | 84 +++--- .../unit_test_submission_bundle.py | 76 +++--- 7 files changed, 434 insertions(+), 401 deletions(-) diff --git a/synapseclient/api/evaluation_services.py b/synapseclient/api/evaluation_services.py index cb5784808..80fee69a8 100644 --- a/synapseclient/api/evaluation_services.py +++ b/synapseclient/api/evaluation_services.py @@ -783,29 +783,23 @@ async def batch_update_submission_statuses( async def get_evaluation_submission_bundles( evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Generator to get all bundled Submissions and SubmissionStatuses to a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -815,48 +809,44 @@ async def get_evaluation_submission_bundles( client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/bundle/all" - query_params = {"limit": limit, "offset": offset} + query_params = {} if status: query_params["status"] = status - response = await client.rest_get_async(uri, params=query_params) - - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item async def get_user_submission_bundles( evaluation_id: str, - limit: int = 10, - offset: int = 0, + *, synapse_client: Optional["Synapse"] = None, -) -> dict: +) -> AsyncGenerator[Dict[str, Any], None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Generator to get all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A PaginatedResults object as a JSON dict containing - a paginated list of the requesting user's submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. """ from synapseclient import Synapse client = Synapse.get_client(synapse_client=synapse_client) uri = f"/evaluation/{evaluation_id}/submission/bundle" - query_params = {"limit": limit, "offset": offset} - - response = await client.rest_get_async(uri, params=query_params) + query_params = {} - return response + async for item in rest_get_paginated_async( + uri=uri, params=query_params, synapse_client=client + ): + yield item diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 5ab8f3c64..08dc642ee 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -651,7 +651,6 @@ async def get_submission_example(): return self - # TODO: Have all staticmethods return generators for pagination @skip_async_to_sync @classmethod async def get_evaluation_submissions_async( diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 6c7ae42e4..75dde8610 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,9 +1,9 @@ from dataclasses import dataclass -from typing import TYPE_CHECKING, Dict, List, Optional, Protocol, Union +from typing import TYPE_CHECKING, AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync +from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, wrap_async_generator_to_sync_generator if TYPE_CHECKING: from synapseclient.models.submission import Submission @@ -13,32 +13,26 @@ class SubmissionBundleSynchronousProtocol(Protocol): """Protocol defining the synchronous interface for SubmissionBundle operations.""" - @staticmethod + @classmethod def get_evaluation_submission_bundles( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> Generator["SubmissionBundle", None, None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Retrieves bundled Submissions and SubmissionStatuses for a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the submission bundles - for the evaluation queue. + SubmissionBundle objects as they are retrieved from the API. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -52,42 +46,40 @@ def get_evaluation_submission_bundles( syn = Synapse() syn.login() - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id="9614543", - status="SCORED", - limit=50 - ) + status="SCORED" + )) print(f"Found {len(bundles)} submission bundles") for bundle in bundles: print(f"Submission ID: {bundle.submission.id if bundle.submission else 'N/A'}") ``` """ - return [] + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_evaluation_submission_bundles_async, + evaluation_id=evaluation_id, + status=status, + synapse_client=synapse_client, + ) - @staticmethod + @classmethod def get_user_submission_bundles( + cls, evaluation_id: str, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> Generator["SubmissionBundle", None, None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Retrieves all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. Returns: - A list of SubmissionBundle objects containing the requesting user's - submission bundles for the evaluation queue. + SubmissionBundle objects as they are retrieved from the API. Example: Getting user submission bundles   @@ -98,16 +90,19 @@ def get_user_submission_bundles( syn = Synapse() syn.login() - bundles = SubmissionBundle.get_user_submission_bundles( - evaluation_id="9999999", - limit=25 - ) + bundles = list(SubmissionBundle.get_user_submission_bundles( + evaluation_id="9999999" + )) print(f"Found {len(bundles)} user submission bundles") for bundle in bundles: print(f"Submission ID: {bundle.submission.id}") ``` """ - return [] + yield from wrap_async_generator_to_sync_generator( + async_gen_func=cls.get_user_submission_bundles_async, + evaluation_id=evaluation_id, + synapse_client=synapse_client, + ) @dataclass @@ -217,32 +212,27 @@ def fill_from_dict( return self - @staticmethod + @skip_async_to_sync + @classmethod async def get_evaluation_submission_bundles_async( + cls, evaluation_id: str, status: Optional[str] = None, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> AsyncGenerator["SubmissionBundle", None]: """ - Gets a collection of bundled Submissions and SubmissionStatuses to a given Evaluation. + Generator to get all bundled Submissions and SubmissionStatuses for a given Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. status: Optionally filter submission bundles by status. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10, max value 100. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A list of SubmissionBundle objects containing the submission bundles - for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Note: The caller must be granted the ACCESS_TYPE.READ_PRIVATE_SUBMISSION on the specified Evaluation. @@ -250,94 +240,84 @@ async def get_evaluation_submission_bundles_async( Example: Getting submission bundles for an evaluation   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import SubmissionBundle syn = Synapse() syn.login() - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id="9999999", - status="SCORED", - limit=50 - ) - print(f"Found {len(bundles)} submission bundles") - for bundle in bundles: - print(f"Submission ID: {bundle.submission.id}") + async def get_submission_bundles_example(): + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id="9999999", + status="SCORED" + ): + bundles.append(bundle) + print(f"Found {len(bundles)} submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id}") + + asyncio.run(get_submission_bundles_example()) ``` """ - response = await evaluation_services.get_evaluation_submission_bundles( + async for bundle_data in evaluation_services.get_evaluation_submission_bundles( evaluation_id=evaluation_id, status=status, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) - - bundles = [] - for bundle_dict in response.get("results", []): - bundle = SubmissionBundle() - bundle.fill_from_dict(bundle_dict) - bundles.append(bundle) + ): + bundle = cls() + bundle.fill_from_dict(bundle_data) + yield bundle - return bundles - - @staticmethod + @skip_async_to_sync + @classmethod async def get_user_submission_bundles_async( + cls, evaluation_id: str, - limit: int = 10, - offset: int = 0, *, synapse_client: Optional[Synapse] = None, - ) -> List["SubmissionBundle"]: + ) -> AsyncGenerator["SubmissionBundle", None]: """ - Gets the requesting user's bundled Submissions and SubmissionStatuses to a specified Evaluation. + Generator to get all user bundled Submissions and SubmissionStatuses for a specified Evaluation. Arguments: evaluation_id: The ID of the specified Evaluation. - limit: Limits the number of entities that will be fetched for this page. - When null it will default to 10. Default to 10. - offset: The offset index determines where this page will start from. - An index of 0 is the first entity. Default to 0. synapse_client: If not passed in and caching was not disabled by `Synapse.allow_client_caching(False)` this will use the last created instance from the Synapse class constructor. - Returns: - A list of SubmissionBundle objects containing the requesting user's - submission bundles for the evaluation queue. + Yields: + Individual SubmissionBundle objects from each page of the response. Example: Getting user submission bundles   ```python + import asyncio from synapseclient import Synapse from synapseclient.models import SubmissionBundle syn = Synapse() syn.login() - bundles = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id="9999999", - limit=25 - ) - print(f"Found {len(bundles)} user submission bundles") - for bundle in bundles: - print(f"Submission ID: {bundle.submission.id}") + async def get_user_submission_bundles_example(): + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( + evaluation_id="9999999" + ): + bundles.append(bundle) + print(f"Found {len(bundles)} user submission bundles") + for bundle in bundles: + print(f"Submission ID: {bundle.submission.id}") + + asyncio.run(get_user_submission_bundles_example()) ``` """ - response = await evaluation_services.get_user_submission_bundles( + async for bundle_data in evaluation_services.get_user_submission_bundles( evaluation_id=evaluation_id, - limit=limit, - offset=offset, synapse_client=synapse_client, - ) - - # Convert response to list of SubmissionBundle objects - bundles = [] - for bundle_dict in response.get("results", []): - bundle = SubmissionBundle() - bundle.fill_from_dict(bundle_dict) - bundles.append(bundle) - - return bundles + ): + bundle = cls() + bundle.fill_from_dict(bundle_data) + yield bundle diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index eb30af938..5f4273826 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -115,14 +115,15 @@ async def test_get_evaluation_submission_bundles_basic_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles for an evaluation using async methods.""" - # WHEN I get submission bundles for an evaluation - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles for an evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved - assert bundles is not None assert len(bundles) >= 1 # At least our test submission # AND each bundle should have proper structure @@ -133,10 +134,28 @@ async def test_get_evaluation_submission_bundles_basic_async( assert bundle.submission.id is not None assert bundle.submission.evaluation_id == test_evaluation.id - if bundle.submission.id == test_submission.id: - found_test_bundle = True - assert bundle.submission.entity_id == test_submission.entity_id - assert bundle.submission.name == test_submission.name + async def test_get_evaluation_submission_bundles_async_generator_behavior( + self, test_evaluation: Evaluation, test_submission: Submission + ): + # WHEN I get submission bundles using the async generator + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles_async( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + bundles = [] + async for bundle in bundles_generator: + assert isinstance(bundle, SubmissionBundle) + bundles.append(bundle) + + # AND all bundles should be valid SubmissionBundle objects + assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) + + if bundle.submission.id == test_submission.id: + found_test_bundle = True + assert bundle.submission.entity_id == test_submission.entity_id + assert bundle.submission.name == test_submission.name # AND our test submission should be found assert found_test_bundle, "Test submission should be found in bundles" @@ -146,11 +165,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter_async( ): """Test getting submission bundles filtered by status using async methods.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved assert bundles is not None @@ -162,63 +183,59 @@ async def test_get_evaluation_submission_bundles_with_status_filter_async( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, status="NONEXISTENT_STATUS", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) assert "NONEXISTENT_STATUS" in str(exc_info.value) - async def test_get_evaluation_submission_bundles_with_pagination_async( + async def test_get_evaluation_submission_bundles_with_automatic_pagination_async( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting submission bundles using async methods.""" - # WHEN I get submission bundles with a limit - bundles_page1 = await SubmissionBundle.get_evaluation_submission_bundles_async( + """Test automatic pagination when getting submission bundles using async generator methods.""" + # WHEN I get submission bundles using async generator (handles pagination automatically) + all_bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + ): + all_bundles.append(bundle) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all bundles for the evaluation + assert all_bundles is not None + assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions - # WHEN I get the next page - bundles_page2 = await SubmissionBundle.get_evaluation_submission_bundles_async( - evaluation_id=test_evaluation.id, - limit=2, - offset=2, - synapse_client=self.syn, - ) + # AND each bundle should be valid + for bundle in all_bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.evaluation_id == test_evaluation.id - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + # AND all our test submissions should be found + found_submission_ids = { + bundle.submission.id for bundle in all_bundles if bundle.submission + } + test_submission_ids = {submission.id for submission in multiple_submissions} + assert test_submission_ids.issubset( + found_submission_ids + ), "All test submissions should be found in the results" async def test_get_evaluation_submission_bundles_invalid_evaluation_async(self): """Test getting submission bundles for invalid evaluation ID using async methods.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - await SubmissionBundle.get_evaluation_submission_bundles_async( + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id="syn999999999999", synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -227,11 +244,13 @@ async def test_get_user_submission_bundles_basic_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting user submission bundles for an evaluation using async methods.""" - # WHEN I get user submission bundles for an evaluation - bundles = await SubmissionBundle.get_user_submission_bundles_async( + # WHEN I get user submission bundles for an evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundles should be retrieved assert bundles is not None @@ -253,44 +272,36 @@ async def test_get_user_submission_bundles_basic_async( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in user bundles" - async def test_get_user_submission_bundles_with_pagination_async( + async def test_get_user_submission_bundles_with_automatic_pagination_async( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting user submission bundles using async methods.""" - # WHEN I get user submission bundles with a limit - bundles_page1 = await SubmissionBundle.get_user_submission_bundles_async( + """Test automatic pagination when getting user submission bundles using async generator methods.""" + # WHEN I get user submission bundles using async generator (handles pagination automatically) + all_bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + ): + all_bundles.append(bundle) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all bundles for the user in this evaluation + assert all_bundles is not None + assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions - # WHEN I get the next page - bundles_page2 = await SubmissionBundle.get_user_submission_bundles_async( - evaluation_id=test_evaluation.id, - limit=2, - offset=2, - synapse_client=self.syn, - ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None + # AND each bundle should be valid + for bundle in all_bundles: + assert isinstance(bundle, SubmissionBundle) + assert bundle.submission is not None + assert bundle.submission.evaluation_id == test_evaluation.id - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + # AND all our test submissions should be found + found_submission_ids = { + bundle.submission.id for bundle in all_bundles if bundle.submission + } + test_submission_ids = {submission.id for submission in multiple_submissions} + assert test_submission_ids.issubset( + found_submission_ids + ), "All test submissions should be found in the user results" class TestSubmissionBundleDataIntegrityAsync: @@ -370,11 +381,13 @@ async def test_submission_bundle_data_consistency_async( self, test_evaluation: Evaluation, test_submission: Submission, test_file: File ): """Test that submission bundles maintain data consistency between submission and status using async methods.""" - # WHEN I get submission bundles for the evaluation - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles for the evaluation using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN I should find our test submission test_bundle = None @@ -415,11 +428,13 @@ async def test_submission_bundle_status_updates_reflected_async( } updated_status = await submission_status.store_async(synapse_client=self.syn) - # AND I get submission bundles again - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # AND I get submission bundles again using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN the bundle should reflect the updated status test_bundle = None @@ -446,11 +461,13 @@ async def test_submission_bundle_evaluation_id_propagation_async( self, test_evaluation: Evaluation, test_submission: Submission ): """Test that evaluation_id is properly propagated from submission to status using async methods.""" - # WHEN I get submission bundles - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN find our test bundle test_bundle = None @@ -542,11 +559,13 @@ async def test_get_evaluation_submission_bundles_empty_evaluation_async( self, test_evaluation: Evaluation ): """Test getting submission bundles from an evaluation with no submissions using async methods.""" - # WHEN I get submission bundles from an evaluation with no submissions - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I get submission bundles from an evaluation with no submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -557,43 +576,47 @@ async def test_get_user_submission_bundles_empty_evaluation_async( self, test_evaluation: Evaluation ): """Test getting user submission bundles from an evaluation with no submissions using async methods.""" - # WHEN I get user submission bundles from an evaluation with no submissions - bundles = await SubmissionBundle.get_user_submission_bundles_async( + # WHEN I get user submission bundles from an evaluation with no submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not None or error) assert bundles is not None assert isinstance(bundles, list) assert len(bundles) == 0 - async def test_get_evaluation_submission_bundles_large_limit_async( + async def test_get_evaluation_submission_bundles_all_results_async( self, test_evaluation: Evaluation ): - """Test getting submission bundles with a very large limit using async methods.""" - # WHEN I request bundles with a large limit (within API bounds) - bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + """Test getting all submission bundles using async generator methods.""" + # WHEN I request all bundles using async generator (no limit needed) + bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, - limit=100, # Maximum allowed by API synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should work without error assert bundles is not None assert isinstance(bundles, list) # The actual count doesn't matter since the evaluation is empty - async def test_get_user_submission_bundles_large_offset_async( + async def test_get_user_submission_bundles_empty_results_async( self, test_evaluation: Evaluation ): - """Test getting user submission bundles with a large offset using async methods.""" - # WHEN I request bundles with a large offset (beyond available data) - bundles = await SubmissionBundle.get_user_submission_bundles_async( + """Test getting user submission bundles when no results exist using async generator methods.""" + # WHEN I request bundles from an evaluation with no user submissions using async generator + bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, - offset=1000, # Large offset beyond any real data synapse_client=self.syn, - ) + ): + bundles.append(bundle) # THEN it should return an empty list (not error) assert bundles is not None @@ -604,15 +627,20 @@ async def test_get_submission_bundles_with_default_parameters_async( self, test_evaluation: Evaluation ): """Test that default parameters work correctly using async methods.""" - # WHEN I call methods without optional parameters - eval_bundles = await SubmissionBundle.get_evaluation_submission_bundles_async( + # WHEN I call methods without optional parameters using async generators + eval_bundles = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) - user_bundles = await SubmissionBundle.get_user_submission_bundles_async( + ): + eval_bundles.append(bundle) + + user_bundles = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + ): + user_bundles.append(bundle) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index a9816765b..cf62e4e22 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -111,14 +111,13 @@ async def test_get_evaluation_submission_bundles_basic( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles for an evaluation.""" - # WHEN I get submission bundles for an evaluation - bundles = SubmissionBundle.get_evaluation_submission_bundles( + # WHEN I get submission bundles for an evaluation using generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) - # THEN the bundles should be retrieved - assert bundles is not None + # THEN I should get at least our test submission assert len(bundles) >= 1 # At least our test submission # AND each bundle should have proper structure @@ -137,16 +136,35 @@ async def test_get_evaluation_submission_bundles_basic( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in bundles" + async def test_get_evaluation_submission_bundles_generator_behavior( + self, test_evaluation: Evaluation + ): + """Test that the generator returns SubmissionBundle objects correctly.""" + # WHEN I get submission bundles using the generator + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + + # THEN I should be able to iterate through the results + bundles = [] + for bundle in bundles_generator: + assert isinstance(bundle, SubmissionBundle) + bundles.append(bundle) + + # AND all bundles should be valid SubmissionBundle objects + assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) + async def test_get_evaluation_submission_bundles_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): """Test getting submission bundles filtered by status.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, status="RECEIVED", synapse_client=self.syn, - ) + )) # THEN the bundles should be retrieved assert bundles is not None @@ -158,63 +176,57 @@ async def test_get_evaluation_submission_bundles_with_status_filter( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - SubmissionBundle.get_evaluation_submission_bundles( + list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, status="NONEXISTENT_STATUS", synapse_client=self.syn, - ) + )) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) assert "NONEXISTENT_STATUS" in str(exc_info.value) - async def test_get_evaluation_submission_bundles_with_pagination( + async def test_get_evaluation_submission_bundles_generator_behavior_with_multiple( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting submission bundles.""" - # WHEN I get submission bundles with a limit - bundles_page1 = SubmissionBundle.get_evaluation_submission_bundles( + """Test generator behavior when getting submission bundles with multiple submissions.""" + # WHEN I get submission bundles using the generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + )) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all available bundles (at least the ones we created) + assert bundles is not None + assert len(bundles) >= len(multiple_submissions) - # WHEN I get the next page - bundles_page2 = SubmissionBundle.get_evaluation_submission_bundles( + # AND I should be able to iterate through the generator multiple times + # by creating a new generator each time + bundles_generator = SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=2, synapse_client=self.syn, ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + + # THEN I should get the same bundles when iterating again + bundles_second_iteration = list(bundles_generator) + assert len(bundles_second_iteration) == len(bundles) + + # AND all created submissions should be found + bundle_submission_ids = { + bundle.submission.id for bundle in bundles if bundle.submission + } + created_submission_ids = {sub.id for sub in multiple_submissions} + assert created_submission_ids.issubset(bundle_submission_ids), \ + "All created submissions should be found in bundles" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - SubmissionBundle.get_evaluation_submission_bundles( + list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id="syn999999999999", synapse_client=self.syn, - ) + )) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -224,10 +236,10 @@ async def test_get_user_submission_bundles_basic( ): """Test getting user submission bundles for an evaluation.""" # WHEN I get user submission bundles for an evaluation - bundles = SubmissionBundle.get_user_submission_bundles( + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN the bundles should be retrieved assert bundles is not None @@ -249,44 +261,38 @@ async def test_get_user_submission_bundles_basic( # AND our test submission should be found assert found_test_bundle, "Test submission should be found in user bundles" - async def test_get_user_submission_bundles_with_pagination( + async def test_get_user_submission_bundles_generator_behavior_with_multiple( self, test_evaluation: Evaluation, multiple_submissions: list[Submission] ): - """Test pagination when getting user submission bundles.""" - # WHEN I get user submission bundles with a limit - bundles_page1 = SubmissionBundle.get_user_submission_bundles( + """Test generator behavior when getting user submission bundles with multiple submissions.""" + # WHEN I get user submission bundles using the generator + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=0, synapse_client=self.syn, - ) + )) - # THEN I should get at most 2 bundles - assert bundles_page1 is not None - assert len(bundles_page1) <= 2 + # THEN I should get all available bundles (at least the ones we created) + assert bundles is not None + assert len(bundles) >= len(multiple_submissions) - # WHEN I get the next page - bundles_page2 = SubmissionBundle.get_user_submission_bundles( + # AND I should be able to iterate through the generator multiple times + # by creating a new generator each time + bundles_generator = SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - limit=2, - offset=2, synapse_client=self.syn, ) - - # THEN I should get different bundles (if there are more than 2 total) - assert bundles_page2 is not None - - # AND the bundle IDs should not overlap if we have enough submissions - if len(bundles_page1) == 2 and len(bundles_page2) > 0: - page1_ids = { - bundle.submission.id for bundle in bundles_page1 if bundle.submission - } - page2_ids = { - bundle.submission.id for bundle in bundles_page2 if bundle.submission - } - assert page1_ids.isdisjoint( - page2_ids - ), "Pages should not have overlapping submissions" + + # THEN I should get the same bundles when iterating again + bundles_second_iteration = list(bundles_generator) + assert len(bundles_second_iteration) == len(bundles) + + # AND all created submissions should be found + bundle_submission_ids = { + bundle.submission.id for bundle in bundles if bundle.submission + } + created_submission_ids = {sub.id for sub in multiple_submissions} + assert created_submission_ids.issubset(bundle_submission_ids), \ + "All created submissions should be found in user bundles" class TestSubmissionBundleDataIntegrity: @@ -363,10 +369,10 @@ async def test_submission_bundle_data_consistency( ): """Test that submission bundles maintain data consistency between submission and status.""" # WHEN I get submission bundles for the evaluation - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN I should find our test submission test_bundle = None @@ -408,10 +414,10 @@ async def test_submission_bundle_status_updates_reflected( updated_status = submission_status.store(synapse_client=self.syn) # AND I get submission bundles again - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN the bundle should reflect the updated status test_bundle = None @@ -439,10 +445,10 @@ async def test_submission_bundle_evaluation_id_propagation( ): """Test that evaluation_id is properly propagated from submission to status.""" # WHEN I get submission bundles - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN find our test bundle test_bundle = None @@ -497,10 +503,10 @@ async def test_get_evaluation_submission_bundles_empty_evaluation( ): """Test getting submission bundles from an evaluation with no submissions.""" # WHEN I get submission bundles from an evaluation with no submissions - bundles = SubmissionBundle.get_evaluation_submission_bundles( + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -512,42 +518,40 @@ async def test_get_user_submission_bundles_empty_evaluation( ): """Test getting user submission bundles from an evaluation with no submissions.""" # WHEN I get user submission bundles from an evaluation with no submissions - bundles = SubmissionBundle.get_user_submission_bundles( + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not None or error) assert bundles is not None assert isinstance(bundles, list) assert len(bundles) == 0 - async def test_get_evaluation_submission_bundles_large_limit( + async def test_get_evaluation_submission_bundles_generator_consistency( self, test_evaluation: Evaluation ): - """Test getting submission bundles with a very large limit.""" - # WHEN I request bundles with a large limit (within API bounds) - bundles = SubmissionBundle.get_evaluation_submission_bundles( + """Test that the generator produces consistent results across multiple iterations.""" + # WHEN I request bundles using the generator + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, - limit=100, # Maximum allowed by API synapse_client=self.syn, - ) + )) # THEN it should work without error assert bundles is not None assert isinstance(bundles, list) # The actual count doesn't matter since the evaluation is empty - async def test_get_user_submission_bundles_large_offset( + async def test_get_user_submission_bundles_generator_empty_results( self, test_evaluation: Evaluation ): - """Test getting user submission bundles with a large offset.""" - # WHEN I request bundles with a large offset (beyond available data) - bundles = SubmissionBundle.get_user_submission_bundles( + """Test that user submission bundles generator handles empty results correctly.""" + # WHEN I request bundles from an empty evaluation + bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, - offset=1000, # Large offset beyond any real data synapse_client=self.syn, - ) + )) # THEN it should return an empty list (not error) assert bundles is not None @@ -559,14 +563,14 @@ async def test_get_submission_bundles_with_default_parameters( ): """Test that default parameters work correctly.""" # WHEN I call methods without optional parameters - eval_bundles = SubmissionBundle.get_evaluation_submission_bundles( + eval_bundles = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) - user_bundles = SubmissionBundle.get_user_submission_bundles( + )) + user_bundles = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=test_evaluation.id, synapse_client=self.syn, - ) + )) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py index 1e2efca02..5253c2f27 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_bundle_async.py @@ -239,24 +239,28 @@ async def test_get_evaluation_submission_bundles_async(self) -> None: # WHEN I call get_evaluation_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = await SubmissionBundle.get_evaluation_submission_bundles_async( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, ) @@ -288,21 +292,27 @@ async def test_get_evaluation_submission_bundles_async_empty_response(self) -> N # WHEN I call get_evaluation_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = await SubmissionBundle.get_evaluation_submission_bundles_async( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield # This will never execute + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_evaluation_submission_bundles_async( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status=None, - limit=10, - offset=0, synapse_client=self.syn, ) @@ -333,22 +343,26 @@ async def test_get_user_submission_bundles_async(self) -> None: # WHEN I call get_user_submission_bundles_async with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = await SubmissionBundle.get_user_submission_bundles_async( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, ) @@ -372,20 +386,26 @@ async def test_get_user_submission_bundles_async_default_params(self) -> None: # WHEN I call get_user_submission_bundles_async with defaults with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = await SubmissionBundle.get_user_submission_bundles_async( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = [] + async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + ): + result.append(bundle) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=10, - offset=0, synapse_client=self.syn, ) diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index 9dc252286..f06f52c4b 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -239,24 +239,26 @@ def test_get_evaluation_submission_bundles(self) -> None: # WHEN I call get_evaluation_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = SubmissionBundle.get_evaluation_submission_bundles( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, - ) + )) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status="RECEIVED", - limit=50, - offset=0, synapse_client=self.syn, ) @@ -288,21 +290,25 @@ def test_get_evaluation_submission_bundles_empty_response(self) -> None: # WHEN I call get_evaluation_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_evaluation_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_evaluation_submission_bundles" ) as mock_get_bundles: - result = SubmissionBundle.get_evaluation_submission_bundles( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield + + # Make the mock return our async generator when called + mock_get_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_evaluation_submission_bundles( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + )) # THEN the service should be called mock_get_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, status=None, - limit=10, - offset=0, synapse_client=self.syn, ) @@ -333,22 +339,24 @@ def test_get_user_submission_bundles(self) -> None: # WHEN I call get_user_submission_bundles (sync method) with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = SubmissionBundle.get_user_submission_bundles( + # Create an async generator function that yields bundle data + async def mock_async_gen(*args, **kwargs): + for bundle_data in mock_response["results"]: + yield bundle_data + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, - ) + )) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=25, - offset=5, synapse_client=self.syn, ) @@ -372,20 +380,24 @@ def test_get_user_submission_bundles_default_params(self) -> None: # WHEN I call get_user_submission_bundles with defaults (sync method) with patch( - "synapseclient.api.evaluation_services.get_user_submission_bundles", - new_callable=AsyncMock, - return_value=mock_response, + "synapseclient.api.evaluation_services.get_user_submission_bundles" ) as mock_get_user_bundles: - result = SubmissionBundle.get_user_submission_bundles( + # Create an async generator function that yields no data + async def mock_async_gen(*args, **kwargs): + return + yield # This will never execute + + # Make the mock return our async generator when called + mock_get_user_bundles.side_effect = mock_async_gen + + result = list(SubmissionBundle.get_user_submission_bundles( evaluation_id=EVALUATION_ID, synapse_client=self.syn, - ) + )) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( evaluation_id=EVALUATION_ID, - limit=10, - offset=0, synapse_client=self.syn, ) From a9977b24cb8800b15f73d42fb797ce549b7cf58c Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 14:16:51 -0500 Subject: [PATCH 38/46] address final todo: implement docker_tag, and note in docs that version_number can be ignored for docker submissions --- synapseclient/models/submission.py | 20 +- synapseclient/models/submission_bundle.py | 16 +- .../models/async/test_submission_async.py | 4 +- .../async/test_submission_bundle_async.py | 12 +- .../models/synchronous/test_submission.py | 24 ++- .../synchronous/test_submission_bundle.py | 184 +++++++++++------- .../async/unit_test_submission_async.py | 2 +- .../synchronous/unit_test_submission.py | 4 +- .../unit_test_submission_bundle.py | 42 ++-- 9 files changed, 189 insertions(+), 119 deletions(-) diff --git a/synapseclient/models/submission.py b/synapseclient/models/submission.py index 08dc642ee..f90660d26 100644 --- a/synapseclient/models/submission.py +++ b/synapseclient/models/submission.py @@ -5,7 +5,12 @@ from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, otel_trace_method, wrap_async_generator_to_sync_generator +from synapseclient.core.async_utils import ( + async_to_sync, + otel_trace_method, + skip_async_to_sync, + wrap_async_generator_to_sync_generator, +) from synapseclient.models.mixins.access_control import AccessControllable @@ -340,7 +345,7 @@ class Submission( version_number: Optional[int] = field(default=None, compare=False) """ - The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. + The version number of the entity at submission. If not provided, it will be automatically retrieved from the entity. If entity is a Docker repository, this attribute should be ignored in favor of `docker_digest` or `docker_tag`. """ evaluation_id: Optional[str] = None @@ -389,6 +394,11 @@ class Submission( For Docker repositories, the digest of the submitted Docker image. """ + docker_tag: Optional[str] = None + """ + For Docker repositories, the tag of the submitted Docker image. + """ + etag: Optional[str] = None """The current eTag of the Entity being submitted. If not provided, it will be automatically retrieved.""" @@ -580,11 +590,11 @@ async def create_submission_example(): entity_info.get("concreteType") == "org.sagebionetworks.repo.model.docker.DockerRepository" ): - self.version_number = ( - 1 # TODO: Docker repositories do not have version numbers - ) self.docker_repository_name = entity_info.get("repositoryName") self.docker_digest = entity_info.get("digest") + self.docker_tag = entity_info.get("tag") + # All docker repositories are assigned version number 1, even if they have multiple tags + self.version_number = 1 else: raise ValueError("entity_id is required to create a submission") diff --git a/synapseclient/models/submission_bundle.py b/synapseclient/models/submission_bundle.py index 75dde8610..ffd56d489 100644 --- a/synapseclient/models/submission_bundle.py +++ b/synapseclient/models/submission_bundle.py @@ -1,9 +1,21 @@ from dataclasses import dataclass -from typing import TYPE_CHECKING, AsyncGenerator, Dict, Generator, List, Optional, Protocol, Union +from typing import ( + TYPE_CHECKING, + AsyncGenerator, + Dict, + Generator, + Optional, + Protocol, + Union, +) from synapseclient import Synapse from synapseclient.api import evaluation_services -from synapseclient.core.async_utils import async_to_sync, skip_async_to_sync, wrap_async_generator_to_sync_generator +from synapseclient.core.async_utils import ( + async_to_sync, + skip_async_to_sync, + wrap_async_generator_to_sync_generator, +) if TYPE_CHECKING: from synapseclient.models.submission import Submission diff --git a/tests/integration/synapseclient/models/async/test_submission_async.py b/tests/integration/synapseclient/models/async/test_submission_async.py index 5ef52ed4e..f7cb4fdda 100644 --- a/tests/integration/synapseclient/models/async/test_submission_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_async.py @@ -290,7 +290,7 @@ async def test_get_evaluation_submissions_async_generator_behavior( async for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) @@ -320,7 +320,7 @@ async def test_get_user_submissions_async_generator_behavior( async for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) diff --git a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py index 5f4273826..b24f8c0ef 100644 --- a/tests/integration/synapseclient/models/async/test_submission_bundle_async.py +++ b/tests/integration/synapseclient/models/async/test_submission_bundle_async.py @@ -148,7 +148,7 @@ async def test_get_evaluation_submission_bundles_async_generator_behavior( async for bundle in bundles_generator: assert isinstance(bundle, SubmissionBundle) bundles.append(bundle) - + # AND all bundles should be valid SubmissionBundle objects assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) @@ -209,7 +209,9 @@ async def test_get_evaluation_submission_bundles_with_automatic_pagination_async # THEN I should get all bundles for the evaluation assert all_bundles is not None - assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions + assert len(all_bundles) >= len( + multiple_submissions + ) # At least our test submissions # AND each bundle should be valid for bundle in all_bundles: @@ -286,7 +288,9 @@ async def test_get_user_submission_bundles_with_automatic_pagination_async( # THEN I should get all bundles for the user in this evaluation assert all_bundles is not None - assert len(all_bundles) >= len(multiple_submissions) # At least our test submissions + assert len(all_bundles) >= len( + multiple_submissions + ) # At least our test submissions # AND each bundle should be valid for bundle in all_bundles: @@ -634,7 +638,7 @@ async def test_get_submission_bundles_with_default_parameters_async( synapse_client=self.syn, ): eval_bundles.append(bundle) - + user_bundles = [] async for bundle in SubmissionBundle.get_user_submission_bundles_async( evaluation_id=test_evaluation.id, diff --git a/tests/integration/synapseclient/models/synchronous/test_submission.py b/tests/integration/synapseclient/models/synchronous/test_submission.py index 16fa52b1a..e89800042 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission.py @@ -239,9 +239,11 @@ async def test_get_evaluation_submissions( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get all submissions for an evaluation - submissions = list(Submission.get_evaluation_submissions( - evaluation_id=test_evaluation.id, synapse_client=self.syn - )) + submissions = list( + Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, synapse_client=self.syn + ) + ) # THEN I should get a list of submission objects assert len(submissions) > 0 @@ -255,11 +257,13 @@ async def test_get_evaluation_submissions_with_status_filter( self, test_evaluation: Evaluation, test_submission: Submission ): # WHEN I get submissions filtered by status - submissions = list(Submission.get_evaluation_submissions( - evaluation_id=test_evaluation.id, - status="RECEIVED", - synapse_client=self.syn, - )) + submissions = list( + Submission.get_evaluation_submissions( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # The test submission should be in the results (initially in RECEIVED status) submission_ids = [sub.id for sub in submissions] @@ -279,7 +283,7 @@ async def test_get_evaluation_submissions_generator_behavior( for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) @@ -308,7 +312,7 @@ async def test_get_user_submissions_generator_behavior( for submission in submissions_generator: assert isinstance(submission, Submission) submissions.append(submission) - + # AND all submissions should be valid Submission objects assert all(isinstance(sub, Submission) for sub in submissions) diff --git a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py index cf62e4e22..70d2bbf79 100644 --- a/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py +++ b/tests/integration/synapseclient/models/synchronous/test_submission_bundle.py @@ -112,10 +112,12 @@ async def test_get_evaluation_submission_bundles_basic( ): """Test getting submission bundles for an evaluation.""" # WHEN I get submission bundles for an evaluation using generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get at least our test submission assert len(bundles) >= 1 # At least our test submission @@ -151,7 +153,7 @@ async def test_get_evaluation_submission_bundles_generator_behavior( for bundle in bundles_generator: assert isinstance(bundle, SubmissionBundle) bundles.append(bundle) - + # AND all bundles should be valid SubmissionBundle objects assert all(isinstance(bundle, SubmissionBundle) for bundle in bundles) @@ -160,11 +162,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter( ): """Test getting submission bundles filtered by status.""" # WHEN I get submission bundles filtered by "RECEIVED" status - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - status="RECEIVED", - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # THEN the bundles should be retrieved assert bundles is not None @@ -176,11 +180,13 @@ async def test_get_evaluation_submission_bundles_with_status_filter( # WHEN I attempt to get submission bundles with an invalid status with pytest.raises(SynapseHTTPError) as exc_info: - list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - status="NONEXISTENT_STATUS", - synapse_client=self.syn, - )) + list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + status="NONEXISTENT_STATUS", + synapse_client=self.syn, + ) + ) # THEN it should raise a SynapseHTTPError (400 for invalid enum) assert exc_info.value.response.status_code == 400 assert "No enum constant" in str(exc_info.value) @@ -191,10 +197,12 @@ async def test_get_evaluation_submission_bundles_generator_behavior_with_multipl ): """Test generator behavior when getting submission bundles with multiple submissions.""" # WHEN I get submission bundles using the generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get all available bundles (at least the ones we created) assert bundles is not None @@ -206,27 +214,30 @@ async def test_get_evaluation_submission_bundles_generator_behavior_with_multipl evaluation_id=test_evaluation.id, synapse_client=self.syn, ) - + # THEN I should get the same bundles when iterating again bundles_second_iteration = list(bundles_generator) assert len(bundles_second_iteration) == len(bundles) - + # AND all created submissions should be found bundle_submission_ids = { bundle.submission.id for bundle in bundles if bundle.submission } created_submission_ids = {sub.id for sub in multiple_submissions} - assert created_submission_ids.issubset(bundle_submission_ids), \ - "All created submissions should be found in bundles" + assert created_submission_ids.issubset( + bundle_submission_ids + ), "All created submissions should be found in bundles" async def test_get_evaluation_submission_bundles_invalid_evaluation(self): """Test getting submission bundles for invalid evaluation ID.""" # WHEN I try to get submission bundles for a non-existent evaluation with pytest.raises(SynapseHTTPError) as exc_info: - list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id="syn999999999999", - synapse_client=self.syn, - )) + list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id="syn999999999999", + synapse_client=self.syn, + ) + ) # THEN it should raise a SynapseHTTPError (likely 403 or 404) assert exc_info.value.response.status_code in [403, 404] @@ -236,10 +247,12 @@ async def test_get_user_submission_bundles_basic( ): """Test getting user submission bundles for an evaluation.""" # WHEN I get user submission bundles for an evaluation - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN the bundles should be retrieved assert bundles is not None @@ -266,10 +279,12 @@ async def test_get_user_submission_bundles_generator_behavior_with_multiple( ): """Test generator behavior when getting user submission bundles with multiple submissions.""" # WHEN I get user submission bundles using the generator - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should get all available bundles (at least the ones we created) assert bundles is not None @@ -281,18 +296,19 @@ async def test_get_user_submission_bundles_generator_behavior_with_multiple( evaluation_id=test_evaluation.id, synapse_client=self.syn, ) - + # THEN I should get the same bundles when iterating again bundles_second_iteration = list(bundles_generator) assert len(bundles_second_iteration) == len(bundles) - + # AND all created submissions should be found bundle_submission_ids = { bundle.submission.id for bundle in bundles if bundle.submission } created_submission_ids = {sub.id for sub in multiple_submissions} - assert created_submission_ids.issubset(bundle_submission_ids), \ - "All created submissions should be found in user bundles" + assert created_submission_ids.issubset( + bundle_submission_ids + ), "All created submissions should be found in user bundles" class TestSubmissionBundleDataIntegrity: @@ -369,10 +385,12 @@ async def test_submission_bundle_data_consistency( ): """Test that submission bundles maintain data consistency between submission and status.""" # WHEN I get submission bundles for the evaluation - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN I should find our test submission test_bundle = None @@ -414,10 +432,12 @@ async def test_submission_bundle_status_updates_reflected( updated_status = submission_status.store(synapse_client=self.syn) # AND I get submission bundles again - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN the bundle should reflect the updated status test_bundle = None @@ -445,10 +465,12 @@ async def test_submission_bundle_evaluation_id_propagation( ): """Test that evaluation_id is properly propagated from submission to status.""" # WHEN I get submission bundles - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN find our test bundle test_bundle = None @@ -503,10 +525,12 @@ async def test_get_evaluation_submission_bundles_empty_evaluation( ): """Test getting submission bundles from an evaluation with no submissions.""" # WHEN I get submission bundles from an evaluation with no submissions - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -518,10 +542,12 @@ async def test_get_user_submission_bundles_empty_evaluation( ): """Test getting user submission bundles from an evaluation with no submissions.""" # WHEN I get user submission bundles from an evaluation with no submissions - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not None or error) assert bundles is not None @@ -533,10 +559,12 @@ async def test_get_evaluation_submission_bundles_generator_consistency( ): """Test that the generator produces consistent results across multiple iterations.""" # WHEN I request bundles using the generator - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should work without error assert bundles is not None @@ -548,10 +576,12 @@ async def test_get_user_submission_bundles_generator_empty_results( ): """Test that user submission bundles generator handles empty results correctly.""" # WHEN I request bundles from an empty evaluation - bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN it should return an empty list (not error) assert bundles is not None @@ -563,14 +593,18 @@ async def test_get_submission_bundles_with_default_parameters( ): """Test that default parameters work correctly.""" # WHEN I call methods without optional parameters - eval_bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) - user_bundles = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=test_evaluation.id, - synapse_client=self.syn, - )) + eval_bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) + user_bundles = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=test_evaluation.id, + synapse_client=self.syn, + ) + ) # THEN both should work with default values assert eval_bundles is not None diff --git a/tests/unit/synapseclient/models/async/unit_test_submission_async.py b/tests/unit/synapseclient/models/async/unit_test_submission_async.py index a18f22fe8..3587c0cdb 100644 --- a/tests/unit/synapseclient/models/async/unit_test_submission_async.py +++ b/tests/unit/synapseclient/models/async/unit_test_submission_async.py @@ -488,7 +488,7 @@ async def test_get_evaluation_submissions_async(self) -> None: async def mock_async_gen(*args, **kwargs): submission_data = self.get_example_submission_response() yield submission_data - + # Make the mock return our async generator when called mock_get_submissions.side_effect = mock_async_gen diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py index c1d2eb2c0..b7d6f2721 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission.py @@ -614,9 +614,7 @@ async def test_cancel_async_success(self) -> None: # Mock the logger self.syn.logger = MagicMock() - await submission.cancel_async( - synapse_client=self.syn - ) + await submission.cancel_async(synapse_client=self.syn) # THEN it should call the API, log the cancellation, and update the object mock_cancel_submission.assert_called_once_with( diff --git a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py index f06f52c4b..cb5d44018 100644 --- a/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py +++ b/tests/unit/synapseclient/models/synchronous/unit_test_submission_bundle.py @@ -249,11 +249,13 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - status="RECEIVED", - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="RECEIVED", + synapse_client=self.syn, + ) + ) # THEN the service should be called with correct parameters mock_get_bundles.assert_called_once_with( @@ -300,10 +302,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called mock_get_bundles.assert_called_once_with( @@ -349,10 +353,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_user_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called with correct parameters mock_get_user_bundles.assert_called_once_with( @@ -390,10 +396,12 @@ async def mock_async_gen(*args, **kwargs): # Make the mock return our async generator when called mock_get_user_bundles.side_effect = mock_async_gen - result = list(SubmissionBundle.get_user_submission_bundles( - evaluation_id=EVALUATION_ID, - synapse_client=self.syn, - )) + result = list( + SubmissionBundle.get_user_submission_bundles( + evaluation_id=EVALUATION_ID, + synapse_client=self.syn, + ) + ) # THEN the service should be called with default parameters mock_get_user_bundles.assert_called_once_with( From 689fcb1459c1f1f9adb9d43b35b566bdf6782ec9 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Tue, 30 Sep 2025 11:37:00 -0400 Subject: [PATCH 39/46] add async page in api references --- docs/reference/experimental/async/submission.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/reference/experimental/async/submission.md diff --git a/docs/reference/experimental/async/submission.md b/docs/reference/experimental/async/submission.md new file mode 100644 index 000000000..a152d1452 --- /dev/null +++ b/docs/reference/experimental/async/submission.md @@ -0,0 +1,5 @@ +# Submission + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. \ No newline at end of file From c54609b9a170233ca4d5b658d193c9610385c1f2 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Fri, 21 Nov 2025 16:25:56 -0500 Subject: [PATCH 40/46] add initial submission, status, bundle docs --- .../experimental/async/submission.md | 16 ++++++++++++- .../experimental/async/submission_bundle.md | 14 +++++++++++ .../experimental/async/submission_status.md | 16 +++++++++++++ .../reference/experimental/sync/submission.md | 19 +++++++++++++++ .../experimental/sync/submission_bundle.md | 14 +++++++++++ .../experimental/sync/submission_status.md | 16 +++++++++++++ docs/tutorials/python/submission.md | 23 +++++++++++++++++++ 7 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 docs/reference/experimental/async/submission_bundle.md create mode 100644 docs/reference/experimental/async/submission_status.md create mode 100644 docs/reference/experimental/sync/submission.md create mode 100644 docs/reference/experimental/sync/submission_bundle.md create mode 100644 docs/reference/experimental/sync/submission_status.md create mode 100644 docs/tutorials/python/submission.md diff --git a/docs/reference/experimental/async/submission.md b/docs/reference/experimental/async/submission.md index a152d1452..21f4a0c9a 100644 --- a/docs/reference/experimental/async/submission.md +++ b/docs/reference/experimental/async/submission.md @@ -2,4 +2,18 @@ Contained within this file are experimental interfaces for working with the Synapse Python Client. Unless otherwise noted these interfaces are subject to change at any time. Use -at your own risk. \ No newline at end of file +at your own risk. + +## API Reference + +::: synapseclient.models.Submission + options: + inherited_members: true + members: + - store_async + - get_async + - delete_async + - cancel_async + - get_evaluation_submissions_async + - get_user_submissions_async + - get_submission_count_async diff --git a/docs/reference/experimental/async/submission_bundle.md b/docs/reference/experimental/async/submission_bundle.md new file mode 100644 index 000000000..60695a4ed --- /dev/null +++ b/docs/reference/experimental/async/submission_bundle.md @@ -0,0 +1,14 @@ +# Submission Bundle + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. + +## API Reference + +::: synapseclient.models.SubmissionBundle + options: + inherited_members: true + members: + - get_evaluation_submission_bundles_async + - get_user_submission_bundles_async \ No newline at end of file diff --git a/docs/reference/experimental/async/submission_status.md b/docs/reference/experimental/async/submission_status.md new file mode 100644 index 000000000..46e1c9ed0 --- /dev/null +++ b/docs/reference/experimental/async/submission_status.md @@ -0,0 +1,16 @@ +# Submission Status + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. + +## API Reference + +::: synapseclient.models.SubmissionStatus + options: + inherited_members: true + members: + - get_async + - store_async + - get_all_submission_statuses_async + - batch_update_submission_statuses_async \ No newline at end of file diff --git a/docs/reference/experimental/sync/submission.md b/docs/reference/experimental/sync/submission.md new file mode 100644 index 000000000..31f5eecde --- /dev/null +++ b/docs/reference/experimental/sync/submission.md @@ -0,0 +1,19 @@ +# Submission + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. + +## API Reference + +::: synapseclient.models.Submission + options: + inherited_members: true + members: + - store + - get + - delete + - cancel + - get_evaluation_submissions + - get_user_submissions + - get_submission_count diff --git a/docs/reference/experimental/sync/submission_bundle.md b/docs/reference/experimental/sync/submission_bundle.md new file mode 100644 index 000000000..7897ac5c2 --- /dev/null +++ b/docs/reference/experimental/sync/submission_bundle.md @@ -0,0 +1,14 @@ +# Submission Bundle + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. + +## API Reference + +::: synapseclient.models.SubmissionBundle + options: + inherited_members: true + members: + - get_evaluation_submission_bundles + - get_user_submission_bundles \ No newline at end of file diff --git a/docs/reference/experimental/sync/submission_status.md b/docs/reference/experimental/sync/submission_status.md new file mode 100644 index 000000000..ee8ffd8f2 --- /dev/null +++ b/docs/reference/experimental/sync/submission_status.md @@ -0,0 +1,16 @@ +# Submission Status + +Contained within this file are experimental interfaces for working with the Synapse Python +Client. Unless otherwise noted these interfaces are subject to change at any time. Use +at your own risk. + +## API Reference + +::: synapseclient.models.SubmissionStatus + options: + inherited_members: true + members: + - get + - store + - get_all_submission_statuses + - batch_update_submission_statuses \ No newline at end of file diff --git a/docs/tutorials/python/submission.md b/docs/tutorials/python/submission.md new file mode 100644 index 000000000..04381be33 --- /dev/null +++ b/docs/tutorials/python/submission.md @@ -0,0 +1,23 @@ +# Submissions, SubmissionStatuses, SubmissionBundles +An Evaluation is essentially a container that organizes and manages the submission, assessment, and scoring of data, models, or other research artifacts. +It allows teams to set up challenges where participants contribute their work, and those contributions can be systematically reviewed or scored. + +This tutorial will walk you through the basics of working with Evaluations using the Synapse Python client. + +## Tutorial Purpose +In this tutorial you will: + +1. Make a submission to an existing evaluation queue on Synapse +1. Fetch your existing submission +1. Count your submissions +1. Fetch all of your submissions from an existing evaluation queue on Synapse +1. Check the status of your submission +1. Annotate a submission +1. Batch-update submission +1. Delete your submission +1. Fetch submission bundle for a given evaluation + +## Prerequisites +* + +## 1. \ No newline at end of file From e0a79ccfba3b99f63ce453c74b92a61107d10f08 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 24 Nov 2025 10:26:52 -0500 Subject: [PATCH 41/46] updated tutorial purpose. add api reference to navbar --- .../experimental/async/submission_bundle.md | 2 +- .../experimental/async/submission_status.md | 2 +- .../reference/experimental/sync/submission.md | 14 ++++---- .../experimental/sync/submission_bundle.md | 4 +-- .../experimental/sync/submission_status.md | 8 ++--- docs/tutorials/python/submission.md | 35 +++++++++++++------ mkdocs.yml | 7 ++++ 7 files changed, 47 insertions(+), 25 deletions(-) diff --git a/docs/reference/experimental/async/submission_bundle.md b/docs/reference/experimental/async/submission_bundle.md index 60695a4ed..39c43a90a 100644 --- a/docs/reference/experimental/async/submission_bundle.md +++ b/docs/reference/experimental/async/submission_bundle.md @@ -11,4 +11,4 @@ at your own risk. inherited_members: true members: - get_evaluation_submission_bundles_async - - get_user_submission_bundles_async \ No newline at end of file + - get_user_submission_bundles_async diff --git a/docs/reference/experimental/async/submission_status.md b/docs/reference/experimental/async/submission_status.md index 46e1c9ed0..c1adfa346 100644 --- a/docs/reference/experimental/async/submission_status.md +++ b/docs/reference/experimental/async/submission_status.md @@ -13,4 +13,4 @@ at your own risk. - get_async - store_async - get_all_submission_statuses_async - - batch_update_submission_statuses_async \ No newline at end of file + - batch_update_submission_statuses_async diff --git a/docs/reference/experimental/sync/submission.md b/docs/reference/experimental/sync/submission.md index 31f5eecde..533c8c84c 100644 --- a/docs/reference/experimental/sync/submission.md +++ b/docs/reference/experimental/sync/submission.md @@ -10,10 +10,10 @@ at your own risk. options: inherited_members: true members: - - store - - get - - delete - - cancel - - get_evaluation_submissions - - get_user_submissions - - get_submission_count + - store + - get + - delete + - cancel + - get_evaluation_submissions + - get_user_submissions + - get_submission_count diff --git a/docs/reference/experimental/sync/submission_bundle.md b/docs/reference/experimental/sync/submission_bundle.md index 7897ac5c2..e7440ee8d 100644 --- a/docs/reference/experimental/sync/submission_bundle.md +++ b/docs/reference/experimental/sync/submission_bundle.md @@ -10,5 +10,5 @@ at your own risk. options: inherited_members: true members: - - get_evaluation_submission_bundles - - get_user_submission_bundles \ No newline at end of file + - get_evaluation_submission_bundles + - get_user_submission_bundles diff --git a/docs/reference/experimental/sync/submission_status.md b/docs/reference/experimental/sync/submission_status.md index ee8ffd8f2..9a2e482a6 100644 --- a/docs/reference/experimental/sync/submission_status.md +++ b/docs/reference/experimental/sync/submission_status.md @@ -10,7 +10,7 @@ at your own risk. options: inherited_members: true members: - - get - - store - - get_all_submission_statuses - - batch_update_submission_statuses \ No newline at end of file + - get + - store + - get_all_submission_statuses + - batch_update_submission_statuses diff --git a/docs/tutorials/python/submission.md b/docs/tutorials/python/submission.md index 04381be33..d5c44de57 100644 --- a/docs/tutorials/python/submission.md +++ b/docs/tutorials/python/submission.md @@ -1,23 +1,38 @@ # Submissions, SubmissionStatuses, SubmissionBundles -An Evaluation is essentially a container that organizes and manages the submission, assessment, and scoring of data, models, or other research artifacts. -It allows teams to set up challenges where participants contribute their work, and those contributions can be systematically reviewed or scored. +Users can work with Submissions on the python client, since these objects are part of the Evaluation API data model. A user in a Synapse Evaluation can submit a Synapse Entity as a Submission to that Evaluation. Submission data is owned by the parent Evaluation, and is immutable. + +The data model includes additional objects to support scoring of Submissions and convenient data access: + +- **SubmissionStatus**: An object used to track scoring information for a single Submission. This object is intended to be modified by the users (or test harnesses) managing the Evaluation. +- **SubmissionBundle**: A convenience object to transport a Submission and its accompanying SubmissionStatus in a single web service call. + +This tutorial will demonstrate how to work with all 3 object types using the python client for 2 different use-cases: + +1. Participating in a Synapse challenge +1. Organizing a Synapse challenge -This tutorial will walk you through the basics of working with Evaluations using the Synapse Python client. ## Tutorial Purpose -In this tutorial you will: +In this tutorial: + +As a participant of a Synapse challenge, you will 1. Make a submission to an existing evaluation queue on Synapse 1. Fetch your existing submission 1. Count your submissions 1. Fetch all of your submissions from an existing evaluation queue on Synapse 1. Check the status of your submission -1. Annotate a submission -1. Batch-update submission -1. Delete your submission -1. Fetch submission bundle for a given evaluation +1. Cancel your submission + +As an organizer of a Synapse challenge, you will + +1. Annotate a submission to score it +1. Batch-update submission statuses +1. Fetch the submission bundle for a given submission +1. Allow cancellation of submissions +1. Delete submissions ## Prerequisites -* +* -## 1. \ No newline at end of file +## 1. diff --git a/mkdocs.yml b/mkdocs.yml index cca09f73a..864a9a4d6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -28,6 +28,7 @@ nav: - Folder: tutorials/python/folder.md - File: tutorials/python/file.md - Evaluation: tutorials/python/evaluation.md + - Submission: tutorials/python/submission.md - Annotation: tutorials/python/annotation.md # - Versions: tutorials/python/versions.md # - Activity/Provenance: tutorials/python/activity.md @@ -89,6 +90,9 @@ nav: - Folder: reference/experimental/sync/folder.md - File: reference/experimental/sync/file.md - Evaluation: reference/experimental/sync/evaluation.md + - Submission: reference/experimental/sync/submission.md + - SubmissionStatus: reference/experimental/sync/submission_status.md + - SubmissionBundle: reference/experimental/sync/submission_bundle.md - Table: reference/experimental/sync/table.md - VirtualTable: reference/experimental/sync/virtualtable.md - Dataset: reference/experimental/sync/dataset.md @@ -113,6 +117,9 @@ nav: - Folder: reference/experimental/async/folder.md - File: reference/experimental/async/file.md - Evaluation: reference/experimental/async/evaluation.md + - Submission: reference/experimental/async/submission.md + - SubmissionStatus: reference/experimental/async/submission_status.md + - SubmissionBundle: reference/experimental/async/submission_bundle.md - Table: reference/experimental/async/table.md - VirtualTable: reference/experimental/async/virtualtable.md - Dataset: reference/experimental/async/dataset.md From 35efed77faa784a7c039787c0c71ae5a13bfaf74 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 24 Nov 2025 13:00:54 -0500 Subject: [PATCH 42/46] new tutorial scripts --- docs/tutorials/python/submission.md | 34 ++- .../tutorial_scripts/submission_organizer.py | 228 ++++++++++++++++++ .../submission_participant.py | 156 ++++++++++++ 3 files changed, 416 insertions(+), 2 deletions(-) create mode 100644 docs/tutorials/python/tutorial_scripts/submission_organizer.py create mode 100644 docs/tutorials/python/tutorial_scripts/submission_participant.py diff --git a/docs/tutorials/python/submission.md b/docs/tutorials/python/submission.md index d5c44de57..dd0a98225 100644 --- a/docs/tutorials/python/submission.md +++ b/docs/tutorials/python/submission.md @@ -33,6 +33,36 @@ As an organizer of a Synapse challenge, you will 1. Delete submissions ## Prerequisites -* +* You have completed the [Evaluation](./evaluation.md) tutorial, or have an existing Evaluation on Synapse to work from +* You have an existing entity with which to make a submission (can be a File or Docker Repository) +* You have the correct permissions on the Evaluation queue for your desired tutorial section (participant or organizer) -## 1. +## 1. Participating in a Synapse challenge + +### 1. Make a submission to an existing evaluation queue on Synapse + +```python +{!docs/tutorials/python/tutorial_scripts/submission.py!lines=5-46} +``` + +### 2. Fetch your existing submission + +### 3. Count your submissions + +### 4. Fetch all of your submissions from an existing evaluation queue on Synapse + +### 5. Check the status of your submission + +### 6. Cancel your submission + +## 2. Organizing a Synapse challenge + +### 1. Annotate a submission to score it + +### 2. Batch-update submission statuses + +### 3. Fetch the submission bundle for a given submission + +### 4. Allow cancellation of submissions + +### 5. Delete submissions diff --git a/docs/tutorials/python/tutorial_scripts/submission_organizer.py b/docs/tutorials/python/tutorial_scripts/submission_organizer.py new file mode 100644 index 000000000..94941c7eb --- /dev/null +++ b/docs/tutorials/python/tutorial_scripts/submission_organizer.py @@ -0,0 +1,228 @@ +""" +Submission Organizer Tutorial - Code for working with Submissions as a challenge organizer. + +This tutorial demonstrates how to: +1. Annotate a submission to score it +2. Batch-update submission statuses +3. Fetch the submission bundle for a given submission +4. Allow cancellation of submissions +5. Delete submissions +""" + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionStatus, SubmissionBundle + +syn = Synapse() +syn.login() + +# REQUIRED: Set these to your actual Synapse IDs +# Do NOT leave these as None - the script will not work properly +EVALUATION_ID = None # Replace with the evaluation queue ID you manage +SUBMISSION_ID = None # Replace with a submission ID from your evaluation + +assert EVALUATION_ID is not None, "EVALUATION_ID must be set to the evaluation queue ID you manage" +assert SUBMISSION_ID is not None, "SUBMISSION_ID must be set to a submission ID from your evaluation" + +print(f"Working with Evaluation: {EVALUATION_ID}") +print(f"Managing Submission: {SUBMISSION_ID}") + +# ============================================================================== +# 1. Annotate a submission to score it +# ============================================================================== + +print("\n=== 1. Annotating a submission with scores ===") + +# First, get the submission status +status = SubmissionStatus(id=SUBMISSION_ID).get() +print(f"Retrieved submission status for submission {SUBMISSION_ID}") +print(f"Current status: {status.status}") + +# Update the submission status with scoring information +status.status = "SCORED" +status.submission_annotations = { + "accuracy": [0.85], + "precision": [0.82], + "feedback": ["Good performance!"], + "validation_errors": "None detected", + "score_errors": "None detected" +} + +# Store the updated status +updated_status = status.store() +print(f"Successfully scored submission!") +print(f"Status: {updated_status.status}") +print(f"Annotations added:") +for key, value in updated_status.submission_annotations.items(): + print(f" {key}: {value}") + +# ============================================================================== +# 2. Batch-update submission statuses +# ============================================================================== + +print("\n=== 2. Batch updating submission statuses ===") + +# First, get all submission statuses that need updating +statuses_to_update = SubmissionStatus.get_all_submission_statuses( + evaluation_id=EVALUATION_ID, + status="RECEIVED", # Get submissions that haven't been scored yet + limit=50 # Limit to 50 for this example (max is 500 for batch operations) +) + +print(f"Found {len(statuses_to_update)} submissions to batch update") + +if statuses_to_update: + # Update each status with validation information + for i, status in enumerate(statuses_to_update): + status.status = "VALIDATED" + status.submission_annotations = { + "validation_status": ["PASSED"], + "validation_timestamp": ["2024-11-24T10:30:00Z"], + "batch_number": [i + 1], + "validator": ["automated_system"] + } + + # Perform batch update + try: + batch_response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses_to_update, + is_first_batch=True, + is_last_batch=True + ) + + print(f"Batch update completed successfully!") + print(f"Batch response: {batch_response}") + + except Exception as e: + print(f"Batch update failed: {e}") + print("This may be due to permissions or invalid submission states") +else: + print("No submissions found with 'RECEIVED' status to update") + +# ============================================================================== +# 3. Fetch the submission bundle for a given submission +# ============================================================================== + +print("\n=== 3. Fetching submission bundle ===") + +# Get all submission bundles for the evaluation +print("Fetching all submission bundles for the evaluation...") + +try: + bundles = list(SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="SCORED" # Only get scored submissions + )) + + print(f"Found {len(bundles)} scored submission bundles") + + for i, bundle in enumerate(bundles[:5]): # Show first 5 + submission = bundle.submission + status = bundle.submission_status + + print(f"\nBundle {i + 1}:") + if submission: + print(f" Submission ID: {submission.id}") + print(f" Submitter: {submission.submitter_alias}") + print(f" Entity ID: {submission.entity_id}") + print(f" Created: {submission.created_on}") + + if status: + print(f" Status: {status.status}") + print(f" Modified: {status.modified_on}") + if status.submission_annotations: + print(f" Scores:") + for key, value in status.submission_annotations.items(): + if key in ['accuracy', 'f1_score', 'precision', 'recall']: + print(f" {key}: {value}") + +except Exception as e: + print(f"Could not fetch submission bundles: {e}") + print("This requires READ_PRIVATE_SUBMISSION permissions on the evaluation") + +# ============================================================================== +# 4. Allow cancellation of submissions +# ============================================================================== + +print("\n=== 4. Managing submission cancellation ===") + +# First, check if any submissions have requested cancellation +try: + all_statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=EVALUATION_ID, + limit=100 + ) + + cancellation_requests = [ + status for status in all_statuses + if status.cancel_requested + ] + + print(f"Found {len(cancellation_requests)} submissions with cancellation requests") + + # Process cancellation requests + for status in cancellation_requests: + print(f"Processing cancellation request for submission {status.id}") + + # Update to allow cancellation (organizer decision) + status.can_cancel = True + status.status = "CANCELLED" + status.submission_annotations.update({ + "cancellation_reason": ["User requested cancellation"], + "cancelled_by": ["organizer"], + "cancellation_date": ["2024-11-24"] + }) + + # Store the update + try: + updated_status = status.store() + print(f" Approved cancellation for submission {updated_status.id}") + except Exception as e: + print(f" Could not process cancellation: {e}") + +except Exception as e: + print(f"Could not check for cancellation requests: {e}") + +# Example: Proactively allow cancellation for a specific submission +print("\nEnabling cancellation for a specific submission...") +try: + target_status = SubmissionStatus(id=SUBMISSION_ID).get() + target_status.can_cancel = True + target_status = target_status.store() + print(f"Cancellation enabled for submission {SUBMISSION_ID}") +except Exception as e: + print(f"Could not enable cancellation: {e}") + +# ============================================================================== +# 5. Delete submissions +# ============================================================================== + +# print("\n=== 5. Deleting submissions ===") +# print("Finding and deleting submissions that have been requested for cancellation...") + +# # Get all submission statuses to check for cancellation requests +# all_statuses = SubmissionStatus.get_all_submission_statuses( +# evaluation_id=EVALUATION_ID, +# ) + +# # Find submissions that have been requested for cancellation +# submissions_to_delete = [] +# for status in all_statuses: +# if status.cancel_requested: +# submissions_to_delete.append(status.id) + +# print(f"Found {len(submissions_to_delete)} submissions with cancellation requests") + +# # Delete each submission that was requested for cancellation +# for submission_id in submissions_to_delete: +# submission = Submission(id=submission_id).get() +# submission.delete() +# print(f"Successfully deleted submission {submission_id}") + +# if submissions_to_delete: +# print(f"Completed deletion of {len(submissions_to_delete)} requested submissions") + +print(f"\nDeletion step is commented out by default.") +print(f"Uncomment the deletion code if you want to test this functionality.") + +print(f"\n=== Organizer tutorial completed! ===") diff --git a/docs/tutorials/python/tutorial_scripts/submission_participant.py b/docs/tutorials/python/tutorial_scripts/submission_participant.py new file mode 100644 index 000000000..d9e465f9f --- /dev/null +++ b/docs/tutorials/python/tutorial_scripts/submission_participant.py @@ -0,0 +1,156 @@ +""" +Submission Participant Tutorial - Code for working with Submissions as a challenge participant. + +This tutorial demonstrates how to: +1. Make a submission to an existing evaluation queue +2. Fetch your existing submission +3. Count your submissions +4. Fetch all of your submissions from an evaluation queue +5. Check the status of your submission +6. Cancel your submission +""" + +from synapseclient import Synapse +from synapseclient.models import Submission, SubmissionStatus + +syn = Synapse() +syn.login() + +# REQUIRED: Set these to your actual Synapse IDs +# Do NOT leave these as None - the script will not work properly +EVALUATION_ID = "9617645" # Replace with the evaluation queue ID you want to submit to +ENTITY_ID = "syn59211514" # Replace with the entity ID you want to submit + +assert EVALUATION_ID is not None, "EVALUATION_ID must be set to the evaluation queue ID" +assert ENTITY_ID is not None, "ENTITY_ID must be set to the entity ID you want to submit" + +print(f"Working with Evaluation: {EVALUATION_ID}") +print(f"Submitting Entity: {ENTITY_ID}") + +# ============================================================================== +# 1. Make a submission to an existing evaluation queue on Synapse +# ============================================================================== + +print("\n=== 1. Making a submission ===") + +# Create a new Submission object +submission = Submission( + entity_id=ENTITY_ID, + evaluation_id=EVALUATION_ID, + name="My Tutorial Submission" +) + +# Submit the entity to the evaluation queue +submission = submission.store() + +print(f"Submission created successfully!") +print(f"Submission ID: {submission.id}") +print(f"Submitted Entity: {submission.entity_id}") +print(f"Evaluation: {submission.evaluation_id}") +print(f"Submission Name: {submission.name}") +print(f"Created On: {submission.created_on}") + +# Store the submission ID for later use +submission_id = submission.id + +# ============================================================================== +# 2. Fetch your existing submission +# ============================================================================== + +print("\n=== 2. Fetching existing submission ===") + +# Retrieve the submission we just created +retrieved_submission = Submission(id=submission_id).get() + +print(f"Retrieved submission:") +print(f" ID: {retrieved_submission.id}") +print(f" Name: {retrieved_submission.name}") +print(f" Entity ID: {retrieved_submission.entity_id}") +print(f" Submitter: {retrieved_submission.submitter_alias}") +print(f" Created On: {retrieved_submission.created_on}") + +# ============================================================================== +# 3. Count your submissions +# ============================================================================== + +print("\n=== 3. Counting submissions ===") + +# Get the total count of submissions for this evaluation +submission_count = Submission.get_submission_count( + evaluation_id=EVALUATION_ID +) + +print(f"Total submissions in evaluation: {submission_count}") + +# Get count of submissions with specific status (optional) +scored_count = Submission.get_submission_count( + evaluation_id=EVALUATION_ID, + status="SCORED" +) + +print(f"SCORED submissions in evaluation: {scored_count}") + +# ============================================================================== +# 4. Fetch all of your submissions from an existing evaluation queue +# ============================================================================== + +print("\n=== 4. Fetching all your submissions ===") + +# Get all of your submissions for this evaluation +user_submissions = list(Submission.get_user_submissions( + evaluation_id=EVALUATION_ID +)) + +print(f"Found {len(user_submissions)} submissions from the current user:") +for i, sub in enumerate(user_submissions, 1): + print(f" {i}. ID: {sub.id}, Name: {sub.name}, Created: {sub.created_on}") + +# ============================================================================== +# 5. Check the status of your submission +# ============================================================================== + +print("\n=== 5. Checking submission status ===") + +# Fetch the status of our submission +status = SubmissionStatus(id=submission_id).get() + +print(f"Submission status details:") +print(f" Status: {status.status}") +print(f" Modified On: {status.modified_on}") +print(f" Can Cancel: {status.can_cancel}") +print(f" Cancel Requested: {status.cancel_requested}") + +# Check if there are any submission annotations (scores, feedback, etc.) +if status.submission_annotations: + print(f" Submission Annotations:") + for key, value in status.submission_annotations.items(): + print(f" {key}: {value}") +else: + print(f" No submission annotations available") + +# ============================================================================== +# 6. Cancel your submission (optional) +# ============================================================================== + +print("\n=== 6. Cancelling submission ===") + +# Note: Only cancel if the submission allows it +# Uncomment the following lines if you want to test cancellation: + +# try: +# cancelled_submission = submission.cancel() +# print(f"Submission {cancelled_submission.id} has been requested for cancellation") +# +# # Check the updated status +# updated_status = SubmissionStatus(id=submission_id).get() +# print(f"Cancel requested: {updated_status.cancel_requested}") +# except Exception as e: +# print(f"Could not cancel submission: {e}") +# print("This may be because cancellation is not allowed for this submission") + +print(f"\nCancellation is commented out by default.") +print(f"Uncomment the cancellation code if you want to test this functionality.") + +print(f"\n=== Tutorial completed! ===") +print(f"Your submission ID {submission_id} is ready for evaluation.") +print(f"Check back later to see if the organizers have scored your submission.") From e07206dd06f7b4241d91a2700816d0e2dec6fd6e Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 10:29:54 -0500 Subject: [PATCH 43/46] remove try/excepts. add line references. add resources and source code sections. --- docs/tutorials/python/submission.md | 65 +++++++- .../tutorial_scripts/submission_organizer.py | 146 ++++++++---------- .../submission_participant.py | 16 +- 3 files changed, 133 insertions(+), 94 deletions(-) diff --git a/docs/tutorials/python/submission.md b/docs/tutorials/python/submission.md index dd0a98225..9abf6037d 100644 --- a/docs/tutorials/python/submission.md +++ b/docs/tutorials/python/submission.md @@ -42,27 +42,90 @@ As an organizer of a Synapse challenge, you will ### 1. Make a submission to an existing evaluation queue on Synapse ```python -{!docs/tutorials/python/tutorial_scripts/submission.py!lines=5-46} +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=30-51} ``` ### 2. Fetch your existing submission +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=56-71} +``` + ### 3. Count your submissions +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=72-92} +``` + ### 4. Fetch all of your submissions from an existing evaluation queue on Synapse +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=93-107} +``` + ### 5. Check the status of your submission +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=108-130} +``` + ### 6. Cancel your submission +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!lines=131-157} +``` + ## 2. Organizing a Synapse challenge ### 1. Annotate a submission to score it +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=29-57} +``` + ### 2. Batch-update submission statuses +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=58-101} +``` + ### 3. Fetch the submission bundle for a given submission +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=102-142} +``` + ### 4. Allow cancellation of submissions +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=143-195} +``` + ### 5. Delete submissions + +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!lines=196-229} +``` + +## Source code for this tutorial + +
+ Click to show me (source code for Participant) + +```python +{!docs/tutorials/python/tutorial_scripts/submission_participant.py!} +``` +
+ +
+ Click to show me (source code for Organizer) + +```python +{!docs/tutorials/python/tutorial_scripts/submission_organizer.py!} +``` +
+ +## References +- [Evaluation][synapseclient.models.Evaluation] +- [Project][synapseclient.models.Project] +- [syn.login][synapseclient.Synapse.login] diff --git a/docs/tutorials/python/tutorial_scripts/submission_organizer.py b/docs/tutorials/python/tutorial_scripts/submission_organizer.py index 94941c7eb..4b9ffc418 100644 --- a/docs/tutorials/python/tutorial_scripts/submission_organizer.py +++ b/docs/tutorials/python/tutorial_scripts/submission_organizer.py @@ -82,20 +82,15 @@ } # Perform batch update - try: - batch_response = SubmissionStatus.batch_update_submission_statuses( - evaluation_id=EVALUATION_ID, - statuses=statuses_to_update, - is_first_batch=True, - is_last_batch=True - ) - - print(f"Batch update completed successfully!") - print(f"Batch response: {batch_response}") - - except Exception as e: - print(f"Batch update failed: {e}") - print("This may be due to permissions or invalid submission states") + batch_response = SubmissionStatus.batch_update_submission_statuses( + evaluation_id=EVALUATION_ID, + statuses=statuses_to_update, + is_first_batch=True, + is_last_batch=True + ) + + print(f"Batch update completed successfully!") + print(f"Batch response: {batch_response}") else: print("No submissions found with 'RECEIVED' status to update") @@ -108,37 +103,32 @@ # Get all submission bundles for the evaluation print("Fetching all submission bundles for the evaluation...") -try: - bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - status="SCORED" # Only get scored submissions - )) +bundles = list(SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, + status="SCORED" # Only get scored submissions +)) + +print(f"Found {len(bundles)} scored submission bundles") + +for i, bundle in enumerate(bundles[:5]): # Show first 5 + submission = bundle.submission + status = bundle.submission_status - print(f"Found {len(bundles)} scored submission bundles") + print(f"\nBundle {i + 1}:") + if submission: + print(f" Submission ID: {submission.id}") + print(f" Submitter: {submission.submitter_alias}") + print(f" Entity ID: {submission.entity_id}") + print(f" Created: {submission.created_on}") - for i, bundle in enumerate(bundles[:5]): # Show first 5 - submission = bundle.submission - status = bundle.submission_status - - print(f"\nBundle {i + 1}:") - if submission: - print(f" Submission ID: {submission.id}") - print(f" Submitter: {submission.submitter_alias}") - print(f" Entity ID: {submission.entity_id}") - print(f" Created: {submission.created_on}") - - if status: - print(f" Status: {status.status}") - print(f" Modified: {status.modified_on}") - if status.submission_annotations: - print(f" Scores:") - for key, value in status.submission_annotations.items(): - if key in ['accuracy', 'f1_score', 'precision', 'recall']: - print(f" {key}: {value}") - -except Exception as e: - print(f"Could not fetch submission bundles: {e}") - print("This requires READ_PRIVATE_SUBMISSION permissions on the evaluation") + if status: + print(f" Status: {status.status}") + print(f" Modified: {status.modified_on}") + if status.submission_annotations: + print(f" Scores:") + for key, value in status.submission_annotations.items(): + if key in ['accuracy', 'f1_score', 'precision', 'recall']: + print(f" {key}: {value}") # ============================================================================== # 4. Allow cancellation of submissions @@ -147,51 +137,41 @@ print("\n=== 4. Managing submission cancellation ===") # First, check if any submissions have requested cancellation -try: - all_statuses = SubmissionStatus.get_all_submission_statuses( - evaluation_id=EVALUATION_ID, - limit=100 - ) - - cancellation_requests = [ - status for status in all_statuses - if status.cancel_requested - ] +all_statuses = SubmissionStatus.get_all_submission_statuses( + evaluation_id=EVALUATION_ID, + limit=100 +) + +cancellation_requests = [ + status for status in all_statuses + if status.cancel_requested +] + +print(f"Found {len(cancellation_requests)} submissions with cancellation requests") + +# Process cancellation requests +for status in cancellation_requests: + print(f"Processing cancellation request for submission {status.id}") - print(f"Found {len(cancellation_requests)} submissions with cancellation requests") + # Update to allow cancellation (organizer decision) + status.can_cancel = True + status.status = "CANCELLED" + status.submission_annotations.update({ + "cancellation_reason": ["User requested cancellation"], + "cancelled_by": ["organizer"], + "cancellation_date": ["2024-11-24"] + }) - # Process cancellation requests - for status in cancellation_requests: - print(f"Processing cancellation request for submission {status.id}") - - # Update to allow cancellation (organizer decision) - status.can_cancel = True - status.status = "CANCELLED" - status.submission_annotations.update({ - "cancellation_reason": ["User requested cancellation"], - "cancelled_by": ["organizer"], - "cancellation_date": ["2024-11-24"] - }) - - # Store the update - try: - updated_status = status.store() - print(f" Approved cancellation for submission {updated_status.id}") - except Exception as e: - print(f" Could not process cancellation: {e}") - -except Exception as e: - print(f"Could not check for cancellation requests: {e}") + # Store the update + updated_status = status.store() + print(f" Approved cancellation for submission {updated_status.id}") # Example: Proactively allow cancellation for a specific submission print("\nEnabling cancellation for a specific submission...") -try: - target_status = SubmissionStatus(id=SUBMISSION_ID).get() - target_status.can_cancel = True - target_status = target_status.store() - print(f"Cancellation enabled for submission {SUBMISSION_ID}") -except Exception as e: - print(f"Could not enable cancellation: {e}") +target_status = SubmissionStatus(id=SUBMISSION_ID).get() +target_status.can_cancel = True +target_status = target_status.store() +print(f"Cancellation enabled for submission {SUBMISSION_ID}") # ============================================================================== # 5. Delete submissions diff --git a/docs/tutorials/python/tutorial_scripts/submission_participant.py b/docs/tutorials/python/tutorial_scripts/submission_participant.py index d9e465f9f..292ce7a75 100644 --- a/docs/tutorials/python/tutorial_scripts/submission_participant.py +++ b/docs/tutorials/python/tutorial_scripts/submission_participant.py @@ -137,16 +137,12 @@ # Note: Only cancel if the submission allows it # Uncomment the following lines if you want to test cancellation: -# try: -# cancelled_submission = submission.cancel() -# print(f"Submission {cancelled_submission.id} has been requested for cancellation") -# -# # Check the updated status -# updated_status = SubmissionStatus(id=submission_id).get() -# print(f"Cancel requested: {updated_status.cancel_requested}") -# except Exception as e: -# print(f"Could not cancel submission: {e}") -# print("This may be because cancellation is not allowed for this submission") +# cancelled_submission = submission.cancel() +# print(f"Submission {cancelled_submission.id} has been requested for cancellation") +# +# # Check the updated status +# updated_status = SubmissionStatus(id=submission_id).get() +# print(f"Cancel requested: {updated_status.cancel_requested}") print(f"\nCancellation is commented out by default.") print(f"Uncomment the cancellation code if you want to test this functionality.") From e1937536ff57ba1b77005e356a9c24b27ff775fc Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 10:42:13 -0500 Subject: [PATCH 44/46] add reference to File model --- docs/tutorials/python/submission.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/python/submission.md b/docs/tutorials/python/submission.md index 9abf6037d..a5f999ba7 100644 --- a/docs/tutorials/python/submission.md +++ b/docs/tutorials/python/submission.md @@ -34,7 +34,7 @@ As an organizer of a Synapse challenge, you will ## Prerequisites * You have completed the [Evaluation](./evaluation.md) tutorial, or have an existing Evaluation on Synapse to work from -* You have an existing entity with which to make a submission (can be a File or Docker Repository) +* You have an existing entity with which to make a submission (can be a [File](./file.md) or Docker Repository) * You have the correct permissions on the Evaluation queue for your desired tutorial section (participant or organizer) ## 1. Participating in a Synapse challenge @@ -127,5 +127,5 @@ As an organizer of a Synapse challenge, you will ## References - [Evaluation][synapseclient.models.Evaluation] -- [Project][synapseclient.models.Project] +- [File][synapseclient.models.File] - [syn.login][synapseclient.Synapse.login] From b710da06fb83dfb11636d5f1d0a960e7b5c9673b Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 10:58:56 -0500 Subject: [PATCH 45/46] deprecate old submission model --- synapseclient/evaluation.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/synapseclient/evaluation.py b/synapseclient/evaluation.py index 507896543..8a84e2aec 100644 --- a/synapseclient/evaluation.py +++ b/synapseclient/evaluation.py @@ -208,10 +208,17 @@ def putACLURI(self): return "/evaluation/acl" +@deprecated( + version="4.11.0", + reason="To be removed in 5.0.0. " + "Use the Submission model from synapseclient.models.submission instead.", +) class Submission(DictObject): """ Builds a Synapse submission object. + WARNING - This class is deprecated and will no longer be maintained. Please use the Submission model from synapseclient.models.submission instead. + Arguments: name: Name of submission entityId: Synapse ID of the Entity to submit From 79170110f78847b6c1346f1f086c1cdf3a9ee518 Mon Sep 17 00:00:00 2001 From: Jenny Medina Date: Mon, 1 Dec 2025 11:00:30 -0500 Subject: [PATCH 46/46] style --- .../tutorial_scripts/submission_organizer.py | 63 ++++++++++--------- .../submission_participant.py | 23 +++---- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/docs/tutorials/python/tutorial_scripts/submission_organizer.py b/docs/tutorials/python/tutorial_scripts/submission_organizer.py index 4b9ffc418..a6aa303dd 100644 --- a/docs/tutorials/python/tutorial_scripts/submission_organizer.py +++ b/docs/tutorials/python/tutorial_scripts/submission_organizer.py @@ -10,7 +10,7 @@ """ from synapseclient import Synapse -from synapseclient.models import Submission, SubmissionStatus, SubmissionBundle +from synapseclient.models import Submission, SubmissionBundle, SubmissionStatus syn = Synapse() syn.login() @@ -20,8 +20,12 @@ EVALUATION_ID = None # Replace with the evaluation queue ID you manage SUBMISSION_ID = None # Replace with a submission ID from your evaluation -assert EVALUATION_ID is not None, "EVALUATION_ID must be set to the evaluation queue ID you manage" -assert SUBMISSION_ID is not None, "SUBMISSION_ID must be set to a submission ID from your evaluation" +assert ( + EVALUATION_ID is not None +), "EVALUATION_ID must be set to the evaluation queue ID you manage" +assert ( + SUBMISSION_ID is not None +), "SUBMISSION_ID must be set to a submission ID from your evaluation" print(f"Working with Evaluation: {EVALUATION_ID}") print(f"Managing Submission: {SUBMISSION_ID}") @@ -41,10 +45,10 @@ status.status = "SCORED" status.submission_annotations = { "accuracy": [0.85], - "precision": [0.82], + "precision": [0.82], "feedback": ["Good performance!"], "validation_errors": "None detected", - "score_errors": "None detected" + "score_errors": "None detected", } # Store the updated status @@ -65,7 +69,7 @@ statuses_to_update = SubmissionStatus.get_all_submission_statuses( evaluation_id=EVALUATION_ID, status="RECEIVED", # Get submissions that haven't been scored yet - limit=50 # Limit to 50 for this example (max is 500 for batch operations) + limit=50, # Limit to 50 for this example (max is 500 for batch operations) ) print(f"Found {len(statuses_to_update)} submissions to batch update") @@ -78,17 +82,17 @@ "validation_status": ["PASSED"], "validation_timestamp": ["2024-11-24T10:30:00Z"], "batch_number": [i + 1], - "validator": ["automated_system"] + "validator": ["automated_system"], } - + # Perform batch update batch_response = SubmissionStatus.batch_update_submission_statuses( evaluation_id=EVALUATION_ID, statuses=statuses_to_update, is_first_batch=True, - is_last_batch=True + is_last_batch=True, ) - + print(f"Batch update completed successfully!") print(f"Batch response: {batch_response}") else: @@ -103,31 +107,32 @@ # Get all submission bundles for the evaluation print("Fetching all submission bundles for the evaluation...") -bundles = list(SubmissionBundle.get_evaluation_submission_bundles( - evaluation_id=EVALUATION_ID, - status="SCORED" # Only get scored submissions -)) +bundles = list( + SubmissionBundle.get_evaluation_submission_bundles( + evaluation_id=EVALUATION_ID, status="SCORED" # Only get scored submissions + ) +) print(f"Found {len(bundles)} scored submission bundles") for i, bundle in enumerate(bundles[:5]): # Show first 5 submission = bundle.submission status = bundle.submission_status - + print(f"\nBundle {i + 1}:") if submission: print(f" Submission ID: {submission.id}") print(f" Submitter: {submission.submitter_alias}") print(f" Entity ID: {submission.entity_id}") print(f" Created: {submission.created_on}") - + if status: print(f" Status: {status.status}") print(f" Modified: {status.modified_on}") if status.submission_annotations: print(f" Scores:") for key, value in status.submission_annotations.items(): - if key in ['accuracy', 'f1_score', 'precision', 'recall']: + if key in ["accuracy", "f1_score", "precision", "recall"]: print(f" {key}: {value}") # ============================================================================== @@ -138,30 +143,28 @@ # First, check if any submissions have requested cancellation all_statuses = SubmissionStatus.get_all_submission_statuses( - evaluation_id=EVALUATION_ID, - limit=100 + evaluation_id=EVALUATION_ID, limit=100 ) -cancellation_requests = [ - status for status in all_statuses - if status.cancel_requested -] +cancellation_requests = [status for status in all_statuses if status.cancel_requested] print(f"Found {len(cancellation_requests)} submissions with cancellation requests") # Process cancellation requests for status in cancellation_requests: print(f"Processing cancellation request for submission {status.id}") - + # Update to allow cancellation (organizer decision) status.can_cancel = True status.status = "CANCELLED" - status.submission_annotations.update({ - "cancellation_reason": ["User requested cancellation"], - "cancelled_by": ["organizer"], - "cancellation_date": ["2024-11-24"] - }) - + status.submission_annotations.update( + { + "cancellation_reason": ["User requested cancellation"], + "cancelled_by": ["organizer"], + "cancellation_date": ["2024-11-24"], + } + ) + # Store the update updated_status = status.store() print(f" Approved cancellation for submission {updated_status.id}") diff --git a/docs/tutorials/python/tutorial_scripts/submission_participant.py b/docs/tutorials/python/tutorial_scripts/submission_participant.py index 292ce7a75..83fb2b0d2 100644 --- a/docs/tutorials/python/tutorial_scripts/submission_participant.py +++ b/docs/tutorials/python/tutorial_scripts/submission_participant.py @@ -4,7 +4,7 @@ This tutorial demonstrates how to: 1. Make a submission to an existing evaluation queue 2. Fetch your existing submission -3. Count your submissions +3. Count your submissions 4. Fetch all of your submissions from an evaluation queue 5. Check the status of your submission 6. Cancel your submission @@ -22,7 +22,9 @@ ENTITY_ID = "syn59211514" # Replace with the entity ID you want to submit assert EVALUATION_ID is not None, "EVALUATION_ID must be set to the evaluation queue ID" -assert ENTITY_ID is not None, "ENTITY_ID must be set to the entity ID you want to submit" +assert ( + ENTITY_ID is not None +), "ENTITY_ID must be set to the entity ID you want to submit" print(f"Working with Evaluation: {EVALUATION_ID}") print(f"Submitting Entity: {ENTITY_ID}") @@ -35,9 +37,7 @@ # Create a new Submission object submission = Submission( - entity_id=ENTITY_ID, - evaluation_id=EVALUATION_ID, - name="My Tutorial Submission" + entity_id=ENTITY_ID, evaluation_id=EVALUATION_ID, name="My Tutorial Submission" ) # Submit the entity to the evaluation queue @@ -76,16 +76,13 @@ print("\n=== 3. Counting submissions ===") # Get the total count of submissions for this evaluation -submission_count = Submission.get_submission_count( - evaluation_id=EVALUATION_ID -) +submission_count = Submission.get_submission_count(evaluation_id=EVALUATION_ID) print(f"Total submissions in evaluation: {submission_count}") # Get count of submissions with specific status (optional) scored_count = Submission.get_submission_count( - evaluation_id=EVALUATION_ID, - status="SCORED" + evaluation_id=EVALUATION_ID, status="SCORED" ) print(f"SCORED submissions in evaluation: {scored_count}") @@ -97,9 +94,7 @@ print("\n=== 4. Fetching all your submissions ===") # Get all of your submissions for this evaluation -user_submissions = list(Submission.get_user_submissions( - evaluation_id=EVALUATION_ID -)) +user_submissions = list(Submission.get_user_submissions(evaluation_id=EVALUATION_ID)) print(f"Found {len(user_submissions)} submissions from the current user:") for i, sub in enumerate(user_submissions, 1): @@ -139,7 +134,7 @@ # cancelled_submission = submission.cancel() # print(f"Submission {cancelled_submission.id} has been requested for cancellation") -# +# # # Check the updated status # updated_status = SubmissionStatus(id=submission_id).get() # print(f"Cancel requested: {updated_status.cancel_requested}")