From 168b5f0cd9cee5a7639aced6ea727f76d5422b6d Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 27 Oct 2025 12:06:32 +0100 Subject: [PATCH 01/46] Disallow non-ASCII in email domain --- server/mergin/auth/forms.py | 20 +++++++++++++++++--- server/mergin/tests/test_auth.py | 1 + 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index b45638eb..99a4232e 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -48,9 +48,12 @@ class ExtendedEmail(Email): 1. spaces, 2. special characters ,:;()<>[]\" 3, multiple @ symbols, - 4, leading, trailing, or consecutive dots in the local part - 5, invalid domain part - missing top level domain (user@example), consecutive dots - Custom check for additional invalid characters disallows |'— because they make our email sending service to fail + 4, leading, trailing, or consecutive dots in the local part, + 5, invalid domain part - missing top level domain (user@example), consecutive dots, + Custom check for + - additional invalid characters disallows |'— + - non-ASCII characters in the domain part + because they make our email sending service to fail """ def __call__(self, form, field): @@ -61,6 +64,17 @@ def __call__(self, form, field): f"Email address '{field.data}' contains an invalid character." ) + try: + local_part, domain_part = field.data.rsplit("@", 1) + except ValueError: + raise ValidationError(f"Invalid email address '{field.data}'.") + + # character is one of the standard ASCII characters (0–127) + if not all(ord(c) < 128 for c in domain_part): + raise ValidationError( + f"Email address '{field.data}' contains non-ASCII characters in the domain part." + ) + class PasswordValidator: def __init__(self, min_length=8): diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index d53b01bc..b953577d 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -128,6 +128,7 @@ def test_logout(client): ("verylonglonglonglonglonglonglongemail@example.com", "#pwd1234", 201), ("us.er@mergin.com", "#pwd1234", 201), # dot is allowed ("us er@mergin.com", "#pwd1234", 400), # space is disallowed + ("test@gmaiñ.com", "#pwd1234", 400), # non-ASCII character in the domain ] From c118ca94cf3d5a89405283f0460d362ec9ec560d Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 30 Oct 2025 16:37:27 +0100 Subject: [PATCH 02/46] Create endpoint - new marshmallow schema - controller function --- server/mergin/sync/files.py | 2 +- server/mergin/sync/public_api_v2.yaml | 106 ++++++++++++++++++ .../mergin/sync/public_api_v2_controller.py | 20 +++- server/mergin/sync/schemas.py | 37 ++++++ 4 files changed, 162 insertions(+), 3 deletions(-) diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index 12b30afe..9f2011f2 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -122,5 +122,5 @@ class ProjectFileSchema(FileSchema): def patch_field(self, data, **kwargs): # drop 'diff' key entirely if empty or None as clients would expect if not data.get("diff"): - data.pop("diff") + data.pop("diff", None) return data diff --git a/server/mergin/sync/public_api_v2.yaml b/server/mergin/sync/public_api_v2.yaml index 04dbce61..6a277b52 100644 --- a/server/mergin/sync/public_api_v2.yaml +++ b/server/mergin/sync/public_api_v2.yaml @@ -76,6 +76,35 @@ paths: "409": $ref: "#/components/responses/Conflict" x-openapi-router-controller: mergin.sync.public_api_v2_controller + get: + tags: + - project + summary: Get project info + operationId: get_project + parameters: + - name: files_at_version + in: query + description: Include list of files at specific version + required: false + schema: + type: string + example: v3 + responses: + "200": + description: Success + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectDetail" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + x-openapi-router-controller: mergin.sync.public_api_v2_controller /projects/{id}/scheduleDelete: post: tags: @@ -287,3 +316,80 @@ components: $ref: "#/components/schemas/ProjectRole" role: $ref: "#/components/schemas/Role" + ProjectDetail: + type: object + required: + - id + - name + - workspace + - role + - version + - created_at + - updated_at + - public + - size + properties: + id: + type: string + description: project uuid + example: c1ae6439-0056-42df-a06d-79cc430dd7df + name: + type: string + example: survey + workspace: + type: object + properties: + id: + type: integer + example: 123 + name: + type: string + example: mergin + role: + $ref: "#/components/schemas/ProjectRole" + version: + type: string + description: latest project version + example: v2 + created_at: + type: string + format: date-time + description: project creation timestamp + example: 2025-10-24T08:27:56Z + updated_at: + type: string + format: date-time + description: last project update timestamp + example: 2025-10-24T08:28:00.279699Z + public: + type: boolean + description: whether the project is public + example: false + size: + type: integer + description: project size in bytes for this version + example: 17092380 + files: + type: array + description: List of files in the project + items: + type: object + properties: + path: + type: string + description: File name including path from project root + example: data/layer.gpkg + mtime: + type: string + format: date-time + description: File modification timestamp + example: 2024-11-19T13:50:00Z + size: + type: integer + description: File size in bytes + example: 1234 + checksum: + type: string + description: File checksum hash + example: 9adb76bf81a34880209040ffe5ee262a090b62ab + diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 7f40c54b..698ce390 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -8,13 +8,14 @@ from flask_login import current_user from mergin.sync.forms import project_name_validation +from .files import ProjectFileSchema -from .schemas import ProjectMemberSchema +from .schemas import ProjectMemberSchema, ProjectSchemaV2 from .workspace import WorkspaceRole from ..app import db from ..auth import auth_required from ..auth.models import User -from .models import Project, ProjectRole, ProjectMember +from .models import Project, ProjectRole, ProjectMember, ProjectVersion from .permissions import ProjectPermissions, require_project_by_uuid @@ -128,3 +129,18 @@ def remove_project_collaborator(id, user_id): project.unset_role(user_id) db.session.commit() return NoContent, 204 + + +def get_project(id, files_at_version=None): + """Get project info. Include list of files at specific version if requested.""" + project = require_project_by_uuid(id, ProjectPermissions.Read) + data = ProjectSchemaV2().dump(project) + if files_at_version: + pv = ProjectVersion.query.filter_by( + project_id=project.id, name=ProjectVersion.from_v_name(files_at_version) + ).first_or_404() + data["files"] = ProjectFileSchema( + only=("path", "mtime", "size", "checksum"), many=True + ).dump(pv.files) + + return data, 200 diff --git a/server/mergin/sync/schemas.py b/server/mergin/sync/schemas.py index 75b6f09e..69ead45e 100644 --- a/server/mergin/sync/schemas.py +++ b/server/mergin/sync/schemas.py @@ -405,3 +405,40 @@ class ProjectMemberSchema(Schema): project_role = fields.Enum(enum=ProjectRole, by_value=True) workspace_role = fields.Enum(enum=WorkspaceRole, by_value=True) role = fields.Enum(enum=ProjectRole, by_value=True) + + +class ProjectSchemaV2(ma.SQLAlchemyAutoSchema): + id = fields.UUID() + name = fields.String() + version = fields.Function(lambda obj: ProjectVersion.to_v_name(obj.latest_version)) + public = fields.Boolean() + size = fields.Integer(attribute="disk_usage") + + created_at = DateTimeWithZ(attribute="created") + updated_at = DateTimeWithZ(attribute="updated") + + workspace = fields.Function( + lambda obj: {"id": obj.workspace.id, "name": obj.workspace.name} + ) + role = fields.Method("_role") + + def _role(self, obj): + role = ProjectPermissions.get_user_project_role(obj, current_user) + if not role: + return None + return role.value + + class Meta: + model = Project + load_instance = True + fields = ( + "id", + "name", + "version", + "public", + "size", + "created_at", + "updated_at", + "workspace", + "role", + ) From 6e9f1774f78b284e3c4cd615cc8f4e9833af41e8 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 31 Oct 2025 15:41:57 +0100 Subject: [PATCH 03/46] add tests --- .../mergin/sync/public_api_v2_controller.py | 5 +- server/mergin/sync/schemas.py | 37 --------- server/mergin/sync/schemas_v2.py | 50 ++++++++++++ server/mergin/tests/test_public_api_v2.py | 79 ++++++++++++++++++- 4 files changed, 131 insertions(+), 40 deletions(-) create mode 100644 server/mergin/sync/schemas_v2.py diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 698ce390..96b7f914 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -10,7 +10,8 @@ from mergin.sync.forms import project_name_validation from .files import ProjectFileSchema -from .schemas import ProjectMemberSchema, ProjectSchemaV2 +from .schemas import ProjectMemberSchema +from .schemas_v2 import ProjectSchema from .workspace import WorkspaceRole from ..app import db from ..auth import auth_required @@ -134,7 +135,7 @@ def remove_project_collaborator(id, user_id): def get_project(id, files_at_version=None): """Get project info. Include list of files at specific version if requested.""" project = require_project_by_uuid(id, ProjectPermissions.Read) - data = ProjectSchemaV2().dump(project) + data = ProjectSchema().dump(project) if files_at_version: pv = ProjectVersion.query.filter_by( project_id=project.id, name=ProjectVersion.from_v_name(files_at_version) diff --git a/server/mergin/sync/schemas.py b/server/mergin/sync/schemas.py index 69ead45e..75b6f09e 100644 --- a/server/mergin/sync/schemas.py +++ b/server/mergin/sync/schemas.py @@ -405,40 +405,3 @@ class ProjectMemberSchema(Schema): project_role = fields.Enum(enum=ProjectRole, by_value=True) workspace_role = fields.Enum(enum=WorkspaceRole, by_value=True) role = fields.Enum(enum=ProjectRole, by_value=True) - - -class ProjectSchemaV2(ma.SQLAlchemyAutoSchema): - id = fields.UUID() - name = fields.String() - version = fields.Function(lambda obj: ProjectVersion.to_v_name(obj.latest_version)) - public = fields.Boolean() - size = fields.Integer(attribute="disk_usage") - - created_at = DateTimeWithZ(attribute="created") - updated_at = DateTimeWithZ(attribute="updated") - - workspace = fields.Function( - lambda obj: {"id": obj.workspace.id, "name": obj.workspace.name} - ) - role = fields.Method("_role") - - def _role(self, obj): - role = ProjectPermissions.get_user_project_role(obj, current_user) - if not role: - return None - return role.value - - class Meta: - model = Project - load_instance = True - fields = ( - "id", - "name", - "version", - "public", - "size", - "created_at", - "updated_at", - "workspace", - "role", - ) diff --git a/server/mergin/sync/schemas_v2.py b/server/mergin/sync/schemas_v2.py new file mode 100644 index 00000000..668a7e20 --- /dev/null +++ b/server/mergin/sync/schemas_v2.py @@ -0,0 +1,50 @@ +# Copyright (C) Lutra Consulting Limited +# +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + +from marshmallow import fields +from flask_login import current_user + +from ..app import DateTimeWithZ, ma +from .permissions import ProjectPermissions +from .models import ( + Project, + ProjectVersion, +) + + +class ProjectSchema(ma.SQLAlchemyAutoSchema): + id = fields.UUID() + name = fields.String() + version = fields.Function(lambda obj: ProjectVersion.to_v_name(obj.latest_version)) + public = fields.Boolean() + size = fields.Integer(attribute="disk_usage") + + created_at = DateTimeWithZ(attribute="created") + updated_at = DateTimeWithZ(attribute="updated") + + workspace = fields.Function( + lambda obj: {"id": obj.workspace.id, "name": obj.workspace.name} + ) + role = fields.Method("_role") + + def _role(self, obj): + role = ProjectPermissions.get_user_project_role(obj, current_user) + if not role: + return None + return role.value + + class Meta: + model = Project + load_instance = True + fields = ( + "id", + "name", + "version", + "public", + "size", + "created_at", + "updated_at", + "workspace", + "role", + ) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 2d88d652..f39f142b 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -1,11 +1,22 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -from .utils import add_user +from datetime import datetime + +from . import DEFAULT_USER +from .utils import ( + add_user, + login, + login_as_admin, + create_workspace, + create_project, + upload_file_to_project, +) from ..app import db from mergin.sync.models import Project from tests import test_project, test_workspace_id +from ..auth.models import User from ..config import Configuration from ..sync.models import ProjectRole @@ -126,3 +137,69 @@ def test_project_members(client): # access provided by workspace role cannot be removed directly response = client.delete(url + f"/{user.id}") assert response.status_code == 404 + + +def test_get_project(client): + """Test get project info endpoint""" + admin = User.query.filter_by(username=DEFAULT_USER[0]).first() + test_workspace = create_workspace() + project = create_project("new_project", test_workspace, admin) + add_user("test_user", "ilovemergin") + login(client, "test_user", "ilovemergin") + # lack of permissions + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 403 + # access public project + project.public = True + db.session.commit() + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 200 + assert response.json["public"] is True + # project scheduled for deletion + login_as_admin(client) + project.public = False + project.removed_at = datetime.utcnow() + db.session.commit() + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 404 + # success + project.removed_at = None + db.session.commit() + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 200 + expected_keys = { + "id", + "name", + "workspace", + "role", + "version", + "created_at", + "updated_at", + "public", + "size", + } + assert expected_keys == response.json.keys() + # create new versions + files = ["test.txt", "test3.txt", "test.qgs"] + for file in files: + upload_file_to_project(project, file, client) + # project version does not exist + response = client.get( + f"v2/projects/{project.id}?files_at_version=v{project.latest_version+1}" + ) + assert response.status_code == 404 + # files + response = client.get( + f"v2/projects/{project.id}?files_at_version=v{project.latest_version-2}" + ) + assert response.status_code == 200 + assert len(response.json["files"]) == 1 + assert any(resp_files["path"] == files[0] for resp_files in response.json["files"]) + assert not any( + resp_files["path"] == files[1] for resp_files in response.json["files"] + ) + response = client.get( + f"v2/projects/{project.id}?files_at_version=v{project.latest_version}" + ) + assert len(response.json["files"]) == 3 + assert {f["path"] for f in response.json["files"]} == set(files) From 39129f22e3f9a913ed3716132bb8f85bea96dd98 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 3 Nov 2025 09:30:51 +0100 Subject: [PATCH 04/46] Fix test helpers order to give time to manifest --- server/mergin/tests/test_public_api_v2.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index f39f142b..6272edb3 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -1,6 +1,7 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import time from datetime import datetime from . import DEFAULT_USER @@ -141,11 +142,11 @@ def test_project_members(client): def test_get_project(client): """Test get project info endpoint""" + add_user("test_user", "ilovemergin") + login(client, "test_user", "ilovemergin") admin = User.query.filter_by(username=DEFAULT_USER[0]).first() test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) - add_user("test_user", "ilovemergin") - login(client, "test_user", "ilovemergin") # lack of permissions response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 From 8635e5ee29b9517b1c0dd44e7ee283556bac1690 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 3 Nov 2025 12:08:44 +0100 Subject: [PATCH 05/46] use email_validator and isascii --- server/mergin/auth/forms.py | 10 +++------- server/mergin/auth/utils.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 server/mergin/auth/utils.py diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index 99a4232e..884d0dad 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -17,6 +17,7 @@ from .models import MAX_USERNAME_LENGTH, User from ..app import UpdateForm, CustomStringField +from .utils import get_email_domain def username_validation(form, field): @@ -64,13 +65,8 @@ def __call__(self, form, field): f"Email address '{field.data}' contains an invalid character." ) - try: - local_part, domain_part = field.data.rsplit("@", 1) - except ValueError: - raise ValidationError(f"Invalid email address '{field.data}'.") - - # character is one of the standard ASCII characters (0–127) - if not all(ord(c) < 128 for c in domain_part): + domain = get_email_domain(field.data) + if not domain.isascii(): raise ValidationError( f"Email address '{field.data}' contains non-ASCII characters in the domain part." ) diff --git a/server/mergin/auth/utils.py b/server/mergin/auth/utils.py new file mode 100644 index 00000000..b81b2eca --- /dev/null +++ b/server/mergin/auth/utils.py @@ -0,0 +1,13 @@ +# Copyright (C) Lutra Consulting Limited +# +# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + +from email_validator import validate_email, EmailNotValidError + + +def get_email_domain(email: str) -> str | None: + try: + result = validate_email(email, check_deliverability=False) + return result.domain + except EmailNotValidError: + return From 889a73c7e4023d2a080e91bf104afdc3516b532a Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 4 Nov 2025 14:28:40 +0100 Subject: [PATCH 06/46] improve tests --- server/mergin/tests/test_public_api_v2.py | 5 ++--- server/mergin/tests/utils.py | 6 ++++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 6272edb3..d23712f2 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -7,7 +7,7 @@ from . import DEFAULT_USER from .utils import ( add_user, - login, + logout, login_as_admin, create_workspace, create_project, @@ -142,11 +142,10 @@ def test_project_members(client): def test_get_project(client): """Test get project info endpoint""" - add_user("test_user", "ilovemergin") - login(client, "test_user", "ilovemergin") admin = User.query.filter_by(username=DEFAULT_USER[0]).first() test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) + logout(client) # lack of permissions response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 diff --git a/server/mergin/tests/utils.py b/server/mergin/tests/utils.py index 94fc033f..02042da0 100644 --- a/server/mergin/tests/utils.py +++ b/server/mergin/tests/utils.py @@ -387,3 +387,9 @@ def modify_file_times(path, time: datetime, accessed=True, modified=True): mtime = epoch_time if modified else file_stat.st_mtime os.utime(path, (atime, mtime)) + + +def logout(client): + """Test helper to log out the client""" + resp = client.get(url_for("/.mergin_auth_controller_logout")) + assert resp.status_code == 200 From e8acc1d76f2ee60c59a9e949d76f9391276b2b31 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 4 Nov 2025 14:49:40 +0100 Subject: [PATCH 07/46] fix json encoding --- server/mergin/sync/public_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index a82c5768..e4c949b8 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -627,7 +627,7 @@ def get_paginated_projects( ) # temporary yield to gevent hub until serialization is fully resolved (#317) data = ProjectListSchema(many=True, context=ctx).dump(result) data = {"projects": data, "count": total} - return data, 200 + return jsonify(data), 200 @auth_required From 178020c28f3ad49b84c47a3827e4d57ad4cffb2a Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 4 Nov 2025 15:26:51 +0100 Subject: [PATCH 08/46] use supplied regex --- server/Pipfile | 1 + server/Pipfile.lock | 124 +++++++++++++++++++++++++++++++++++- server/mergin/auth/forms.py | 25 ++++---- 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/server/Pipfile b/server/Pipfile index 9859c84e..397f4b69 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -42,6 +42,7 @@ typing_extensions = "==4.12.2" python-magic = "==0.4.27" # requirements for development on windows colorama = "==0.4.5" +regex = "2025.11.3" [dev-packages] pytest = "==8.3.2" diff --git a/server/Pipfile.lock b/server/Pipfile.lock index 1e02fadf..27026077 100644 --- a/server/Pipfile.lock +++ b/server/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5d7a107dfa425254eebbe86700e139290dc6986e54bd18e1538a2d87a60282f3" + "sha256": "e097e0123cf67b8856804f44a67ba31e6c7e2c28ebf81c0884883f59d9a2a130" }, "pipfile-spec": 6, "requires": { @@ -994,6 +994,128 @@ "markers": "python_version >= '3.9'", "version": "==0.36.2" }, + "regex": { + "hashes": [ + "sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a", + "sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a", + "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", + "sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b", + "sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6", + "sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e", + "sha256:149eb0bba95231fb4f6d37c8f760ec9fa6fabf65bab555e128dde5f2475193ec", + "sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39", + "sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281", + "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", + "sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc", + "sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467", + "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", + "sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267", + "sha256:22dd622a402aad4558277305350699b2be14bc59f64d64ae1d928ce7d072dced", + "sha256:22e7d1cdfa88ef33a2ae6aa0d707f9255eb286ffbd90045f1088246833223aee", + "sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c", + "sha256:2ab815eb8a96379a27c3b6157fcb127c8f59c36f043c1678110cea492868f1d5", + "sha256:2b441a4ae2c8049106e8b39973bfbddfb25a179dda2bdb99b0eeb60c40a6a3af", + "sha256:2fa2eed3f76677777345d2f81ee89f5de2f5745910e805f7af7386a920fa7313", + "sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02", + "sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009", + "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", + "sha256:38af559ad934a7b35147716655d4a2f79fcef2d695ddfe06a06ba40ae631fa7e", + "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", + "sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50", + "sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc", + "sha256:3bf28b1873a8af8bbb58c26cc56ea6e534d80053b41fb511a35795b6de507e6a", + "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", + "sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf", + "sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154", + "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", + "sha256:43b4fb020e779ca81c1b5255015fe2b82816c76ec982354534ad9ec09ad7c9e3", + "sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49", + "sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed", + "sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7", + "sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850", + "sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38", + "sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2", + "sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d", + "sha256:509dc827f89c15c66a0c216331260d777dd6c81e9a4e4f830e662b0bb296c313", + "sha256:51c1c1847128238f54930edb8805b660305dca164645a9fd29243f5610beea34", + "sha256:5cf77eac15bd264986c4a2c63353212c095b40f3affb2bc6b4ef80c4776c1a28", + "sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361", + "sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73", + "sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379", + "sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160", + "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", + "sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be", + "sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f", + "sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62", + "sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118", + "sha256:728a9d2d173a65b62bdc380b7932dd8e74ed4295279a8fe1021204ce210803e7", + "sha256:732aea6de26051af97b94bc98ed86448821f839d058e5d259c72bf6d73ad0fc0", + "sha256:74d04244852ff73b32eeede4f76f51c5bcf44bc3c207bc3e6cf1c5c45b890708", + "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", + "sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9", + "sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0", + "sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4", + "sha256:7a50cd39f73faa34ec18d6720ee25ef10c4c1839514186fcda658a06c06057a2", + "sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c", + "sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19", + "sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d", + "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", + "sha256:81519e25707fc076978c6143b81ea3dc853f176895af05bf7ec51effe818aeec", + "sha256:838441333bc90b829406d4a03cb4b8bf7656231b84358628b0406d803931ef32", + "sha256:849202cd789e5f3cf5dcc7822c34b502181b4824a65ff20ce82da5524e45e8e9", + "sha256:856a25c73b697f2ce2a24e7968285579e62577a048526161a2c0f53090bea9f9", + "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", + "sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab", + "sha256:8a3d571bd95fade53c86c0517f859477ff3a93c3fde10c9e669086f038e0f207", + "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", + "sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6", + "sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76", + "sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a", + "sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad", + "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", + "sha256:a295ca2bba5c1c885826ce3125fa0b9f702a1be547d821c01d65f199e10c01e2", + "sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7", + "sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de", + "sha256:b4774ff32f18e0504bfc4e59a3e71e18d83bc1e171a3c8ed75013958a03b2f14", + "sha256:b6f78f98741dcc89607c16b1e9426ee46ce4bf31ac5e6b0d40e81c89f3481ea5", + "sha256:b7f9ee819f94c6abfa56ec7b1dbab586f41ebbdc0a57e6524bd5e7f487a878c7", + "sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc", + "sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5", + "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", + "sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db", + "sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95", + "sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4", + "sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801", + "sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2", + "sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f", + "sha256:cc4076a5b4f36d849fd709284b4a3b112326652f3b0466f04002a6c15a0c96c1", + "sha256:cfe6d3f0c9e3b7e8c0c694b24d25e677776f5ca26dce46fd6b0489f9c8339391", + "sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6", + "sha256:d8b4a27eebd684319bdf473d39f1d79eed36bf2cd34bd4465cdb4618d82b3d56", + "sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b", + "sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce", + "sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e", + "sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e", + "sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f", + "sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4", + "sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a", + "sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58", + "sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed", + "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", + "sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031", + "sha256:ee3a83ce492074c35a74cc76cf8235d49e77b757193a5365ff86e3f2f93db9fd", + "sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe", + "sha256:f3b5a391c7597ffa96b41bd5cbd2ed0305f515fcbb367dfa72735679d5502364", + "sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9", + "sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f", + "sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536", + "sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4", + "sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2025.11.3" + }, "requests": { "hashes": [ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index 884d0dad..e5339b77 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -1,7 +1,8 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -import re + +import regex import safe from flask_wtf import FlaskForm from sqlalchemy import func @@ -17,7 +18,6 @@ from .models import MAX_USERNAME_LENGTH, User from ..app import UpdateForm, CustomStringField -from .utils import get_email_domain def username_validation(form, field): @@ -57,19 +57,20 @@ class ExtendedEmail(Email): because they make our email sending service to fail """ + EMAIL_PATTERN = regex.compile( + r"""(?i)^[\x60#&*\/=?^{!}~'_\p{L}0-9\-\+]+ + (\.[\x60#&*\/=?^{!}~'_\p{L}0-9\-\+]+)*\.?@ + ([_a-z0-9-]+(\.[_a-z0-9-]+)*\.) + [a-z0-9-]*[a-z0-9]{2,}$""", + regex.VERBOSE, + ) + def __call__(self, form, field): super().__call__(form, field) - if re.search(r"[|'—]", field.data): - raise ValidationError( - f"Email address '{field.data}' contains an invalid character." - ) - - domain = get_email_domain(field.data) - if not domain.isascii(): - raise ValidationError( - f"Email address '{field.data}' contains non-ASCII characters in the domain part." - ) + value = field.data.strip() + if not self.EMAIL_PATTERN.match(value): + raise ValidationError(f"Email address '{value}' is invalid.") class PasswordValidator: From 5c9b768fd1ab3cfcd8355c163a0389398ae44e3f Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 5 Nov 2025 07:43:42 +0100 Subject: [PATCH 09/46] Revert "fix json encoding" This reverts commit e8acc1d76f2ee60c59a9e949d76f9391276b2b31. --- server/mergin/sync/public_api_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index e4c949b8..a82c5768 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -627,7 +627,7 @@ def get_paginated_projects( ) # temporary yield to gevent hub until serialization is fully resolved (#317) data = ProjectListSchema(many=True, context=ctx).dump(result) data = {"projects": data, "count": total} - return jsonify(data), 200 + return data, 200 @auth_required From 40ce2d84f69349208a80899f78accb8f2d667821 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 5 Nov 2025 13:37:01 +0100 Subject: [PATCH 10/46] fix auth tests --- server/Pipfile | 2 +- server/mergin/tests/test_auth.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/Pipfile b/server/Pipfile index 4d19a818..8f724a68 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -41,9 +41,9 @@ importlib-metadata = "==8.4.0" # https://github.com/pallets/flask/issues/4502 typing_extensions = "==4.12.2" python-magic = "==0.4.27" click = "==8.2.0" +regex = "==2025.11.3" # requirements for development on windows colorama = "==0.4.5" -regex = "2025.11.3" [dev-packages] pytest = "==8.3.2" diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index b953577d..1afd5cc8 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -937,12 +937,12 @@ def test_server_usage(client): ("日人日本人", True), # non-ascii character ("usér", True), # non-ascii character ("user\\", False), # disallowed character - ("user\260", True), # non-ascii character (°) + ("user\260", False), # not letter character (°) ("user|", False), # vertical bar ("us er", False), # space in the middle ("us,er", False), # comma ("us—er", False), # dash - ("us'er", False), # apostrophe + ("us´er", False), # acute accent (" user", True), # starting with space (will be stripped) ("us.er", True), # dot in the middle (".user", False), # starting with dot From d4f267371a4d89ac6992c91186e028e284482c03 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 6 Nov 2025 12:49:20 +0100 Subject: [PATCH 11/46] Revert "use supplied regex" This reverts commit 178020c28f3ad49b84c47a3827e4d57ad4cffb2a. --- server/Pipfile.lock | 122 ------------------------------------ server/mergin/auth/forms.py | 25 ++++---- 2 files changed, 12 insertions(+), 135 deletions(-) diff --git a/server/Pipfile.lock b/server/Pipfile.lock index aa7566b8..ac41e613 100644 --- a/server/Pipfile.lock +++ b/server/Pipfile.lock @@ -1028,128 +1028,6 @@ "markers": "python_version >= '3.9'", "version": "==0.36.2" }, - "regex": { - "hashes": [ - "sha256:04d2765516395cf7dda331a244a3282c0f5ae96075f728629287dfa6f76ba70a", - "sha256:087511f5c8b7dfbe3a03f5d5ad0c2a33861b1fc387f21f6f60825a44865a385a", - "sha256:08b884f4226602ad40c5d55f52bf91a9df30f513864e0054bad40c0e9cf1afb7", - "sha256:0d31e08426ff4b5b650f68839f5af51a92a5b51abd8554a60c2fbc7c71f25d0b", - "sha256:0f9397d561a4c16829d4e6ff75202c1c08b68a3bdbfe29dbfcdb31c9830907c6", - "sha256:10483eefbfb0adb18ee9474498c9a32fcf4e594fbca0543bb94c48bac6183e2e", - "sha256:149eb0bba95231fb4f6d37c8f760ec9fa6fabf65bab555e128dde5f2475193ec", - "sha256:1e00ec2970aab10dc5db34af535f21fcf32b4a31d99e34963419636e2f85ae39", - "sha256:1eb1ebf6822b756c723e09f5186473d93236c06c579d2cc0671a722d2ab14281", - "sha256:1fedc720f9bb2494ce31a58a1631f9c82df6a09b49c19517ea5cc280b4541e01", - "sha256:1ff0d190c7f68ae7769cd0313fe45820ba07ffebfddfaa89cc1eb70827ba0ddc", - "sha256:2292cd5a90dab247f9abe892ac584cb24f0f54680c73fcb4a7493c66c2bf2467", - "sha256:22b29dda7e1f7062a52359fca6e58e548e28c6686f205e780b02ad8ef710de36", - "sha256:22c12d837298651e5550ac1d964e4ff57c3f56965fc1812c90c9fb2028eaf267", - "sha256:22dd622a402aad4558277305350699b2be14bc59f64d64ae1d928ce7d072dced", - "sha256:22e7d1cdfa88ef33a2ae6aa0d707f9255eb286ffbd90045f1088246833223aee", - "sha256:28ba4d69171fc6e9896337d4fc63a43660002b7da53fc15ac992abcf3410917c", - "sha256:2ab815eb8a96379a27c3b6157fcb127c8f59c36f043c1678110cea492868f1d5", - "sha256:2b441a4ae2c8049106e8b39973bfbddfb25a179dda2bdb99b0eeb60c40a6a3af", - "sha256:2fa2eed3f76677777345d2f81ee89f5de2f5745910e805f7af7386a920fa7313", - "sha256:32f74f35ff0f25a5021373ac61442edcb150731fbaa28286bbc8bb1582c89d02", - "sha256:3809988f0a8b8c9dcc0f92478d6501fac7200b9ec56aecf0ec21f4a2ec4b6009", - "sha256:3839967cf4dc4b985e1570fd8d91078f0c519f30491c60f9ac42a8db039be204", - "sha256:38af559ad934a7b35147716655d4a2f79fcef2d695ddfe06a06ba40ae631fa7e", - "sha256:3a91e4a29938bc1a082cc28fdea44be420bf2bebe2665343029723892eb073e1", - "sha256:3b30bc921d50365775c09a7ed446359e5c0179e9e2512beec4a60cbcef6ddd50", - "sha256:3b3a5f320136873cc5561098dfab677eea139521cb9a9e8db98b7e64aef44cbc", - "sha256:3bf28b1873a8af8bbb58c26cc56ea6e534d80053b41fb511a35795b6de507e6a", - "sha256:3e0b11b2b2433d1c39c7c7a30e3f3d0aeeea44c2a8d0bae28f6b95f639927a69", - "sha256:3e816cc9aac1cd3cc9a4ec4d860f06d40f994b5c7b4d03b93345f44e08cc68bf", - "sha256:3f8bf11a4827cc7ce5a53d4ef6cddd5ad25595d3c1435ef08f76825851343154", - "sha256:435bbad13e57eb5606a68443af62bed3556de2f46deb9f7d4237bc2f1c9fb3a0", - "sha256:43b4fb020e779ca81c1b5255015fe2b82816c76ec982354534ad9ec09ad7c9e3", - "sha256:442d86cf1cfe4faabf97db7d901ef58347efd004934da045c745e7b5bd57ac49", - "sha256:44f264d4bf02f3176467d90b294d59bf1db9fe53c141ff772f27a8b456b2a9ed", - "sha256:454d9b4ae7881afbc25015b8627c16d88a597479b9dea82b8c6e7e2e07240dc7", - "sha256:4aecb6f461316adf9f1f0f6a4a1a3d79e045f9b71ec76055a791affa3b285850", - "sha256:4bf146dca15cdd53224a1bf46d628bd7590e4a07fbb69e720d561aea43a32b38", - "sha256:4c5238d32f3c5269d9e87be0cf096437b7622b6920f5eac4fd202468aaeb34d2", - "sha256:4e1e592789704459900728d88d41a46fe3969b82ab62945560a31732ffc19a6d", - "sha256:509dc827f89c15c66a0c216331260d777dd6c81e9a4e4f830e662b0bb296c313", - "sha256:51c1c1847128238f54930edb8805b660305dca164645a9fd29243f5610beea34", - "sha256:5cf77eac15bd264986c4a2c63353212c095b40f3affb2bc6b4ef80c4776c1a28", - "sha256:5d9903ca42bfeec4cebedba8022a7c97ad2aab22e09573ce9976ba01b65e4361", - "sha256:61a08bcb0ec14ff4e0ed2044aad948d0659604f824cbd50b55e30b0ec6f09c73", - "sha256:62ba394a3dda9ad41c7c780f60f6e4a70988741415ae96f6d1bf6c239cf01379", - "sha256:639431bdc89d6429f6721625e8129413980ccd62e9d3f496be618a41d205f160", - "sha256:64350685ff08b1d3a6fff33f45a9ca183dc1d58bbfe4981604e70ec9801bbc26", - "sha256:6538241f45eb5a25aa575dbba1069ad786f68a4f2773a29a2bd3dd1f9de787be", - "sha256:669dcfb2e38f9e8c69507bace46f4889e3abbfd9b0c29719202883c0a603598f", - "sha256:66d559b21d3640203ab9075797a55165d79017520685fb407b9234d72ab63c62", - "sha256:6dd329a1b61c0ee95ba95385fb0c07ea0d3fe1a21e1349fa2bec272636217118", - "sha256:728a9d2d173a65b62bdc380b7932dd8e74ed4295279a8fe1021204ce210803e7", - "sha256:732aea6de26051af97b94bc98ed86448821f839d058e5d259c72bf6d73ad0fc0", - "sha256:74d04244852ff73b32eeede4f76f51c5bcf44bc3c207bc3e6cf1c5c45b890708", - "sha256:7521684c8c7c4f6e88e35ec89680ee1aa8358d3f09d27dfbdf62c446f5d4c695", - "sha256:75fa6f0056e7efb1f42a1c34e58be24072cb9e61a601340cc1196ae92326a4f9", - "sha256:78c2d02bb6e1da0720eedc0bad578049cad3f71050ef8cd065ecc87691bed2b0", - "sha256:795ea137b1d809eb6836b43748b12634291c0ed55ad50a7d72d21edf1cd565c4", - "sha256:7a50cd39f73faa34ec18d6720ee25ef10c4c1839514186fcda658a06c06057a2", - "sha256:7a7c7fdf755032ffdd72c77e3d8096bdcb0eb92e89e17571a196f03d88b11b3c", - "sha256:7be0277469bf3bd7a34a9c57c1b6a724532a0d235cd0dc4e7f4316f982c28b19", - "sha256:7eb542fd347ce61e1321b0a6b945d5701528dca0cd9759c2e3bb8bd57e47964d", - "sha256:7fe6e5440584e94cc4b3f5f4d98a25e29ca12dccf8873679a635638349831b98", - "sha256:81519e25707fc076978c6143b81ea3dc853f176895af05bf7ec51effe818aeec", - "sha256:838441333bc90b829406d4a03cb4b8bf7656231b84358628b0406d803931ef32", - "sha256:849202cd789e5f3cf5dcc7822c34b502181b4824a65ff20ce82da5524e45e8e9", - "sha256:856a25c73b697f2ce2a24e7968285579e62577a048526161a2c0f53090bea9f9", - "sha256:87eb52a81ef58c7ba4d45c3ca74e12aa4b4e77816f72ca25258a85b3ea96cb48", - "sha256:885b26aa3ee56433b630502dc3d36ba78d186a00cc535d3806e6bfd9ed3c70ab", - "sha256:8a3d571bd95fade53c86c0517f859477ff3a93c3fde10c9e669086f038e0f207", - "sha256:8e026094aa12b43f4fd74576714e987803a315c76edb6b098b9809db5de58f74", - "sha256:9697a52e57576c83139d7c6f213d64485d3df5bf84807c35fa409e6c970801c6", - "sha256:9b5aca4d5dfd7fbfbfbdaf44850fcc7709a01146a797536a8f84952e940cca76", - "sha256:9ddc42e68114e161e51e272f667d640f97e84a2b9ef14b7477c53aac20c2d59a", - "sha256:9f95fbaa0ee1610ec0fc6b26668e9917a582ba80c52cc6d9ada15e30aa9ab9ad", - "sha256:a12ab1f5c29b4e93db518f5e3872116b7e9b1646c9f9f426f777b50d44a09e8c", - "sha256:a295ca2bba5c1c885826ce3125fa0b9f702a1be547d821c01d65f199e10c01e2", - "sha256:a4cb042b615245d5ff9b3794f56be4138b5adc35a4166014d31d1814744148c7", - "sha256:adad1a1bcf1c9e76346e091d22d23ac54ef28e1365117d99521631078dfec9de", - "sha256:b4774ff32f18e0504bfc4e59a3e71e18d83bc1e171a3c8ed75013958a03b2f14", - "sha256:b6f78f98741dcc89607c16b1e9426ee46ce4bf31ac5e6b0d40e81c89f3481ea5", - "sha256:b7f9ee819f94c6abfa56ec7b1dbab586f41ebbdc0a57e6524bd5e7f487a878c7", - "sha256:ba0d8a5d7f04f73ee7d01d974d47c5834f8a1b0224390e4fe7c12a3a92a78ecc", - "sha256:bac4200befe50c670c405dc33af26dad5a3b6b255dd6c000d92fe4629f9ed6a5", - "sha256:bc8ab71e2e31b16e40868a40a69007bc305e1109bd4658eb6cad007e0bf67c41", - "sha256:bce22519c989bb72a7e6b36a199384c53db7722fe669ba891da75907fe3587db", - "sha256:bf3490bcbb985a1ae97b2ce9ad1c0f06a852d5b19dde9b07bdf25bf224248c95", - "sha256:c1e448051717a334891f2b9a620fe36776ebf3dd8ec46a0b877c8ae69575feb4", - "sha256:c54f768482cef41e219720013cd05933b6f971d9562544d691c68699bf2b6801", - "sha256:c56b4d162ca2b43318ac671c65bd4d563e841a694ac70e1a976ac38fcf4ca1d2", - "sha256:c9c30003b9347c24bcc210958c5d167b9e4f9be786cb380a7d32f14f9b84674f", - "sha256:cc4076a5b4f36d849fd709284b4a3b112326652f3b0466f04002a6c15a0c96c1", - "sha256:cfe6d3f0c9e3b7e8c0c694b24d25e677776f5ca26dce46fd6b0489f9c8339391", - "sha256:d6c2d5919075a1f2e413c00b056ea0c2f065b3f5fe83c3d07d325ab92dce51d6", - "sha256:d8b4a27eebd684319bdf473d39f1d79eed36bf2cd34bd4465cdb4618d82b3d56", - "sha256:dbe6095001465294f13f1adcd3311e50dd84e5a71525f20a10bd16689c61ce0b", - "sha256:dd16e78eb18ffdb25ee33a0682d17912e8cc8a770e885aeee95020046128f1ce", - "sha256:ddd76a9f58e6a00f8772e72cff8ebcff78e022be95edf018766707c730593e1e", - "sha256:df9eb838c44f570283712e7cff14c16329a9f0fb19ca492d21d4b7528ee6821e", - "sha256:dfec44d532be4c07088c3de2876130ff0fbeeacaa89a137decbbb5f665855a0f", - "sha256:e18bc3f73bd41243c9b38a6d9f2366cd0e0137a9aebe2d8ff76c5b67d4c0a3f4", - "sha256:e43586ce5bd28f9f285a6e729466841368c4a0353f6fd08d4ce4630843d3648a", - "sha256:e6b49cd2aad93a1790ce9cffb18964f6d3a4b0b3dbdbd5de094b65296fce6e58", - "sha256:e6c7a21dffba883234baefe91bc3388e629779582038f75d2a5be918e250f0ed", - "sha256:e721d1b46e25c481dc5ded6f4b3f66c897c58d2e8cfdf77bbced84339108b0b9", - "sha256:eadade04221641516fa25139273505a1c19f9bf97589a05bc4cfcd8b4a618031", - "sha256:ee3a83ce492074c35a74cc76cf8235d49e77b757193a5365ff86e3f2f93db9fd", - "sha256:f117efad42068f9715677c8523ed2be1518116d1c49b1dd17987716695181efe", - "sha256:f3b5a391c7597ffa96b41bd5cbd2ed0305f515fcbb367dfa72735679d5502364", - "sha256:f4ff94e58e84aedb9c9fce66d4ef9f27a190285b451420f297c9a09f2b9abee9", - "sha256:f99be08cfead2020c7ca6e396c13543baea32343b7a9a5780c462e323bd8872f", - "sha256:fd0a5e563c756de210bb964789b5abe4f114dacae9104a47e1a649b910361536", - "sha256:feff9e54ec0dd3833d659257f5c3f5322a12eee58ffa360984b716f8b92983f4", - "sha256:ffcca5b9efe948ba0661e9df0fa50d2bc4b097c70b9810212d6b62f05d83b2dd" - ], - "index": "pypi", - "markers": "python_version >= '3.9'", - "version": "==2025.11.3" - }, "requests": { "hashes": [ "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index e5339b77..884d0dad 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -1,8 +1,7 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - -import regex +import re import safe from flask_wtf import FlaskForm from sqlalchemy import func @@ -18,6 +17,7 @@ from .models import MAX_USERNAME_LENGTH, User from ..app import UpdateForm, CustomStringField +from .utils import get_email_domain def username_validation(form, field): @@ -57,20 +57,19 @@ class ExtendedEmail(Email): because they make our email sending service to fail """ - EMAIL_PATTERN = regex.compile( - r"""(?i)^[\x60#&*\/=?^{!}~'_\p{L}0-9\-\+]+ - (\.[\x60#&*\/=?^{!}~'_\p{L}0-9\-\+]+)*\.?@ - ([_a-z0-9-]+(\.[_a-z0-9-]+)*\.) - [a-z0-9-]*[a-z0-9]{2,}$""", - regex.VERBOSE, - ) - def __call__(self, form, field): super().__call__(form, field) - value = field.data.strip() - if not self.EMAIL_PATTERN.match(value): - raise ValidationError(f"Email address '{value}' is invalid.") + if re.search(r"[|'—]", field.data): + raise ValidationError( + f"Email address '{field.data}' contains an invalid character." + ) + + domain = get_email_domain(field.data) + if not domain.isascii(): + raise ValidationError( + f"Email address '{field.data}' contains non-ASCII characters in the domain part." + ) class PasswordValidator: From ea719023ca138acc029a44fd6e7ccea3b2b3ab7f Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Nov 2025 07:33:11 +0100 Subject: [PATCH 12/46] rm get_email_domain in auth utils --- server/mergin/auth/utils.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 server/mergin/auth/utils.py diff --git a/server/mergin/auth/utils.py b/server/mergin/auth/utils.py deleted file mode 100644 index b81b2eca..00000000 --- a/server/mergin/auth/utils.py +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright (C) Lutra Consulting Limited -# -# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial - -from email_validator import validate_email, EmailNotValidError - - -def get_email_domain(email: str) -> str | None: - try: - result = validate_email(email, check_deliverability=False) - return result.domain - except EmailNotValidError: - return From d4ac7ed2ee6b91db02082eaffc8cfd8d090e8f4b Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Nov 2025 07:33:37 +0100 Subject: [PATCH 13/46] use re lib to validate emails --- server/mergin/auth/forms.py | 23 +++++++++-------------- server/mergin/tests/test_auth.py | 7 ++++++- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/server/mergin/auth/forms.py b/server/mergin/auth/forms.py index 884d0dad..f7403bb8 100644 --- a/server/mergin/auth/forms.py +++ b/server/mergin/auth/forms.py @@ -1,6 +1,7 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial + import re import safe from flask_wtf import FlaskForm @@ -17,7 +18,6 @@ from .models import MAX_USERNAME_LENGTH, User from ..app import UpdateForm, CustomStringField -from .utils import get_email_domain def username_validation(form, field): @@ -51,25 +51,20 @@ class ExtendedEmail(Email): 3, multiple @ symbols, 4, leading, trailing, or consecutive dots in the local part, 5, invalid domain part - missing top level domain (user@example), consecutive dots, - Custom check for - - additional invalid characters disallows |'— - - non-ASCII characters in the domain part - because they make our email sending service to fail + The extended validation checks email addresses using the regex provided by Brevo, + so that we stay consistent with their validation rules and avoid API failures. """ def __call__(self, form, field): super().__call__(form, field) - if re.search(r"[|'—]", field.data): - raise ValidationError( - f"Email address '{field.data}' contains an invalid character." - ) + email = field.data.strip() - domain = get_email_domain(field.data) - if not domain.isascii(): - raise ValidationError( - f"Email address '{field.data}' contains non-ASCII characters in the domain part." - ) + pattern = r"^[\x60#&*\/=?^{!}~'+\w-]+(\.[\x60#&*\/=?^{!}~'+\w-]+)*\.?@([_a-zA-Z0-9-]+(\.[_a-zA-Z0-9-]+)*\.)[a-zA-Z0-9-]*[a-zA-Z0-9]{2,}$" + email_regexp = re.compile(pattern, re.IGNORECASE) + + if not email_regexp.match(email): + raise ValidationError(f"Email address '{email}' is invalid.") class PasswordValidator: diff --git a/server/mergin/tests/test_auth.py b/server/mergin/tests/test_auth.py index 1afd5cc8..69ba37c6 100644 --- a/server/mergin/tests/test_auth.py +++ b/server/mergin/tests/test_auth.py @@ -125,7 +125,11 @@ def test_logout(client): 400, ), # tests with upper case, but email already exists (" mergin@mergin.com ", "#pwd123", 400), # invalid password - ("verylonglonglonglonglonglonglongemail@example.com", "#pwd1234", 201), + ( + "verylonglonglonglonglonglonglongemail@lutra-consulting.co.uk", + "#pwd1234", + 201, + ), # long local part, second-level domain, dash in domain ("us.er@mergin.com", "#pwd1234", 201), # dot is allowed ("us er@mergin.com", "#pwd1234", 400), # space is disallowed ("test@gmaiñ.com", "#pwd1234", 400), # non-ASCII character in the domain @@ -946,6 +950,7 @@ def test_server_usage(client): (" user", True), # starting with space (will be stripped) ("us.er", True), # dot in the middle (".user", False), # starting with dot + ("us-er", True), # hyphen ] From 830ef0f886c455782d8b5f264b0e9f37a3fb099c Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Nov 2025 12:18:09 +0100 Subject: [PATCH 14/46] address review - survive non-existing project version --- server/mergin/sync/public_api_v2_controller.py | 9 +++++---- server/mergin/sync/schemas_v2.py | 4 +--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 96b7f914..712fe81d 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -139,9 +139,10 @@ def get_project(id, files_at_version=None): if files_at_version: pv = ProjectVersion.query.filter_by( project_id=project.id, name=ProjectVersion.from_v_name(files_at_version) - ).first_or_404() - data["files"] = ProjectFileSchema( - only=("path", "mtime", "size", "checksum"), many=True - ).dump(pv.files) + ).first() + if pv: + data["files"] = ProjectFileSchema( + only=("path", "mtime", "size", "checksum"), many=True + ).dump(pv.files) return data, 200 diff --git a/server/mergin/sync/schemas_v2.py b/server/mergin/sync/schemas_v2.py index 668a7e20..d6b781ee 100644 --- a/server/mergin/sync/schemas_v2.py +++ b/server/mergin/sync/schemas_v2.py @@ -30,9 +30,7 @@ class ProjectSchema(ma.SQLAlchemyAutoSchema): def _role(self, obj): role = ProjectPermissions.get_user_project_role(obj, current_user) - if not role: - return None - return role.value + return role.value if role else None class Meta: model = Project From caed4dff69e2feb985850d5ecd49671e57bbd490 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 7 Nov 2025 12:22:07 +0100 Subject: [PATCH 15/46] rm regex from pipfile --- server/Pipfile | 1 - 1 file changed, 1 deletion(-) diff --git a/server/Pipfile b/server/Pipfile index 8f724a68..3f9cb0a5 100644 --- a/server/Pipfile +++ b/server/Pipfile @@ -41,7 +41,6 @@ importlib-metadata = "==8.4.0" # https://github.com/pallets/flask/issues/4502 typing_extensions = "==4.12.2" python-magic = "==0.4.27" click = "==8.2.0" -regex = "==2025.11.3" # requirements for development on windows colorama = "==0.4.5" From 6adb3f24ac25c2202ba99a80d6766dc378a0dc3c Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Mon, 10 Nov 2025 07:55:09 +0100 Subject: [PATCH 16/46] fix test --- server/mergin/tests/test_public_api_v2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index d23712f2..542736ef 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -187,7 +187,9 @@ def test_get_project(client): response = client.get( f"v2/projects/{project.id}?files_at_version=v{project.latest_version+1}" ) - assert response.status_code == 404 + assert response.status_code == 200 + assert response.json["id"] == str(project.id) + assert "files" not in response.json.keys() # files response = client.get( f"v2/projects/{project.id}?files_at_version=v{project.latest_version-2}" From 0958fe7ea7f1e9adea6c60cd5a3a27b6898bd122 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Thu, 13 Nov 2025 14:53:32 +0100 Subject: [PATCH 17/46] Accept locale for currency calculation --- web-app/packages/lib/src/common/number_utils.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/web-app/packages/lib/src/common/number_utils.ts b/web-app/packages/lib/src/common/number_utils.ts index 7d7cd834..a1128973 100644 --- a/web-app/packages/lib/src/common/number_utils.ts +++ b/web-app/packages/lib/src/common/number_utils.ts @@ -39,9 +39,10 @@ export function formatFileSize( export function formatToCurrency( value: number, currency: string, - digits = 2 + digits = 2, + locale = 'en-UK' ): string { - return value.toLocaleString('en-UK', { + return value.toLocaleString(locale, { style: 'currency', currency, currencySign: 'accounting', From 3f660415932841c646d46b496d4d288e2e2380dc Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Nov 2025 11:41:57 +0100 Subject: [PATCH 18/46] Simplify condition & improve readbility --- server/mergin/sync/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index b73da860..5e732881 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -25,6 +25,7 @@ ) import magic from flask import current_app +from pathlib import Path def generate_checksum(file, chunk_size=4096): @@ -338,7 +339,7 @@ def files_size(): def is_valid_path(filepath: str) -> bool: """Check filepath and filename for invalid characters, absolute path or path traversal""" return ( - not len(re.split(r"\.[/\\]", filepath)) > 1 # ./ or .\ + not re.search(r"\.[/\\]", filepath) # ./ or .\ and is_valid_filepath(filepath) # invalid characters in filepath, absolute path and is_valid_filename( os.path.basename(filepath) From 75c2da7e9a4bd4d802ab9b58bec6f450c740eec5 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Nov 2025 12:53:26 +0100 Subject: [PATCH 19/46] Don't allow trailing space for new files --- server/mergin/sync/files.py | 10 +++++++++- server/mergin/sync/utils.py | 5 +++++ server/mergin/tests/test_utils.py | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index fd77c597..94cf602d 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -13,10 +13,10 @@ from .utils import ( is_file_name_blacklisted, - is_qgis, is_supported_extension, is_valid_path, is_versioned_file, + has_trailing_space, ) from ..app import DateTimeWithZ, ma @@ -221,6 +221,14 @@ def validate(self, data, **kwargs): f"Please remove the file or try compressing it into a ZIP file before uploading.", ) + # new checks must restrict only new files not to block existing projects + for file in data["added"]: + file_path = file["path"] + if has_trailing_space(file_path): + raise ValidationError( + f"Folder name contains a trailing space. Please remove the space in: {file_path}" + ) + class ProjectFileSchema(FileSchema): mtime = DateTimeWithZ() diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index 5e732881..5a98573f 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -347,6 +347,11 @@ def is_valid_path(filepath: str) -> bool: ) +def has_trailing_space(filepath: str) -> bool: + """Check filepath for trailing spaces that makes the project impossible to download on Windows""" + return all(part == part.rstrip() for part in Path(filepath).parts) + + def is_supported_extension(filepath) -> bool: """Check whether file's extension is supported.""" ext = os.path.splitext(filepath)[1].lower() diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index ed66cdd0..1ffaaf37 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -5,7 +5,6 @@ import base64 from datetime import datetime import json -import os import pytest from flask import url_for, current_app from sqlalchemy import desc @@ -24,6 +23,7 @@ is_valid_path, get_x_accel_uri, wkb2wkt, + has_trailing_space, ) from ..auth.models import LoginHistory, User from . import json_headers @@ -228,6 +228,10 @@ def test_is_valid_path(client, filepath, allow): assert is_valid_path(filepath) == allow +def test_has_trailing_space(): + assert has_trailing_space("photos /lutraHQ.jpg") is False + + def test_get_x_accell_uri(client): """Test get_x_accell_uri""" client.application.config["LOCAL_PROJECTS"] = "/data/" From d92cf798db253a0697fe006d15e56eb52fe08c61 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Nov 2025 12:53:49 +0100 Subject: [PATCH 20/46] rm unused imports --- server/mergin/sync/public_api_controller.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 6aa8f63e..42a713fb 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -55,9 +55,7 @@ from .files import ( ProjectFileChange, ChangesSchema, - UploadFileSchema, ProjectFileSchema, - FileSchema, files_changes_from_upload, mergin_secure_filename, ) @@ -83,17 +81,11 @@ generate_checksum, Toucher, get_x_accel_uri, - is_file_name_blacklisted, get_ip, get_user_agent, generate_location, is_valid_uuid, - is_versioned_file, - get_project_path, get_device_id, - is_valid_path, - is_supported_type, - is_supported_extension, get_mimetype, wkb2wkt, ) From b402d7253bef613467ffff05b7bff01d1c675542 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Nov 2025 13:10:42 +0100 Subject: [PATCH 21/46] fix logic --- server/mergin/sync/files.py | 1 - server/mergin/sync/utils.py | 2 +- server/mergin/tests/test_utils.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index 94cf602d..6c820823 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -220,7 +220,6 @@ def validate(self, data, **kwargs): f"Unsupported file type detected: {file_path}. " f"Please remove the file or try compressing it into a ZIP file before uploading.", ) - # new checks must restrict only new files not to block existing projects for file in data["added"]: file_path = file["path"] diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index 5a98573f..634a2081 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -349,7 +349,7 @@ def is_valid_path(filepath: str) -> bool: def has_trailing_space(filepath: str) -> bool: """Check filepath for trailing spaces that makes the project impossible to download on Windows""" - return all(part == part.rstrip() for part in Path(filepath).parts) + return any(part != part.rstrip() for part in Path(filepath).parts) def is_supported_extension(filepath) -> bool: diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index 1ffaaf37..117be8c4 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -229,7 +229,7 @@ def test_is_valid_path(client, filepath, allow): def test_has_trailing_space(): - assert has_trailing_space("photos /lutraHQ.jpg") is False + assert has_trailing_space("photos /lutraHQ.jpg") is True def test_get_x_accell_uri(client): From 072e4bb52ab998084c9b47ed0ff89ed91c9b8398 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 14 Nov 2025 14:07:10 +0100 Subject: [PATCH 22/46] fix schema import --- server/mergin/sync/public_api_v2.yaml | 28 ++++++------------- .../mergin/sync/public_api_v2_controller.py | 4 +-- 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/server/mergin/sync/public_api_v2.yaml b/server/mergin/sync/public_api_v2.yaml index 1f7ef843..bf3db007 100644 --- a/server/mergin/sync/public_api_v2.yaml +++ b/server/mergin/sync/public_api_v2.yaml @@ -588,25 +588,15 @@ components: type: array description: List of files in the project items: - type: object - properties: - path: - type: string - description: File name including path from project root - example: data/layer.gpkg - mtime: - type: string - format: date-time - description: File modification timestamp - example: 2024-11-19T13:50:00Z - size: - type: integer - description: File size in bytes - example: 1234 - checksum: - type: string - description: File checksum hash - example: 9adb76bf81a34880209040ffe5ee262a090b62ab + allOf: + - $ref: '#/components/schemas/File' + - type: object + properties: + mtime: + type: string + format: date-time + description: File modification timestamp + example: 2024-11-19T13:50:00Z File: type: object description: Project file metadata diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 662462b2..27e0355a 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -14,7 +14,7 @@ from marshmallow import ValidationError from sqlalchemy.exc import IntegrityError -from .schemas_v2 import ProjectSchema +from .schemas_v2 import ProjectSchema as ProjectSchemaV2 from ..app import db from ..auth import auth_required from ..auth.models import User @@ -166,7 +166,7 @@ def remove_project_collaborator(id, user_id): def get_project(id, files_at_version=None): """Get project info. Include list of files at specific version if requested.""" project = require_project_by_uuid(id, ProjectPermissions.Read) - data = ProjectSchema().dump(project) + data = ProjectSchemaV2().dump(project) if files_at_version: pv = ProjectVersion.query.filter_by( project_id=project.id, name=ProjectVersion.from_v_name(files_at_version) From df29847f432ad2fcafd8544a9e49e62271ea87bb Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Wed, 19 Nov 2025 11:34:50 +0100 Subject: [PATCH 23/46] add negative test case --- server/mergin/tests/test_utils.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index 117be8c4..14578863 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -228,8 +228,15 @@ def test_is_valid_path(client, filepath, allow): assert is_valid_path(filepath) == allow -def test_has_trailing_space(): - assert has_trailing_space("photos /lutraHQ.jpg") is True +test_paths = [ + ("photos /lutraHQ.jpg", True), + ("photo s/ lutraHQ.jpg", False), +] + + +@pytest.mark.parametrize("path,result", test_paths) +def test_has_trailing_space(path, result): + assert has_trailing_space(path) is result def test_get_x_accell_uri(client): From 43b560c7cd0c6bbd7b34e0315733553bf01dd0f1 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 20 Nov 2025 09:27:39 +0100 Subject: [PATCH 24/46] cleanup --- server/mergin/sync/files.py | 6 +++--- server/mergin/sync/utils.py | 5 ----- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/server/mergin/sync/files.py b/server/mergin/sync/files.py index 6c820823..415e9ec5 100644 --- a/server/mergin/sync/files.py +++ b/server/mergin/sync/files.py @@ -212,12 +212,12 @@ def validate(self, data, **kwargs): if not is_valid_path(file_path): raise ValidationError( - f"Unsupported file name detected: {file_path}. Please remove the invalid characters." + f"Unsupported file name detected: '{file_path}'. Please remove the invalid characters." ) if not is_supported_extension(file_path): raise ValidationError( - f"Unsupported file type detected: {file_path}. " + f"Unsupported file type detected: '{file_path}'. " f"Please remove the file or try compressing it into a ZIP file before uploading.", ) # new checks must restrict only new files not to block existing projects @@ -225,7 +225,7 @@ def validate(self, data, **kwargs): file_path = file["path"] if has_trailing_space(file_path): raise ValidationError( - f"Folder name contains a trailing space. Please remove the space in: {file_path}" + f"Folder name contains a trailing space. Please remove the space in: '{file_path}'." ) diff --git a/server/mergin/sync/utils.py b/server/mergin/sync/utils.py index 634a2081..de0fbe94 100644 --- a/server/mergin/sync/utils.py +++ b/server/mergin/sync/utils.py @@ -101,11 +101,6 @@ def is_qgis(path: str) -> bool: return ext.lower() in [".qgs", ".qgz"] -def int_version(version): - """Convert v format of version to integer representation.""" - return int(version.lstrip("v")) if re.match(r"v\d", version) else None - - def is_versioned_file(file): """Check if file is compatible with geodiff lib and hence suitable for versioning.""" diff_extensions = [".gpkg", ".sqlite"] From 99ded71285fd0c469dd9467c8add5a9e37f7a80c Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 20 Nov 2025 11:58:44 +0100 Subject: [PATCH 25/46] add windows tests --- server/mergin/tests/test_utils.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/server/mergin/tests/test_utils.py b/server/mergin/tests/test_utils.py index 14578863..bf5f4666 100644 --- a/server/mergin/tests/test_utils.py +++ b/server/mergin/tests/test_utils.py @@ -12,6 +12,7 @@ from unittest.mock import patch from pathvalidate import sanitize_filename from pygeodiff import GeoDiff +from pathlib import PureWindowsPath from ..utils import save_diagnostic_log_file @@ -228,15 +229,22 @@ def test_is_valid_path(client, filepath, allow): assert is_valid_path(filepath) == allow -test_paths = [ - ("photos /lutraHQ.jpg", True), - ("photo s/ lutraHQ.jpg", False), +trailing_spaces_paths = [ + ("photos /lutraHQ.jpg", "posix", True), + ("photo s/ lutraHQ.jpg", "posix", False), + ("assets\photos \lutraHQ.jpg", "windows", True), + ("assets\ photos\lutraHQ.jpg", "windows", False), ] -@pytest.mark.parametrize("path,result", test_paths) -def test_has_trailing_space(path, result): - assert has_trailing_space(path) is result +@pytest.mark.parametrize("path,path_platform,result", trailing_spaces_paths) +def test_has_trailing_space(path, path_platform, result): + if path_platform == "windows": + # we must mock Path to instantiate as Windows path + with patch("mergin.sync.utils.Path", PureWindowsPath): + assert has_trailing_space(path) is result + else: + assert has_trailing_space(path) is result def test_get_x_accell_uri(client): From 6dc9bb75c43587e52975f866ca5d107a5503684c Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 21 Nov 2025 12:42:30 +0100 Subject: [PATCH 26/46] fix tests --- server/mergin/sync/public_api_controller.py | 2 +- server/mergin/tests/test_project_controller.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 42a713fb..f0c9047c 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -972,7 +972,7 @@ def push_finish(transaction_id): if len(unsupported_files): abort( 400, - f"Unsupported file type detected: {unsupported_files[0]}. " + f"Unsupported file type detected: '{unsupported_files[0]}'. " f"Please remove the file or try compressing it into a ZIP file before uploading.", ) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index 1cba91cc..c7a0550e 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -2495,7 +2495,7 @@ def test_filepath_manipulation(client): assert resp.status_code == 400 assert ( resp.json["detail"] - == f"Unsupported file name detected: {manipulated_path}. Please remove the invalid characters." + == f"Unsupported file name detected: '{manipulated_path}'. Please remove the invalid characters." ) @@ -2528,7 +2528,7 @@ def test_supported_file_upload(client): assert resp.status_code == 400 assert ( resp.json["detail"] - == f"Unsupported file type detected: {script_filename}. Please remove the file or try compressing it into a ZIP file before uploading." + == f"Unsupported file type detected: '{script_filename}'. Please remove the file or try compressing it into a ZIP file before uploading." ) # Extension spoofing to trick the validator spoof_name = "script.gpkg" @@ -2567,7 +2567,7 @@ def test_supported_file_upload(client): assert resp.status_code == 400 assert ( resp.json["detail"] - == f"Unsupported file type detected: {spoof_name}. Please remove the file or try compressing it into a ZIP file before uploading." + == f"Unsupported file type detected: '{spoof_name}'. Please remove the file or try compressing it into a ZIP file before uploading." ) From cac45a8880b5f690c9cb379cf2e949177ab988cd Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 24 Nov 2025 10:26:19 +0100 Subject: [PATCH 27/46] remove bottlnech with removing chunks --- server/mergin/sync/public_api_controller.py | 8 -------- server/mergin/sync/public_api_v2_controller.py | 9 --------- 2 files changed, 17 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index f0c9047c..0b487874 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -1028,14 +1028,6 @@ def push_finish(transaction_id): # let's move uploaded files where they are expected to be os.renames(files_dir, version_dir) - # remove used chunks - for file in upload.changes["added"] + upload.changes["updated"]: - file_chunks = file.get("chunks", []) - for chunk_id in file_chunks: - chunk_file = os.path.join(upload.upload_dir, "chunks", chunk_id) - if os.path.exists(chunk_file): - move_to_tmp(chunk_file) - logging.info( f"Push finished for project: {project.id}, project version: {v_next_version}, transaction id: {transaction_id}." ) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 27e0355a..d317e273 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -318,14 +318,6 @@ def create_project_version(id): temp_files_dir = os.path.join(upload.upload_dir, "files", v_next_version) os.renames(temp_files_dir, version_dir) - # remove used chunks - for file in to_be_added_files + to_be_updated_files: - file_chunks = file.get("chunks", []) - for chunk_id in file_chunks: - chunk_file = get_chunk_location(chunk_id) - if os.path.exists(chunk_file): - move_to_tmp(chunk_file) - logging.info( f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}." ) @@ -377,7 +369,6 @@ def upload_chunk(id: str): # we could have used request.data here, but it could eventually cause OOM issue save_to_file(request.stream, dest_file, current_app.config["MAX_CHUNK_SIZE"]) except IOError: - move_to_tmp(dest_file, chunk_id) return BigChunkError().response(413) except Exception as e: return UploadError(error="Error saving chunk").response(400) From 85a58341b66958c9a53aec5f747a3a3f17992a0d Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 24 Nov 2025 10:33:07 +0100 Subject: [PATCH 28/46] update tests for chunks - they exist --- server/mergin/tests/test_public_api_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index dda0bc53..d6d3150a 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -378,7 +378,7 @@ def test_create_version(client, data, expected, err_code): if expected == 201: assert response.json["version"] == "v2" assert project.latest_version == 2 - assert all(not os.path.exists(chunk) for chunk in chunks) + assert all(os.path.exists(chunk) for chunk in chunks) else: assert project.latest_version == 1 if err_code: From 88e5490992ccda615eca1856dc43b625a7400bf6 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 24 Nov 2025 11:13:14 +0100 Subject: [PATCH 29/46] update tests --- server/mergin/tests/test_public_api_v2.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index d6d3150a..bfb705f3 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +from mergin.sync.tasks import remove_unused_chunks from . import DEFAULT_USER from .utils import ( add_user, @@ -22,6 +23,7 @@ from mergin.app import db from mergin.config import Configuration +from mergin.sync.config import Configuration as SyncConfiguration from mergin.sync.errors import ( BigChunkError, ProjectLocked, @@ -375,10 +377,15 @@ def test_create_version(client, data, expected, err_code): response = client.post(f"v2/projects/{project.id}/versions", json=data) assert response.status_code == expected + # mock chunks expiration to check if removed if expected == 201: assert response.json["version"] == "v2" assert project.latest_version == 2 + # chunks exists after upload, cleanup job did not remove them assert all(os.path.exists(chunk) for chunk in chunks) + with patch.object(SyncConfiguration, "UPLOAD_CHUNKS_EXPIRATION", 0): + remove_unused_chunks() + assert all(not os.path.exists(chunk) for chunk in chunks) else: assert project.latest_version == 1 if err_code: From 2149c0c4a4fe720525e9d73232e27cec533d319f Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Mon, 24 Nov 2025 17:10:09 +0100 Subject: [PATCH 30/46] call remove unused chunks async job --- server/mergin/sync/public_api_v2_controller.py | 10 ++++++++++ server/mergin/sync/tasks.py | 11 ++++++++++- server/mergin/tests/test_public_api_v2.py | 15 ++++++++++----- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index d317e273..217204c1 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -14,6 +14,8 @@ from marshmallow import ValidationError from sqlalchemy.exc import IntegrityError +from mergin.sync.tasks import remove_transaction_chunks + from .schemas_v2 import ProjectSchema as ProjectSchemaV2 from ..app import db from ..auth import auth_required @@ -318,6 +320,14 @@ def create_project_version(id): temp_files_dir = os.path.join(upload.upload_dir, "files", v_next_version) os.renames(temp_files_dir, version_dir) + # remove used chunks + # get chunks from added and updated files + chunks_ids = [] + for file in to_be_added_files + to_be_updated_files: + file_chunks = file.get("chunks", []) + chunks_ids.extend(file_chunks) + remove_transaction_chunks.delay(chunks_ids) + logging.info( f"Push finished for project: {project.id}, project version: {v_next_version}, upload id: {upload.id}." ) diff --git a/server/mergin/sync/tasks.py b/server/mergin/sync/tasks.py index 7688a3ee..bc1ae044 100644 --- a/server/mergin/sync/tasks.py +++ b/server/mergin/sync/tasks.py @@ -13,7 +13,7 @@ from .models import Project, ProjectVersion, FileHistory from .storages.disk import move_to_tmp from .config import Configuration -from .utils import remove_outdated_files +from .utils import get_chunk_location, remove_outdated_files from ..celery import celery from ..app import db @@ -169,3 +169,12 @@ def remove_unused_chunks(): if not os.path.isdir(dir): continue remove_outdated_files(dir, time_delta) + + +@celery.task +def remove_transaction_chunks(chunks=[]): + """Remove chunks related to a specific sync transaction""" + for chunk in chunks: + chunk_path = get_chunk_location(chunk) + if os.path.exists(chunk_path): + os.remove(chunk_path) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index bfb705f3..6a4243fd 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -from mergin.sync.tasks import remove_unused_chunks +from mergin.sync.tasks import remove_transaction_chunks, remove_unused_chunks from . import DEFAULT_USER from .utils import ( add_user, @@ -358,6 +358,7 @@ def test_create_version(client, data, expected, err_code): assert project.latest_version == 1 chunks = [] + chunk_ids = [] if expected == 201: # mimic chunks were uploaded for f in data["changes"]["added"] + data["changes"]["updated"]: @@ -374,17 +375,21 @@ def test_create_version(client, data, expected, err_code): out_file.write(in_file.read(CHUNK_SIZE)) chunks.append(chunk_location) + chunk_ids.append(chunk) - response = client.post(f"v2/projects/{project.id}/versions", json=data) + with patch( + "mergin.sync.public_api_v2_controller.remove_transaction_chunks.delay" + ) as mock_remove: + response = client.post(f"v2/projects/{project.id}/versions", json=data) assert response.status_code == expected - # mock chunks expiration to check if removed if expected == 201: assert response.json["version"] == "v2" assert project.latest_version == 2 # chunks exists after upload, cleanup job did not remove them assert all(os.path.exists(chunk) for chunk in chunks) - with patch.object(SyncConfiguration, "UPLOAD_CHUNKS_EXPIRATION", 0): - remove_unused_chunks() + if chunk_ids: + assert mock_remove.called_once_with(chunk_ids) + remove_transaction_chunks(chunk_ids) assert all(not os.path.exists(chunk) for chunk in chunks) else: assert project.latest_version == 1 From 05fb028eba5baf26c67b25d6ca5f4acafc63a364 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 25 Nov 2025 12:10:46 +0100 Subject: [PATCH 31/46] type for transaction chunks --- server/mergin/sync/tasks.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/mergin/sync/tasks.py b/server/mergin/sync/tasks.py index bc1ae044..9392997c 100644 --- a/server/mergin/sync/tasks.py +++ b/server/mergin/sync/tasks.py @@ -7,6 +7,7 @@ import os import time from datetime import datetime, timedelta, timezone +from typing import List, Optional from zipfile import ZIP_DEFLATED, ZipFile from flask import current_app @@ -172,8 +173,10 @@ def remove_unused_chunks(): @celery.task -def remove_transaction_chunks(chunks=[]): +def remove_transaction_chunks(chunks: Optional[List[str]] = None): """Remove chunks related to a specific sync transaction""" + if not chunks: + return for chunk in chunks: chunk_path = get_chunk_location(chunk) if os.path.exists(chunk_path): From 1a46828fd745a5f533cb1a307a14dc95acf18ffe Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 28 Nov 2025 12:50:11 +0100 Subject: [PATCH 32/46] return 404 to anonymous user requesting private project info --- server/mergin/sync/permissions.py | 3 +++ server/mergin/tests/test_public_api_v2.py | 8 +++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 4305a15f..b249e93c 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -219,6 +219,9 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled if not scheduled: project = project.filter(Project.removed_at.is_(None)) project = project.first_or_404() + # we don't want to tell anonymous user if a private project exists + if current_user.is_anonymous and not project.public: + abort(404) workspace = project.workspace if not workspace: abort(404) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index dda0bc53..9f636dc1 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -10,6 +10,8 @@ create_workspace, create_project, upload_file_to_project, + login, + file_info, ) from ..auth.models import User @@ -47,7 +49,6 @@ _get_changes_with_diff_0_size, _get_changes_without_added, ) -from .utils import add_user, file_info def test_schedule_delete_project(client): @@ -173,8 +174,13 @@ def test_get_project(client): admin = User.query.filter_by(username=DEFAULT_USER[0]).first() test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) + user = add_user("tests", "tests") logout(client) + # anonymous user cannot access the resource + response = client.get(f"v2/projects/{project.id}") + assert response.status_code == 404 # lack of permissions + login(client, user.username, "tests") response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project From e8b1ba78d37364f278cbbb130d36c16c8145948d Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 28 Nov 2025 13:12:29 +0100 Subject: [PATCH 33/46] failing test --- server/mergin/tests/test_public_api_v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 9f636dc1..7ab7c190 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -174,12 +174,12 @@ def test_get_project(client): admin = User.query.filter_by(username=DEFAULT_USER[0]).first() test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) - user = add_user("tests", "tests") logout(client) # anonymous user cannot access the resource response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 # lack of permissions + user = add_user("tests", "tests") login(client, user.username, "tests") response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 From 1152229cf316a2d48212f6c50d19217b88de9a0e Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 28 Nov 2025 13:44:13 +0100 Subject: [PATCH 34/46] failing test 2 --- server/mergin/tests/test_public_api_v2.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 7ab7c190..41100f41 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -1,6 +1,7 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import time from . import DEFAULT_USER from .utils import ( @@ -175,12 +176,13 @@ def test_get_project(client): test_workspace = create_workspace() project = create_project("new_project", test_workspace, admin) logout(client) - # anonymous user cannot access the resource + # anonymous user cannot access the private resource response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 # lack of permissions user = add_user("tests", "tests") login(client, user.username, "tests") + time.sleep(1) response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project From b350cd506530cb74922ecadb124358d0a6413a7d Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 28 Nov 2025 15:29:21 +0100 Subject: [PATCH 35/46] DEBUG failing test 3 --- server/mergin/sync/permissions.py | 4 ++++ server/mergin/tests/test_public_api_v2.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index b249e93c..65399e4e 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -222,6 +222,10 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled # we don't want to tell anonymous user if a private project exists if current_user.is_anonymous and not project.public: abort(404) + if current_user.is_anonymous: + print("Permissions info: User is anonymous") + else: + print(f"Permissions info: Logged in as {current_user.username}") workspace = project.workspace if not workspace: abort(404) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 41100f41..6e0bfef4 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -1,7 +1,6 @@ # Copyright (C) Lutra Consulting Limited # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -import time from . import DEFAULT_USER from .utils import ( @@ -181,8 +180,9 @@ def test_get_project(client): assert response.status_code == 404 # lack of permissions user = add_user("tests", "tests") + print(f"Test info: User '{user.username}' created.") login(client, user.username, "tests") - time.sleep(1) + print(f"Test info: User '{user.username}' logged in.") response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project From 05c88d8c834876e213e073b99200474251db7576 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Fri, 28 Nov 2025 15:47:30 +0100 Subject: [PATCH 36/46] DEBUG try to print to stderr - 4 --- server/mergin/sync/permissions.py | 7 +++++-- server/mergin/tests/test_public_api_v2.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 65399e4e..1f5fee05 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial +import sys import os from functools import wraps from typing import Optional @@ -223,9 +224,11 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled if current_user.is_anonymous and not project.public: abort(404) if current_user.is_anonymous: - print("Permissions info: User is anonymous") + print("Permissions info: User is anonymous", file=sys.stderr) else: - print(f"Permissions info: Logged in as {current_user.username}") + print( + f"Permissions info: Logged in as {current_user.username}", file=sys.stderr + ) workspace = project.workspace if not workspace: abort(404) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 6e0bfef4..5487fac8 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -21,6 +21,7 @@ from sqlalchemy.exc import IntegrityError import pytest from datetime import datetime, timedelta, timezone +import sys from mergin.app import db from mergin.config import Configuration @@ -180,9 +181,9 @@ def test_get_project(client): assert response.status_code == 404 # lack of permissions user = add_user("tests", "tests") - print(f"Test info: User '{user.username}' created.") + print(f"Test info: User '{user.username}' created.", file=sys.stderr) login(client, user.username, "tests") - print(f"Test info: User '{user.username}' logged in.") + print(f"Test info: User '{user.username}' logged in.", file=sys.stderr) response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project From be212c2ccaae300fb98fccaea47ab6a15d6a9819 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 11:31:10 +0100 Subject: [PATCH 37/46] DEBUG 5: reset Global read --- server/mergin/tests/test_public_api_v2.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 5487fac8..59312744 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -168,6 +168,7 @@ def test_project_members(client): # access provided by workspace role cannot be removed directly response = client.delete(url + f"/{user.id}") assert response.status_code == 404 + Configuration.GLOBAL_READ = 0 def test_get_project(client): @@ -180,6 +181,10 @@ def test_get_project(client): response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 # lack of permissions + print( + f"Global permissions: Global read is {Configuration.GLOBAL_READ}, Global write is {Configuration.GLOBAL_WRITE}, Global admin is {Configuration.GLOBAL_ADMIN}", + file=sys.stderr, + ) user = add_user("tests", "tests") print(f"Test info: User '{user.username}' created.", file=sys.stderr) login(client, user.username, "tests") From 19798f5dbba5980e9fdaecedf8683b6140a5dc1b Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 11:49:03 +0100 Subject: [PATCH 38/46] validate version param schema --- server/mergin/sync/public_api_v2.yaml | 11 ++++++----- server/mergin/tests/test_public_api_v2.py | 3 +++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/server/mergin/sync/public_api_v2.yaml b/server/mergin/sync/public_api_v2.yaml index bf3db007..c1c74f68 100644 --- a/server/mergin/sync/public_api_v2.yaml +++ b/server/mergin/sync/public_api_v2.yaml @@ -87,8 +87,7 @@ paths: description: Include list of files at specific version required: false schema: - type: string - example: v3 + $ref: "#/components/schemas/VersionName" responses: "200": description: Success @@ -305,9 +304,7 @@ paths: default: false example: true version: - type: string - pattern: '^$|^v\d+$' - example: v2 + $ref: "#/components/schemas/VersionName" changes: type: object required: @@ -849,3 +846,7 @@ components: - editor - writer - owner + VersionName: + type: string + pattern: '^$|^v\d+$' + example: v2 diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 59312744..d30fab2f 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -247,6 +247,9 @@ def test_get_project(client): ) assert len(response.json["files"]) == 3 assert {f["path"] for f in response.json["files"]} == set(files) + # invalid version format parameter + response = client.get(f"v2/projects/{project.id}?files_at_version=3") + assert response.status_code == 400 push_data = [ From 99e4414b454fdf880841e71bb29f940570518403 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 11:49:44 +0100 Subject: [PATCH 39/46] rm prints --- server/mergin/sync/permissions.py | 6 ------ server/mergin/tests/test_public_api_v2.py | 6 ------ 2 files changed, 12 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 1f5fee05..28dbadee 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -223,12 +223,6 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled # we don't want to tell anonymous user if a private project exists if current_user.is_anonymous and not project.public: abort(404) - if current_user.is_anonymous: - print("Permissions info: User is anonymous", file=sys.stderr) - else: - print( - f"Permissions info: Logged in as {current_user.username}", file=sys.stderr - ) workspace = project.workspace if not workspace: abort(404) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index d30fab2f..8b57e781 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -181,14 +181,8 @@ def test_get_project(client): response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 # lack of permissions - print( - f"Global permissions: Global read is {Configuration.GLOBAL_READ}, Global write is {Configuration.GLOBAL_WRITE}, Global admin is {Configuration.GLOBAL_ADMIN}", - file=sys.stderr, - ) user = add_user("tests", "tests") - print(f"Test info: User '{user.username}' created.", file=sys.stderr) login(client, user.username, "tests") - print(f"Test info: User '{user.username}' logged in.", file=sys.stderr) response = client.get(f"v2/projects/{project.id}") assert response.status_code == 403 # access public project From e1fab33e62bac4629ec81b5032f0a4408255337d Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 12:19:30 +0100 Subject: [PATCH 40/46] add flag for v1 compatibility --- server/mergin/sync/permissions.py | 26 +++++++++++++++---- .../mergin/sync/public_api_v2_controller.py | 18 +++++++------ 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 28dbadee..b18f5835 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -210,7 +210,21 @@ def require_project(ws, project_name, permission) -> Project: return project -def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled=False): +def require_project_by_uuid( + uuid: str, permission: ProjectPermissions, scheduled=False, expose=True +) -> Project: + """ + Retrieves a project by UUID after validating existence, workspace status, and permissions. + + Args: + uuid (str): The unique identifier of the project. + permission (ProjectPermissions): The permission level required to access the project. + scheduled (bool, optional): If ``True``, bypasses the check for projects marked for deletion. + expose (bool, optional): Controls security disclosure behavior on permission failure. + - If `True`: Returns 403 Forbidden (reveals project exists but access is denied). + - If `False`: Returns 404 Not Found (hides project existence for security). + Defaults to `True` for v1 endpoints compatibility. + """ if not is_valid_uuid(uuid): abort(404) @@ -220,16 +234,18 @@ def require_project_by_uuid(uuid: str, permission: ProjectPermissions, scheduled if not scheduled: project = project.filter(Project.removed_at.is_(None)) project = project.first_or_404() - # we don't want to tell anonymous user if a private project exists - if current_user.is_anonymous and not project.public: - abort(404) + workspace = project.workspace if not workspace: abort(404) if not is_active_workspace(workspace): abort(404, "Workspace doesn't exist") if not permission.check(project, current_user): - abort(403, "You do not have permissions for this project") + # we don't want to tell anonymous user if a private project exists + if expose: + abort(403, "You do not have permissions for this project") + else: + abort(404) return project diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 27e0355a..48469353 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -57,7 +57,7 @@ def schedule_delete_project(id): Celery job `mergin.sync.tasks.remove_projects_backups` does the rest. """ - project = require_project_by_uuid(id, ProjectPermissions.Delete) + project = require_project_by_uuid(id, ProjectPermissions.Delete, expose=False) project.removed_at = datetime.utcnow() project.removed_by = current_user.id db.session.commit() @@ -68,7 +68,9 @@ def schedule_delete_project(id): @auth_required def delete_project_now(id): """Delete the project immediately""" - project = require_project_by_uuid(id, ProjectPermissions.Delete, scheduled=True) + project = require_project_by_uuid( + id, ProjectPermissions.Delete, scheduled=True, expose=False + ) project.delete() return NoContent, 204 @@ -77,7 +79,7 @@ def delete_project_now(id): @auth_required def update_project(id): """Rename project""" - project = require_project_by_uuid(id, ProjectPermissions.Update) + project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) new_name = request.json["name"].strip() validation_error = project_name_validation(new_name) if validation_error: @@ -100,7 +102,7 @@ def update_project(id): @auth_required def get_project_collaborators(id): """Get project collaborators, with both direct role and inherited role from workspace""" - project = require_project_by_uuid(id, ProjectPermissions.Update) + project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) project_members = [] for user, workspace_role in project.workspace.members(): project_role = project.get_role(user.id) @@ -123,7 +125,7 @@ def get_project_collaborators(id): @auth_required def add_project_collaborator(id): """Add project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update) + project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) user = User.get_by_login(request.json["user"]) if not user: abort(404) @@ -140,7 +142,7 @@ def add_project_collaborator(id): @auth_required def update_project_collaborator(id, user_id): """Update project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update) + project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) user = User.query.filter_by(id=user_id, active=True).first_or_404() if not project.get_role(user_id): abort(404) @@ -154,7 +156,7 @@ def update_project_collaborator(id, user_id): @auth_required def remove_project_collaborator(id, user_id): """Remove project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update) + project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) if not project.get_role(user_id): abort(404) @@ -165,7 +167,7 @@ def remove_project_collaborator(id, user_id): def get_project(id, files_at_version=None): """Get project info. Include list of files at specific version if requested.""" - project = require_project_by_uuid(id, ProjectPermissions.Read) + project = require_project_by_uuid(id, ProjectPermissions.Read, expose=False) data = ProjectSchemaV2().dump(project) if files_at_version: pv = ProjectVersion.query.filter_by( From d80900bea3e05ca36676c4f281ec531e46c56469 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 12:47:14 +0100 Subject: [PATCH 41/46] actions should expose --- server/mergin/sync/public_api_v2_controller.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index 48469353..dadd4a34 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -57,7 +57,7 @@ def schedule_delete_project(id): Celery job `mergin.sync.tasks.remove_projects_backups` does the rest. """ - project = require_project_by_uuid(id, ProjectPermissions.Delete, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Delete) project.removed_at = datetime.utcnow() project.removed_by = current_user.id db.session.commit() @@ -68,9 +68,7 @@ def schedule_delete_project(id): @auth_required def delete_project_now(id): """Delete the project immediately""" - project = require_project_by_uuid( - id, ProjectPermissions.Delete, scheduled=True, expose=False - ) + project = require_project_by_uuid(id, ProjectPermissions.Delete, scheduled=True) project.delete() return NoContent, 204 @@ -79,7 +77,7 @@ def delete_project_now(id): @auth_required def update_project(id): """Rename project""" - project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Update) new_name = request.json["name"].strip() validation_error = project_name_validation(new_name) if validation_error: @@ -102,7 +100,7 @@ def update_project(id): @auth_required def get_project_collaborators(id): """Get project collaborators, with both direct role and inherited role from workspace""" - project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Update) project_members = [] for user, workspace_role in project.workspace.members(): project_role = project.get_role(user.id) @@ -125,7 +123,7 @@ def get_project_collaborators(id): @auth_required def add_project_collaborator(id): """Add project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Update) user = User.get_by_login(request.json["user"]) if not user: abort(404) @@ -142,7 +140,7 @@ def add_project_collaborator(id): @auth_required def update_project_collaborator(id, user_id): """Update project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Update) user = User.query.filter_by(id=user_id, active=True).first_or_404() if not project.get_role(user_id): abort(404) @@ -156,7 +154,7 @@ def update_project_collaborator(id, user_id): @auth_required def remove_project_collaborator(id, user_id): """Remove project collaborator""" - project = require_project_by_uuid(id, ProjectPermissions.Update, expose=False) + project = require_project_by_uuid(id, ProjectPermissions.Update) if not project.get_role(user_id): abort(404) From e8451429b41e3a39687f546041eed6c390b0729f Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 13:08:33 +0100 Subject: [PATCH 42/46] GET project never exposes --- server/mergin/sync/permissions.py | 2 +- server/mergin/tests/test_public_api_v2.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index b18f5835..0d61e407 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -223,7 +223,7 @@ def require_project_by_uuid( expose (bool, optional): Controls security disclosure behavior on permission failure. - If `True`: Returns 403 Forbidden (reveals project exists but access is denied). - If `False`: Returns 404 Not Found (hides project existence for security). - Defaults to `True` for v1 endpoints compatibility. + Standard is that reading results in 404, while writing results in 403 """ if not is_valid_uuid(uuid): abort(404) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 8b57e781..63336be5 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -180,11 +180,11 @@ def test_get_project(client): # anonymous user cannot access the private resource response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 - # lack of permissions + # lack of permissions also results in 404 for GET project user = add_user("tests", "tests") login(client, user.username, "tests") response = client.get(f"v2/projects/{project.id}") - assert response.status_code == 403 + assert response.status_code == 404 # access public project project.public = True db.session.commit() From b1e895c896e54acb73dea79bfe96216f80a5d65e Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 14:19:26 +0100 Subject: [PATCH 43/46] cleanup --- server/mergin/sync/permissions.py | 3 +-- server/mergin/sync/public_api_v2_controller.py | 1 - server/mergin/tests/test_public_api_v2.py | 1 - 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 0d61e407..6401663b 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial -import sys import os from functools import wraps from typing import Optional @@ -241,8 +240,8 @@ def require_project_by_uuid( if not is_active_workspace(workspace): abort(404, "Workspace doesn't exist") if not permission.check(project, current_user): - # we don't want to tell anonymous user if a private project exists if expose: + # we don't want to tell anonymous user if a private project exists abort(403, "You do not have permissions for this project") else: abort(404) diff --git a/server/mergin/sync/public_api_v2_controller.py b/server/mergin/sync/public_api_v2_controller.py index dadd4a34..67b75820 100644 --- a/server/mergin/sync/public_api_v2_controller.py +++ b/server/mergin/sync/public_api_v2_controller.py @@ -42,7 +42,6 @@ from .public_api_controller import catch_sync_failure from .schemas import ( ProjectMemberSchema, - ProjectVersionSchema, UploadChunkSchema, ProjectSchema, ) diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 63336be5..98850b82 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -21,7 +21,6 @@ from sqlalchemy.exc import IntegrityError import pytest from datetime import datetime, timedelta, timezone -import sys from mergin.app import db from mergin.config import Configuration From 0007fa2fea9147413861f236e300489acf6849bb Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Tue, 2 Dec 2025 16:09:45 +0100 Subject: [PATCH 44/46] return 403 for authenticated without permission --- server/mergin/sync/permissions.py | 10 +++++----- server/mergin/tests/test_public_api_v2.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/mergin/sync/permissions.py b/server/mergin/sync/permissions.py index 6401663b..7dd042d5 100644 --- a/server/mergin/sync/permissions.py +++ b/server/mergin/sync/permissions.py @@ -233,6 +233,9 @@ def require_project_by_uuid( if not scheduled: project = project.filter(Project.removed_at.is_(None)) project = project.first_or_404() + if not expose and current_user.is_anonymous and not project.public: + # we don't want to tell anonymous user if a private project exists + abort(404) workspace = project.workspace if not workspace: @@ -240,11 +243,8 @@ def require_project_by_uuid( if not is_active_workspace(workspace): abort(404, "Workspace doesn't exist") if not permission.check(project, current_user): - if expose: - # we don't want to tell anonymous user if a private project exists - abort(403, "You do not have permissions for this project") - else: - abort(404) + abort(403, "You do not have permissions for this project") + return project diff --git a/server/mergin/tests/test_public_api_v2.py b/server/mergin/tests/test_public_api_v2.py index 98850b82..b890f0b9 100644 --- a/server/mergin/tests/test_public_api_v2.py +++ b/server/mergin/tests/test_public_api_v2.py @@ -179,11 +179,11 @@ def test_get_project(client): # anonymous user cannot access the private resource response = client.get(f"v2/projects/{project.id}") assert response.status_code == 404 - # lack of permissions also results in 404 for GET project + # lack of permissions user = add_user("tests", "tests") login(client, user.username, "tests") response = client.get(f"v2/projects/{project.id}") - assert response.status_code == 404 + assert response.status_code == 403 # access public project project.public = True db.session.commit() From 31518fb1589041ab6748850b4b55a12a47354794 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Tue, 25 Nov 2025 16:01:34 +0100 Subject: [PATCH 45/46] Fix failing tests with random 504 Do not update global config variable for gevent mode. Make sure we do not use gevent env for tests apart of dedicated tests. In those tests mock configuration rather than modifing global variable. --- server/.test.env | 1 + server/mergin/tests/test_middleware.py | 99 +++++++++++++++----------- 2 files changed, 59 insertions(+), 41 deletions(-) diff --git a/server/.test.env b/server/.test.env index bdaa7bfa..63294a3f 100644 --- a/server/.test.env +++ b/server/.test.env @@ -24,3 +24,4 @@ SECURITY_BEARER_SALT='bearer' SECURITY_EMAIL_SALT='email' SECURITY_PASSWORD_SALT='password' DIAGNOSTIC_LOGS_DIR=/tmp/diagnostic_logs +GEVENT_WORKER=0 \ No newline at end of file diff --git a/server/mergin/tests/test_middleware.py b/server/mergin/tests/test_middleware.py index 82b9cf26..2f5cbe4f 100644 --- a/server/mergin/tests/test_middleware.py +++ b/server/mergin/tests/test_middleware.py @@ -6,6 +6,7 @@ import psycogreen.gevent import pytest import sqlalchemy +from unittest.mock import patch from ..app import create_simple_app, GeventTimeoutMiddleware, db from ..config import Configuration @@ -14,58 +15,74 @@ @pytest.mark.parametrize("use_middleware", [True, False]) def test_use_middleware(use_middleware): """Test using middleware""" - Configuration.GEVENT_WORKER = use_middleware - Configuration.GEVENT_REQUEST_TIMEOUT = 1 - application = create_simple_app() + with patch.object( + Configuration, + "GEVENT_WORKER", + use_middleware, + ), patch.object( + Configuration, + "GEVENT_REQUEST_TIMEOUT", + 1, + ): + application = create_simple_app() - def ping(): - gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) - return "pong" + def ping(): + gevent.sleep(Configuration.GEVENT_REQUEST_TIMEOUT + 1) + return "pong" - application.add_url_rule("/test", "ping", ping) - app_context = application.app_context() - app_context.push() + application.add_url_rule("/test", "ping", ping) + app_context = application.app_context() + app_context.push() - assert isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware - # in case of gevent, dummy endpoint it set to time out - assert application.test_client().get("/test").status_code == ( - 504 if use_middleware else 200 - ) + assert ( + isinstance(application.wsgi_app, GeventTimeoutMiddleware) == use_middleware + ) + # in case of gevent, dummy endpoint it set to time out + assert application.test_client().get("/test").status_code == ( + 504 if use_middleware else 200 + ) def test_catch_timeout(): """Test proper handling of gevent timeout with db.session.rollback""" psycogreen.gevent.patch_psycopg() - Configuration.GEVENT_WORKER = True - Configuration.GEVENT_REQUEST_TIMEOUT = 1 - application = create_simple_app() + with patch.object( + Configuration, + "GEVENT_WORKER", + True, + ), patch.object( + Configuration, + "GEVENT_REQUEST_TIMEOUT", + 1, + ): + application = create_simple_app() - def unhandled(): - try: - db.session.execute("SELECT pg_sleep(1.1);") - finally: - db.session.execute("SELECT 1;") - return "" + def unhandled(): + try: + db.session.execute("SELECT pg_sleep(1.1);") + finally: + db.session.execute("SELECT 1;") + return "" - def timeout(): - try: - db.session.execute("SELECT pg_sleep(1.1);") - except gevent.timeout.Timeout: - db.session.rollback() - raise - finally: - db.session.execute("SELECT 1;") - return "" + def timeout(): + try: + db.session.execute("SELECT pg_sleep(1.1);") + except gevent.timeout.Timeout: + db.session.rollback() + raise + finally: + db.session.execute("SELECT 1;") + return "" - application.add_url_rule("/unhandled", "unhandled", unhandled) - application.add_url_rule("/timeout", "timeout", timeout) - app_context = application.app_context() - app_context.push() + application.add_url_rule("/unhandled", "unhandled", unhandled) + application.add_url_rule("/timeout", "timeout", timeout) + app_context = application.app_context() + app_context.push() - assert application.test_client().get("/timeout").status_code == 504 + assert application.test_client().get("/timeout").status_code == 504 - # in case of missing rollback sqlalchemy would raise error - with pytest.raises(sqlalchemy.exc.PendingRollbackError): - application.test_client().get("/unhandled") + # in case of missing rollback sqlalchemy would raise error + with pytest.raises(sqlalchemy.exc.PendingRollbackError): + application.test_client().get("/unhandled") - db.session.rollback() + db.session.rollback() From b45cef1f795140e6f3879091c5ca511c9349b3c8 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Tue, 9 Dec 2025 13:44:26 +0100 Subject: [PATCH 46/46] Bump version --- server/mergin/version.py | 2 +- server/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/server/mergin/version.py b/server/mergin/version.py index cdacf710..10d74e1e 100644 --- a/server/mergin/version.py +++ b/server/mergin/version.py @@ -4,4 +4,4 @@ def get_version(): - return "2025.7.3" + return "2025.8.2" diff --git a/server/setup.py b/server/setup.py index 33f41ea7..7a3d1939 100644 --- a/server/setup.py +++ b/server/setup.py @@ -6,7 +6,7 @@ setup( name="mergin", - version="2025.7.3", + version="2025.8.2", url="https://github.com/MerginMaps/mergin", license="AGPL-3.0-only", author="Lutra Consulting Limited",