From 843e161035786187839d5500bbdb363b4dfc7497 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 11 Dec 2025 09:22:37 +0100 Subject: [PATCH 1/3] Filter out specific filenames when cloning project --- server/mergin/sync/public_api_controller.py | 10 +++++++++- server/mergin/sync/storages/disk.py | 6 +++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 0b487874..07104808 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -92,6 +92,10 @@ from .errors import StorageLimitHit, ProjectLocked from ..utils import format_time_delta +EXCLUDED_CLONE_FILENAMES = { + "qgis_cfg.xml", +} + def parse_project_access_update_request(access: Dict) -> Dict: """Parse raw project access update request and filter out invalid entries. @@ -1138,7 +1142,9 @@ def clone_project(namespace, project_name): # noqa: E501 db.session.add(p) try: - p.storage.initialize(template_project=cloned_project) + p.storage.initialize( + template_project=cloned_project, excluded_files=EXCLUDED_CLONE_FILENAMES + ) except InitializationError as e: abort(400, f"Failed to clone project: {str(e)}") @@ -1149,6 +1155,8 @@ def clone_project(namespace, project_name): # noqa: E501 # transform source files to new uploaded files file_changes = [] for file in cloned_project.files: + if os.path.basename(file.path) in EXCLUDED_CLONE_FILENAMES: + continue file_changes.append( ProjectFileChange( file.path, diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index 4491ad98..a62e165e 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -178,7 +178,7 @@ def _project_dir(self): ) return project_dir - def initialize(self, template_project=None): + def initialize(self, template_project=None, excluded_files=None): if os.path.exists(self.project_dir): raise InitializationError( "Project directory already exists: {}".format(self.project_dir) @@ -193,8 +193,12 @@ def initialize(self, template_project=None): if ws.disk_usage() + template_project.disk_usage > ws.storage: self.delete() raise InitializationError("Disk quota reached") + if excluded_files is None: + excluded_files = set() for file in template_project.files: + if os.path.basename(file.path) in excluded_files: + continue src = os.path.join(template_project.storage.project_dir, file.location) dest = os.path.join( self.project_dir, From 33183c5ef88dd96de021c5621c34339970bb4fa7 Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 11 Dec 2025 11:13:13 +0100 Subject: [PATCH 2/3] test for files exclution when cloning --- server/mergin/sync/config.py | 3 +++ server/mergin/sync/public_api_controller.py | 9 +++------ server/mergin/sync/storages/disk.py | 2 +- server/mergin/tests/test_project_controller.py | 13 ++++++++++++- server/mergin/tests/test_projects/test/qgis_cfg.xml | 4 ++++ 5 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 server/mergin/tests/test_projects/test/qgis_cfg.xml diff --git a/server/mergin/sync/config.py b/server/mergin/sync/config.py index 7200dae5..e616a0ca 100644 --- a/server/mergin/sync/config.py +++ b/server/mergin/sync/config.py @@ -75,3 +75,6 @@ class Configuration(object): UPLOAD_CHUNKS_EXPIRATION = config( "UPLOAD_CHUNKS_EXPIRATION", default=86400, cast=int ) + EXCLUDED_CLONE_FILENAMES = config( + "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv() + ) diff --git a/server/mergin/sync/public_api_controller.py b/server/mergin/sync/public_api_controller.py index 07104808..f8b88cd1 100644 --- a/server/mergin/sync/public_api_controller.py +++ b/server/mergin/sync/public_api_controller.py @@ -92,10 +92,6 @@ from .errors import StorageLimitHit, ProjectLocked from ..utils import format_time_delta -EXCLUDED_CLONE_FILENAMES = { - "qgis_cfg.xml", -} - def parse_project_access_update_request(access: Dict) -> Dict: """Parse raw project access update request and filter out invalid entries. @@ -1140,10 +1136,11 @@ def clone_project(namespace, project_name): # noqa: E501 ) p.updated = datetime.utcnow() db.session.add(p) + files_to_exclude = current_app.config.get("EXCLUDED_CLONE_FILENAMES", []) try: p.storage.initialize( - template_project=cloned_project, excluded_files=EXCLUDED_CLONE_FILENAMES + template_project=cloned_project, excluded_files=files_to_exclude ) except InitializationError as e: abort(400, f"Failed to clone project: {str(e)}") @@ -1155,7 +1152,7 @@ def clone_project(namespace, project_name): # noqa: E501 # transform source files to new uploaded files file_changes = [] for file in cloned_project.files: - if os.path.basename(file.path) in EXCLUDED_CLONE_FILENAMES: + if os.path.basename(file.path) in files_to_exclude: continue file_changes.append( ProjectFileChange( diff --git a/server/mergin/sync/storages/disk.py b/server/mergin/sync/storages/disk.py index a62e165e..7b038755 100644 --- a/server/mergin/sync/storages/disk.py +++ b/server/mergin/sync/storages/disk.py @@ -194,7 +194,7 @@ def initialize(self, template_project=None, excluded_files=None): self.delete() raise InitializationError("Disk quota reached") if excluded_files is None: - excluded_files = set() + excluded_files = [] for file in template_project.files: if os.path.basename(file.path) in excluded_files: diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index c7a0550e..e7840977 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -1728,6 +1728,8 @@ def test_clone_project(client, data, username, expected): assert resp.json["code"] == "StorageLimitHit" assert resp.json["detail"] == "You have reached a data limit (StorageLimitHit)" if expected == 200: + excluded_filenames = current_app.config.get("EXCLUDED_CLONE_FILENAMES") + proj = data.get("project", test_project).strip() template = Project.query.filter_by( name=test_project, workspace_id=test_workspace_id @@ -1735,9 +1737,12 @@ def test_clone_project(client, data, username, expected): project = Project.query.filter_by( name=proj, workspace_id=test_workspace_id ).first() + template_files_filtered = [ + f for f in template.files if f.path not in excluded_filenames + ] assert not any( x.checksum != y.checksum and x.path != y.path - for x, y in zip(project.files, template.files) + for x, y in zip(project.files, template_files_filtered) ) assert os.path.exists( os.path.join(project.storage.project_dir, project.files[0].location) @@ -1755,6 +1760,12 @@ def test_clone_project(client, data, username, expected): item for item in changes if item.change == PushChangeType.UPDATE.value ] assert pv.device_id == json_headers["X-Device-Id"] + + assert not any(f.path == excluded_filenames[0] for f in project.files) + assert not os.path.exists( + os.path.join(project.storage.project_dir, excluded_filenames[0]) + ) + assert len(project.files) == len(template.files) - 1 # cleanup shutil.rmtree(project.storage.project_dir) diff --git a/server/mergin/tests/test_projects/test/qgis_cfg.xml b/server/mergin/tests/test_projects/test/qgis_cfg.xml new file mode 100644 index 00000000..c2eece89 --- /dev/null +++ b/server/mergin/tests/test_projects/test/qgis_cfg.xml @@ -0,0 +1,4 @@ + +0737fe398eb9f26bd847fb9da2407646a2e8c89dc2f93eba5a059f19eedd8017e50c557d2c7d435a2701d881cdaf4fbbd3a892e4367053b5bfc348b556ae252314d9b06fc70a4f184362d064023c1ed6c4dd7ee14dce10ea91595e8548f7ba3d3eaca1d41063f50d1ccc12bfb90c059271254dca780e0d60e68bd234844fda81a0781977907485b397aa1263aef81863625eb439dc349fca0dbc641b4a606657f17e55d2c02fbc95388bf9f96977c65fcf7b723689d5fcdaf73190a5597425b3d33c858c2c4ef8c334b5f601c98db05557c8f690cdb9f73c725bf7ee420fffb6037cc9e80c7374a55ab55baf4aaa7f1c957fd40bb69b9fcb41ec42b063330bbddcd73f4de69e47772309167cb20ef4fe3250db96c29b71772edd18c7e73c501d569f4f8deda15fb0bcf6701d81902a6fc6c722db9e0d766d18a45297232224738c07a1a4f8fe490954efcae05fc1e43eac4eb5efc8b9008dcff4cf3688db4b7e268c9adf75d88d4d1d3648232c6fc2ac98b49b3bbb19368ce460b4a7a9828558d473eb0f1ba34c09f1ba9ce0170ac6d6c656176760e4012c56daef3f5f05320d5f84260d2e6b5a0b15620c33802d1c8c2f28084eef63b32f8130edd4789972b25960e12eb79351d11316f78aea3a941b9e7f1f4042f708d873ac7807ce0652819b2e2f77f9aac1c50cf72d2341118c41419f5f4c31474d8dbe56558dab9cb4b8e4fe8df9c8b4a057d9c6fe6b098b78e150aadd2a45cf1ea15e02f8f1f8b1b46d1a5513c26a63ea08788675feaa912e884ceee57adc120393c8a5bc42988f7b210195f6eff5de3e332d0d67321d05b907f836eb0f0f9e97388f89b699638804978639ad8b4c889f1a56952f949242679506cba5cc35538ea01b1621dd6a154f92b721b5247e294a5394df9c87765675b737dcb28346fc4032b68f87f46150aa4aa136378903036aff61fd41cf0cbcdd0865660f26d7f1d49f29ff5962adc209b9db71d12bf49bf67950496f18ca1de0a5cd7186e1bc0fcf826ffd1bb91ab36412c43730db5ff9ec57990fee27c5158446294bf0d8e61e12ee53e80b606b541c754ed45b2289079df8b647a8ca12fb1706e371523a581af50d333adfe5e84bcce2a60e84e24bdc1eb74610bc28b279b15c4f2020b045d2e4a7f846e488d74e761d98c05f105452235f602b3fe8beccf4b11d35ba6042dcc97f68090f40edbd6e8497434c193343cca98ebeabdd8620ec7eec642efda7cd45f0a9547ea821ac193eb1a8fb8c9c71d2e607b4651de5b8b613bc38aa4ba06bdb65a3d6b6e92546f1a4113e0bbce99aadbab3bbb07f31d6f90b3ff58b4494815e97a265c1c5e8a826bf14177427e03247395a18941753c0e580c42661a9c959ad57b93b97fb4adeca49927f3bec95eff361e95c324623f1c7c4d39e71250938d4189461cd6c1e978a5445f88eaf47670f23145cb7c8faf42ac83158743004fefb17a37a25edcf2425d530dd12ca52fdcfc399542cd288773c06931ce9aaac94df69dc6514fa3b1b8629dcfe725c0dcd77b5db967c5620dbc2444f4b78fb247e33a54ed2cbcaba3b92833b6d75b4900697da646f04da9a04d6353556b0ab70f8dd952eed9bd9cee1d53e760b292080862a74f625eb402662aadd94efaa6cce0727d3ccab5b6e112f25562effadfcf70307800e0d28976327576e99380facea2828ddb6a85addb4d4c0cfdb73cd848a9f707f8f978caf5de82756c80f42d53719987d5b4826397de8674d75dc1308dd3e96af37e9b3e42175dca1a5ff58a4aa4881a344113711a93340ee6515e5b9d03d1f4979531c84ec187b9303ea763b2641f530144cf52a81812349511219fc92bb038ec62d438c3beaf723 + + \ No newline at end of file From 22b481fe50fa2fa433ed00e86b79f2e0070899cb Mon Sep 17 00:00:00 2001 From: Herman Snevajs Date: Thu, 11 Dec 2025 12:09:24 +0100 Subject: [PATCH 3/3] fix tests --- server/mergin/tests/test_project_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/mergin/tests/test_project_controller.py b/server/mergin/tests/test_project_controller.py index e7840977..054ba063 100644 --- a/server/mergin/tests/test_project_controller.py +++ b/server/mergin/tests/test_project_controller.py @@ -2011,7 +2011,7 @@ def test_get_projects_by_uuids(client): {"page": 1, "per_page": 5, "desc": False}, 200, "v1", - {"added": 12, "removed": 0, "updated": 0, "updated_diff": 0}, + {"added": 13, "removed": 0, "updated": 0, "updated_diff": 0}, ), ( {"page": 2, "per_page": 3, "desc": True},