From 9769f63b8db365e3bb10eceefa2edf135b524f88 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 6 Aug 2025 16:29:39 +0200 Subject: [PATCH 01/22] optionaly store temp directory on download job for cleanup --- mergin/client_pull.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index a46e6f96..3a025bb8 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -44,7 +44,16 @@ class DownloadJob: """ def __init__( - self, project_path, total_size, version, update_tasks, download_queue_items, directory, mp, project_info + self, + project_path, + total_size, + version, + update_tasks, + download_queue_items, + directory, + mp, + project_info, + download_tmp_dir: typing.Optional[tempfile.TemporaryDirectory] = None, ): self.project_path = project_path self.total_size = total_size # size of data to download (in bytes) @@ -57,6 +66,7 @@ def __init__( self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server self.failure_log_file = None # log file, copied from the project directory if download fails + self.download_tmp_dir = download_tmp_dir # temporary directory for downloads if specified def dump(self): print("--- JOB ---", self.total_size, "bytes") From f45af09d7d8383cb728fbc7007243d7d8682a287 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 6 Aug 2025 16:30:17 +0200 Subject: [PATCH 02/22] download items to temp directory to avoid issues --- mergin/client_pull.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 3a025bb8..931ba308 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -164,11 +164,13 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"got project info. version {version}") + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + # prepare download update_tasks = [] # stuff to do at the end of download for file in project_info["files"]: file["version"] = version - items = _download_items(file, directory) + items = _download_items(file, tmp_dir.name) is_latest_version = project_version == latest_proj_info["version"] update_tasks.append(UpdateTask(file["path"], items, latest_version=is_latest_version)) @@ -182,7 +184,9 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"will download {len(update_tasks)} files in {len(download_list)} chunks, total size {total_size}") - job = DownloadJob(project_path, total_size, version, update_tasks, download_list, directory, mp, project_info) + job = DownloadJob( + project_path, total_size, version, update_tasks, download_list, directory, mp, project_info, tmp_dir + ) # start download job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) From c68e9def31b7503adcc739cd2dea8feabd4d9b7e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Wed, 6 Aug 2025 16:30:42 +0200 Subject: [PATCH 03/22] if job has download temp directory clean it up --- mergin/client_pull.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 931ba308..4698d5d7 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -255,6 +255,9 @@ def download_project_finalize(job): # final update of project metadata job.mp.update_metadata(job.project_info) + if job.download_tmp_dir: + job.download_tmp_dir.cleanup() + def download_project_cancel(job): """ From 58dd75734c4b586aa3f785b9ed7b1ac4d6b4d268 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:25:38 +0200 Subject: [PATCH 04/22] copy to mergin project dir --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 5eec1f57..ca624e1f 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -250,7 +250,7 @@ def download_project_finalize(job): for task in job.update_tasks: # right now only copy tasks... - task.apply(job.directory, job.mp) + task.apply(job.mp.dir, job.mp) # final update of project metadata job.mp.update_metadata(job.project_info) From e217b9a8baba8e1e9a813967fd5252e2a9ec3420 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:25:48 +0200 Subject: [PATCH 05/22] fix tests --- mergin/test/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 32bc192f..1c350a11 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2549,7 +2549,7 @@ def test_download_failure(mc): # download project async with pytest.raises(IsADirectoryError): job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(download_dir, "base.gpkg.0")) + os.makedirs(os.path.join(job.download_tmp_dir.name, "base.gpkg.0")) download_project_wait(job) download_project_finalize(job) @@ -2561,7 +2561,7 @@ def test_download_failure(mc): # active waiting remove_folders([download_dir]) job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(download_dir, "base.gpkg.0")) + os.makedirs(os.path.join(job.download_tmp_dir.name, "base.gpkg.0")) with pytest.raises(IsADirectoryError): while True: assert download_project_is_running(job) From 0d339af87c758070fa322b9a88caa41fe1c9c972 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:30:41 +0200 Subject: [PATCH 06/22] use dir --- mergin/client_pull.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index ca624e1f..4f07a153 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -50,10 +50,9 @@ def __init__( version, update_tasks, download_queue_items, - directory, + directory: tempfile.TemporaryDirectory, mp, project_info, - download_tmp_dir: typing.Optional[tempfile.TemporaryDirectory] = None, ): self.project_path = project_path self.total_size = total_size # size of data to download (in bytes) @@ -66,7 +65,6 @@ def __init__( self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server self.failure_log_file = None # log file, copied from the project directory if download fails - self.download_tmp_dir = download_tmp_dir # temporary directory for downloads if specified def dump(self): print("--- JOB ---", self.total_size, "bytes") From f29b5ed081c16a28b8ce9ee27402b2bf06225760 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:31:00 +0200 Subject: [PATCH 07/22] store log in mergin project dir --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 4f07a153..e61c3ff4 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -117,7 +117,7 @@ def _cleanup_failed_download(directory, mergin_project=None): mergin_project.remove_logging_handler() # keep log file as it might contain useful debug info - log_file = os.path.join(directory, ".mergin", "client-log.txt") + log_file = os.path.join(mergin_project.dir, ".mergin", "client-log.txt") dest_path = None if os.path.exists(log_file): From 7f753018bc512f4486ce913f734fccf641de7c7d Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:31:29 +0200 Subject: [PATCH 08/22] while downloading use temp directory --- mergin/client_pull.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index e61c3ff4..ae6c31b5 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -146,6 +146,8 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info("--- version: " + mc.user_agent_info()) mp.log.info(f"--- start download {project_path}") + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + try: # check whether we download the latest version or not latest_proj_info = mc.project_info(project_path) @@ -155,15 +157,13 @@ def download_project_async(mc, project_path, directory, project_version=None): project_info = latest_proj_info except ClientError: - _cleanup_failed_download(directory, mp) + _cleanup_failed_download(tmp_dir.name, mp) raise version = project_info["version"] if project_info["version"] else "v0" mp.log.info(f"got project info. version {version}") - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") - # prepare download update_tasks = [] # stuff to do at the end of download for file in project_info["files"]: @@ -182,9 +182,7 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info(f"will download {len(update_tasks)} files in {len(download_list)} chunks, total size {total_size}") - job = DownloadJob( - project_path, total_size, version, update_tasks, download_list, directory, mp, project_info, tmp_dir - ) + job = DownloadJob(project_path, total_size, version, update_tasks, download_list, tmp_dir, mp, project_info) # start download job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) From e692a9b8da38137c73bb30a9e72f5da4b782b393 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:31:56 +0200 Subject: [PATCH 09/22] use directory properly --- mergin/client_pull.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index ae6c31b5..ec01eb15 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -213,7 +213,7 @@ def download_project_is_running(job): traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) job.mp.log.info("--- download aborted") - job.failure_log_file = _cleanup_failed_download(job.directory, job.mp) + job.failure_log_file = _cleanup_failed_download(job.directory.name, job.mp) raise future.exception() if future.running(): return True @@ -239,7 +239,7 @@ def download_project_finalize(job): traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) job.mp.log.info("--- download aborted") - job.failure_log_file = _cleanup_failed_download(job.directory, job.mp) + job.failure_log_file = _cleanup_failed_download(job.directory.name, job.mp) raise future.exception() job.mp.log.info("--- download finished") @@ -251,8 +251,7 @@ def download_project_finalize(job): # final update of project metadata job.mp.update_metadata(job.project_info) - if job.download_tmp_dir: - job.download_tmp_dir.cleanup() + job.directory.cleanup() def download_project_cancel(job): From b3771e06766b97ca7f34f1c48890e456c44e2c27 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 13:32:09 +0200 Subject: [PATCH 10/22] update test --- mergin/test/test_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 1c350a11..b58cf63d 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2549,7 +2549,7 @@ def test_download_failure(mc): # download project async with pytest.raises(IsADirectoryError): job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.download_tmp_dir.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.directory.name, "base.gpkg.0")) download_project_wait(job) download_project_finalize(job) @@ -2561,7 +2561,7 @@ def test_download_failure(mc): # active waiting remove_folders([download_dir]) job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.download_tmp_dir.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.directory.name, "base.gpkg.0")) with pytest.raises(IsADirectoryError): while True: assert download_project_is_running(job) From fce855ebdf26e225d880eda62be1b1b43f0f210c Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 14:26:18 +0200 Subject: [PATCH 11/22] add options to TemporaryDirectory --- mergin/client_push.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_push.py b/mergin/client_push.py index 885db9ac..af92b865 100644 --- a/mergin/client_push.py +++ b/mergin/client_push.py @@ -124,7 +124,7 @@ def push_project_async(mc, directory): changes = filter_changes(mc, project_info, changes) mp.log.debug("push changes:\n" + pprint.pformat(changes)) - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-", ignore_cleanup_errors=True, delete=True) # If there are any versioned files (aka .gpkg) that are not updated through a diff, # we need to make a temporary copy somewhere to be sure that we are uploading full content. From ed5a89b3769c86375b35173077d34a5def6af396 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Thu, 7 Aug 2025 14:29:41 +0200 Subject: [PATCH 12/22] add options to TemporaryDirectory --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index ec01eb15..34b84aae 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -146,7 +146,7 @@ def download_project_async(mc, project_path, directory, project_version=None): mp.log.info("--- version: " + mc.user_agent_info()) mp.log.info(f"--- start download {project_path}") - tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-") + tmp_dir = tempfile.TemporaryDirectory(prefix="python-api-client-", ignore_cleanup_errors=True, delete=True) try: # check whether we download the latest version or not From 24213e4677c2cfd730fff6a18a85cdbdda243880 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 8 Aug 2025 10:35:31 +0200 Subject: [PATCH 13/22] Temp dir pull backkup --- mergin/client_pull.py | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index a46e6f96..f93c7092 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -351,7 +351,7 @@ def __init__( self.version = version self.files_to_merge = files_to_merge # list of FileToMerge instances self.download_queue_items = download_queue_items - self.temp_dir = temp_dir # full path to temporary directory where we store downloaded files + self.temp_dir = temp_dir # TemporaryDirectory instance where we store downloaded files self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server @@ -413,8 +413,7 @@ def pull_project_async(mc, directory): # then we just download the whole file _pulling_file_with_diffs = lambda f: "diffs" in f and len(f["diffs"]) != 0 - temp_dir = mp.fpath_meta(f"fetch_{local_version}-{server_version}") - os.makedirs(temp_dir, exist_ok=True) + temp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-", ignore_cleanup_errors=True, delete=True) pull_changes = mp.get_pull_changes(server_info["files"]) mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes)) fetch_files = [] @@ -441,10 +440,10 @@ def pull_project_async(mc, directory): for file in fetch_files: diff_only = _pulling_file_with_diffs(file) - items = _download_items(file, temp_dir, diff_only) + items = _download_items(file, temp_dir.name, diff_only) # figure out destination path for the file - file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir, file["path"]))) + file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir.name, file["path"]))) basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) dest_file_path = os.path.join(file_dir, basename) os.makedirs(file_dir, exist_ok=True) @@ -465,8 +464,8 @@ def pull_project_async(mc, directory): file_path = file["path"] mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})") file["version"] = server_version - items = _download_items(file, temp_dir, diff_only=False) - dest_file_path = mp.fpath(file["path"], temp_dir) + items = _download_items(file, temp_dir.name, diff_only=False) + dest_file_path = mp.fpath(file["path"], temp_dir.name) # dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) files_to_merge.append(FileToMerge(dest_file_path, items)) continue @@ -604,10 +603,10 @@ def pull_project_finalize(job: PullJob): # download their full versions so we have them up-to-date for applying changes for file_path, file_diffs in job.basefiles_to_patch: basefile = job.mp.fpath_meta(file_path) - server_file = job.mp.fpath(file_path, job.temp_dir) + server_file = job.mp.fpath(file_path, job.temp_dir.name) shutil.copy(basefile, server_file) - diffs = [job.mp.fpath(f, job.temp_dir) for f in file_diffs] + diffs = [job.mp.fpath(f, job.temp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: # that's weird that we are unable to apply diffs to the basefile! @@ -623,7 +622,7 @@ def pull_project_finalize(job: PullJob): raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) try: - conflicts = job.mp.apply_pull_changes(job.pull_changes, job.temp_dir, job.project_info, job.mc) + conflicts = job.mp.apply_pull_changes(job.pull_changes, job.temp_dir.name, job.project_info, job.mc) except Exception as e: job.mp.log.error("Failed to apply pull changes: " + str(e)) job.mp.log.info("--- pull aborted") @@ -636,7 +635,7 @@ def pull_project_finalize(job: PullJob): else: job.mp.log.info("--- pull finished -- at version " + job.mp.version()) - shutil.rmtree(job.temp_dir) + job.temp_dir.cleanup() # delete our temporary dir and all its content return conflicts @@ -866,4 +865,4 @@ def download_files_finalize(job): # Remove temporary download directory if job.directory is not None and os.path.exists(job.directory): - shutil.rmtree(job.directory) + shutil.rmtree(job.directory) \ No newline at end of file From 9523f686ca8367a5b150d20983e0a403c2d601c2 Mon Sep 17 00:00:00 2001 From: "marcel.kocisek" Date: Fri, 8 Aug 2025 10:36:54 +0200 Subject: [PATCH 14/22] black --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index f93c7092..29336732 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -865,4 +865,4 @@ def download_files_finalize(job): # Remove temporary download directory if job.directory is not None and os.path.exists(job.directory): - shutil.rmtree(job.directory) \ No newline at end of file + shutil.rmtree(job.directory) From fb7f37192851293c30363d81119275a9fbac0102 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 14:11:17 +0200 Subject: [PATCH 15/22] rename to temp_directory --- mergin/client_pull.py | 4 ++-- mergin/test/test_client.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 34b84aae..ff92b07a 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -50,7 +50,7 @@ def __init__( version, update_tasks, download_queue_items, - directory: tempfile.TemporaryDirectory, + temp_directory: tempfile.TemporaryDirectory, mp, project_info, ): @@ -60,7 +60,7 @@ def __init__( self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.directory = directory # project's directory + self.temp_directory = temp_directory # project's directory self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index b58cf63d..2d183abf 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2549,7 +2549,7 @@ def test_download_failure(mc): # download project async with pytest.raises(IsADirectoryError): job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.directory.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.temp_directory.name, "base.gpkg.0")) download_project_wait(job) download_project_finalize(job) @@ -2561,7 +2561,7 @@ def test_download_failure(mc): # active waiting remove_folders([download_dir]) job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.directory.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.temp_directory.name, "base.gpkg.0")) with pytest.raises(IsADirectoryError): while True: assert download_project_is_running(job) From fc57d4f635b5f6278d5f6d7f3e1ad2f981d433c1 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 14:19:06 +0200 Subject: [PATCH 16/22] remove directory param from _cleanup_failed_download() and do not delete from there --- mergin/client_pull.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index ff92b07a..9f249945 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -104,7 +104,7 @@ def _do_download(item, mc, mp, project_path, job): job.transferred_size += item.size -def _cleanup_failed_download(directory, mergin_project=None): +def _cleanup_failed_download(mergin_project: MerginProject = None): """ If a download job fails, there will be the newly created directory left behind with some temporary files in it. We want to remove it because a new download would fail because @@ -126,7 +126,6 @@ def _cleanup_failed_download(directory, mergin_project=None): dest_path = tmp_file.name shutil.copyfile(log_file, dest_path) - shutil.rmtree(directory) return dest_path @@ -157,7 +156,7 @@ def download_project_async(mc, project_path, directory, project_version=None): project_info = latest_proj_info except ClientError: - _cleanup_failed_download(tmp_dir.name, mp) + _cleanup_failed_download(mp) raise version = project_info["version"] if project_info["version"] else "v0" @@ -213,7 +212,7 @@ def download_project_is_running(job): traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) job.mp.log.info("--- download aborted") - job.failure_log_file = _cleanup_failed_download(job.directory.name, job.mp) + job.failure_log_file = _cleanup_failed_download(job.mp) raise future.exception() if future.running(): return True @@ -239,7 +238,7 @@ def download_project_finalize(job): traceback_lines = traceback.format_exception(type(exc), exc, exc.__traceback__) job.mp.log.error("Error while downloading project: " + "".join(traceback_lines)) job.mp.log.info("--- download aborted") - job.failure_log_file = _cleanup_failed_download(job.directory.name, job.mp) + job.failure_log_file = _cleanup_failed_download(job.mp) raise future.exception() job.mp.log.info("--- download finished") From ab44d6aec680e7c462c0724d1d557761f2785a81 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 16:00:19 +0200 Subject: [PATCH 17/22] fix error --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 9f249945..12130681 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -250,7 +250,7 @@ def download_project_finalize(job): # final update of project metadata job.mp.update_metadata(job.project_info) - job.directory.cleanup() + job.temp_directory.cleanup() def download_project_cancel(job): From 613d0a9e193c2cdf1418f54489dfb42c360f0f95 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 16:00:58 +0200 Subject: [PATCH 18/22] Update mergin/client_pull.py Co-authored-by: Martin Dobias --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 12130681..783837e7 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -60,7 +60,7 @@ def __init__( self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.temp_directory = temp_directory # project's directory + self.temp_directory = temp_directory self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server From fd82bd227b9f17627e8fb8da8a2faae0b6b1fdd2 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 16:17:23 +0200 Subject: [PATCH 19/22] fix variable --- mergin/client_pull.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index 783837e7..cca1534c 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -872,8 +872,8 @@ def download_files_finalize(job): job.mp.log.info("--- download finished") for task in job.update_tasks: - task.apply(job.directory, job.mp) + task.apply(job.temp_directory, job.mp) # Remove temporary download directory - if job.directory is not None and os.path.exists(job.directory): - shutil.rmtree(job.directory) + if job.temp_directory is not None and os.path.exists(job.temp_directory): + shutil.rmtree(job.temp_directory) From a6f4a6ab81cb4ade84c3b3e7a55eb05d1a760f2d Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Mon, 1 Sep 2025 16:17:36 +0200 Subject: [PATCH 20/22] remove --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index cca1534c..f210df1c 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -60,7 +60,7 @@ def __init__( self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.temp_directory = temp_directory + self.temp_directory = temp_directory # project's directory self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server From 36249cb891fd9828d0c09d1cb74ab19aa240ccc0 Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Tue, 2 Sep 2025 13:14:40 +0200 Subject: [PATCH 21/22] remove comment Co-authored-by: Martin Dobias --- mergin/client_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index f210df1c..cca1534c 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -60,7 +60,7 @@ def __init__( self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.temp_directory = temp_directory # project's directory + self.temp_directory = temp_directory self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server From c47d45a07d868bc5f27ca7e7d6474beabd76c08e Mon Sep 17 00:00:00 2001 From: Jan Caha Date: Tue, 2 Sep 2025 13:21:48 +0200 Subject: [PATCH 22/22] renam to tmp_dir --- mergin/client_pull.py | 46 +++++++++++++++++++------------------- mergin/test/test_client.py | 4 ++-- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/mergin/client_pull.py b/mergin/client_pull.py index cca1534c..c3eb422f 100644 --- a/mergin/client_pull.py +++ b/mergin/client_pull.py @@ -50,7 +50,7 @@ def __init__( version, update_tasks, download_queue_items, - temp_directory: tempfile.TemporaryDirectory, + tmp_dir: tempfile.TemporaryDirectory, mp, project_info, ): @@ -60,7 +60,7 @@ def __init__( self.version = version self.update_tasks = update_tasks self.download_queue_items = download_queue_items - self.temp_directory = temp_directory + self.tmp_dir = tmp_dir self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server @@ -250,7 +250,7 @@ def download_project_finalize(job): # final update of project metadata job.mp.update_metadata(job.project_info) - job.temp_directory.cleanup() + job.tmp_dir.cleanup() def download_project_cancel(job): @@ -347,7 +347,7 @@ def __init__( version, files_to_merge, download_queue_items, - temp_dir, + tmp_dir, mp, project_info, basefiles_to_patch, @@ -362,7 +362,7 @@ def __init__( self.version = version self.files_to_merge = files_to_merge # list of FileToMerge instances self.download_queue_items = download_queue_items - self.temp_dir = temp_dir # TemporaryDirectory instance where we store downloaded files + self.tmp_dir = tmp_dir # TemporaryDirectory instance where we store downloaded files self.mp = mp # MerginProject instance self.is_cancelled = False self.project_info = project_info # parsed JSON with project info returned from the server @@ -424,7 +424,7 @@ def pull_project_async(mc, directory): # then we just download the whole file _pulling_file_with_diffs = lambda f: "diffs" in f and len(f["diffs"]) != 0 - temp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-", ignore_cleanup_errors=True, delete=True) + tmp_dir = tempfile.TemporaryDirectory(prefix="mm-pull-", ignore_cleanup_errors=True, delete=True) pull_changes = mp.get_pull_changes(server_info["files"]) mp.log.debug("pull changes:\n" + pprint.pformat(pull_changes)) fetch_files = [] @@ -451,10 +451,10 @@ def pull_project_async(mc, directory): for file in fetch_files: diff_only = _pulling_file_with_diffs(file) - items = _download_items(file, temp_dir.name, diff_only) + items = _download_items(file, tmp_dir.name, diff_only) # figure out destination path for the file - file_dir = os.path.dirname(os.path.normpath(os.path.join(temp_dir.name, file["path"]))) + file_dir = os.path.dirname(os.path.normpath(os.path.join(tmp_dir.name, file["path"]))) basename = os.path.basename(file["diff"]["path"]) if diff_only else os.path.basename(file["path"]) dest_file_path = os.path.join(file_dir, basename) os.makedirs(file_dir, exist_ok=True) @@ -475,8 +475,8 @@ def pull_project_async(mc, directory): file_path = file["path"] mp.log.info(f"missing base file for {file_path} -> going to download it (version {server_version})") file["version"] = server_version - items = _download_items(file, temp_dir.name, diff_only=False) - dest_file_path = mp.fpath(file["path"], temp_dir.name) + items = _download_items(file, tmp_dir.name, diff_only=False) + dest_file_path = mp.fpath(file["path"], tmp_dir.name) # dest_file_path = os.path.join(os.path.dirname(os.path.normpath(os.path.join(temp_dir, file['path']))), os.path.basename(file['path'])) files_to_merge.append(FileToMerge(dest_file_path, items)) continue @@ -500,7 +500,7 @@ def pull_project_async(mc, directory): server_version, files_to_merge, download_list, - temp_dir, + tmp_dir, mp, server_info, basefiles_to_patch, @@ -614,10 +614,10 @@ def pull_project_finalize(job: PullJob): # download their full versions so we have them up-to-date for applying changes for file_path, file_diffs in job.basefiles_to_patch: basefile = job.mp.fpath_meta(file_path) - server_file = job.mp.fpath(file_path, job.temp_dir.name) + server_file = job.mp.fpath(file_path, job.tmp_dir.name) shutil.copy(basefile, server_file) - diffs = [job.mp.fpath(f, job.temp_dir.name) for f in file_diffs] + diffs = [job.mp.fpath(f, job.tmp_dir.name) for f in file_diffs] patch_error = job.mp.apply_diffs(server_file, diffs) if patch_error: # that's weird that we are unable to apply diffs to the basefile! @@ -633,7 +633,7 @@ def pull_project_finalize(job: PullJob): raise ClientError("Cannot patch basefile {}! Please try syncing again.".format(basefile)) try: - conflicts = job.mp.apply_pull_changes(job.pull_changes, job.temp_dir.name, job.project_info, job.mc) + conflicts = job.mp.apply_pull_changes(job.pull_changes, job.tmp_dir.name, job.project_info, job.mc) except Exception as e: job.mp.log.error("Failed to apply pull changes: " + str(e)) job.mp.log.info("--- pull aborted") @@ -646,7 +646,7 @@ def pull_project_finalize(job: PullJob): else: job.mp.log.info("--- pull finished -- at version " + job.mp.version()) - job.temp_dir.cleanup() # delete our temporary dir and all its content + job.tmp_dir.cleanup() # delete our temporary dir and all its content return conflicts @@ -798,7 +798,7 @@ def download_files_async( mp.log.info(f"Got project info. version {project_info['version']}") # set temporary directory for download - temp_dir = tempfile.mkdtemp(prefix="python-api-client-") + tmp_dir = tempfile.mkdtemp(prefix="python-api-client-") if output_paths is None: output_paths = [] @@ -808,7 +808,7 @@ def download_files_async( if len(output_paths) != len(file_paths): warn = "Output file paths are not of the same length as file paths. Cannot store required files." mp.log.warning(warn) - shutil.rmtree(temp_dir) + shutil.rmtree(tmp_dir) raise ClientError(warn) download_list = [] @@ -822,7 +822,7 @@ def download_files_async( if file["path"] in file_paths: index = file_paths.index(file["path"]) file["version"] = version - items = _download_items(file, temp_dir) + items = _download_items(file, tmp_dir) is_latest_version = version == latest_proj_info["version"] task = UpdateTask(file["path"], items, output_paths[index], latest_version=is_latest_version) download_list.extend(task.download_queue_items) @@ -842,13 +842,13 @@ def download_files_async( if not download_list or missing_files: warn = f"No [{', '.join(missing_files)}] exists at version {version}" mp.log.warning(warn) - shutil.rmtree(temp_dir) + shutil.rmtree(tmp_dir) raise ClientError(warn) mp.log.info( f"will download files [{', '.join(files_to_download)}] in {len(download_list)} chunks, total size {total_size}" ) - job = DownloadJob(project_path, total_size, version, update_tasks, download_list, temp_dir, mp, project_info) + job = DownloadJob(project_path, total_size, version, update_tasks, download_list, tmp_dir, mp, project_info) job.executor = concurrent.futures.ThreadPoolExecutor(max_workers=4) job.futures = [] for item in download_list: @@ -872,8 +872,8 @@ def download_files_finalize(job): job.mp.log.info("--- download finished") for task in job.update_tasks: - task.apply(job.temp_directory, job.mp) + task.apply(job.tmp_dir, job.mp) # Remove temporary download directory - if job.temp_directory is not None and os.path.exists(job.temp_directory): - shutil.rmtree(job.temp_directory) + if job.tmp_dir is not None and os.path.exists(job.tmp_dir): + shutil.rmtree(job.tmp_dir) diff --git a/mergin/test/test_client.py b/mergin/test/test_client.py index 2d183abf..d86e17fb 100644 --- a/mergin/test/test_client.py +++ b/mergin/test/test_client.py @@ -2549,7 +2549,7 @@ def test_download_failure(mc): # download project async with pytest.raises(IsADirectoryError): job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.temp_directory.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.tmp_dir.name, "base.gpkg.0")) download_project_wait(job) download_project_finalize(job) @@ -2561,7 +2561,7 @@ def test_download_failure(mc): # active waiting remove_folders([download_dir]) job = download_project_async(mc, project, download_dir) - os.makedirs(os.path.join(job.temp_directory.name, "base.gpkg.0")) + os.makedirs(os.path.join(job.tmp_dir.name, "base.gpkg.0")) with pytest.raises(IsADirectoryError): while True: assert download_project_is_running(job)