From 23f21eec4d4c495cd03d4a952bad0f61fdc58bfb Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Sat, 6 Sep 2025 16:18:51 +0100 Subject: [PATCH 1/6] operations/files.copy: basic copy functionality --- src/pyinfra/operations/files.py | 22 +++++++++++++++++++ tests/operations/files.copy/copies_file.json | 17 ++++++++++++++ .../files.copy/copies_file_overwriting.json | 17 ++++++++++++++ tests/operations/files.copy/invalid_dest.json | 20 +++++++++++++++++ .../files.copy/invalid_overwrite.json | 20 +++++++++++++++++ tests/operations/files.copy/invalid_src.json | 20 +++++++++++++++++ 6 files changed, 116 insertions(+) create mode 100644 tests/operations/files.copy/copies_file.json create mode 100644 tests/operations/files.copy/copies_file_overwriting.json create mode 100644 tests/operations/files.copy/invalid_dest.json create mode 100644 tests/operations/files.copy/invalid_overwrite.json create mode 100644 tests/operations/files.copy/invalid_src.json diff --git a/src/pyinfra/operations/files.py b/src/pyinfra/operations/files.py index 2d59cc85e..746842cee 100644 --- a/src/pyinfra/operations/files.py +++ b/src/pyinfra/operations/files.py @@ -1312,6 +1312,28 @@ def move(src: str, dest: str, overwrite=False): yield StringCommand("mv", QuoteString(src), QuoteString(dest)) +@operation() +def copy(src: str, dest: str, overwrite=False): + if not host.get_fact(File, src): + raise OperationError(f"src {src} does not exist") + + if not host.get_fact(Directory, dest): + raise OperationError(f"dest {dest} is not an existing directory") + + dest_file_path = os.path.join(dest, os.path.basename(src)) + dest_file_exists = host.get_fact(File, dest_file_path) + + if dest_file_exists and not overwrite: + raise OperationError( + f"dest {dest_file_path} already exists and `overwrite` is unset" + ) + + cp_cmd = ["cp"] + if overwrite: + cp_cmd.append("-f") + yield StringCommand(*cp_cmd, QuoteString(src), QuoteString(dest)) + + def _validate_path(path): try: return os.fspath(path) diff --git a/tests/operations/files.copy/copies_file.json b/tests/operations/files.copy/copies_file.json new file mode 100644 index 000000000..69578add2 --- /dev/null +++ b/tests/operations/files.copy/copies_file.json @@ -0,0 +1,17 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + "path=/tmp/dest_dir/file": null + }, + "files.Directory": { + "path=/tmp/dest_dir": true + } + }, + "commands": ["cp /tmp/src_dir/file /tmp/dest_dir"] +} diff --git a/tests/operations/files.copy/copies_file_overwriting.json b/tests/operations/files.copy/copies_file_overwriting.json new file mode 100644 index 000000000..12fbff5f5 --- /dev/null +++ b/tests/operations/files.copy/copies_file_overwriting.json @@ -0,0 +1,17 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": true + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + "path=/tmp/dest_dir/file": true + }, + "files.Directory": { + "path=/tmp/dest_dir": true + } + }, + "commands": ["cp -f /tmp/src_dir/file /tmp/dest_dir"] +} diff --git a/tests/operations/files.copy/invalid_dest.json b/tests/operations/files.copy/invalid_dest.json new file mode 100644 index 000000000..077bce76f --- /dev/null +++ b/tests/operations/files.copy/invalid_dest.json @@ -0,0 +1,20 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + "path=/tmp/dest_dir/file": null + }, + "files.Directory": { + "path=/tmp/dest_dir": null + } + }, + "exception": { + "name": "OperationError", + "message": "dest /tmp/dest_dir is not an existing directory" + } +} diff --git a/tests/operations/files.copy/invalid_overwrite.json b/tests/operations/files.copy/invalid_overwrite.json new file mode 100644 index 000000000..fdf2a9668 --- /dev/null +++ b/tests/operations/files.copy/invalid_overwrite.json @@ -0,0 +1,20 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + "path=/tmp/dest_dir/file": true + }, + "files.Directory": { + "path=/tmp/dest_dir": true + } + }, + "exception": { + "name": "OperationError", + "message": "dest /tmp/dest_dir/file already exists and `overwrite` is unset" + } +} diff --git a/tests/operations/files.copy/invalid_src.json b/tests/operations/files.copy/invalid_src.json new file mode 100644 index 000000000..e70d7cc91 --- /dev/null +++ b/tests/operations/files.copy/invalid_src.json @@ -0,0 +1,20 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": null, + "path=/tmp/dest_dir/file": null + }, + "files.Directory": { + "path=/tmp/dest_dir": true + } + }, + "exception": { + "name": "OperationError", + "message": "src /tmp/src_dir/file does not exist" + } +} From 78c79664a5c77791cd29c422b67a5aaf387572e9 Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Sat, 6 Sep 2025 16:42:58 +0100 Subject: [PATCH 2/6] operations/files.copy: copy dirs --- src/pyinfra/operations/files.py | 12 +++++------ .../files.copy/copies_directory.json | 21 +++++++++++++++++++ .../copies_directory_overwriting.json | 21 +++++++++++++++++++ tests/operations/files.copy/copies_file.json | 5 +++-- .../files.copy/copies_file_overwriting.json | 5 +++-- tests/operations/files.copy/invalid_dest.json | 3 ++- .../files.copy/invalid_overwrite.json | 3 ++- tests/operations/files.copy/invalid_src.json | 3 ++- 8 files changed, 60 insertions(+), 13 deletions(-) create mode 100644 tests/operations/files.copy/copies_directory.json create mode 100644 tests/operations/files.copy/copies_directory_overwriting.json diff --git a/src/pyinfra/operations/files.py b/src/pyinfra/operations/files.py index 746842cee..d37a1ec76 100644 --- a/src/pyinfra/operations/files.py +++ b/src/pyinfra/operations/files.py @@ -1314,7 +1314,8 @@ def move(src: str, dest: str, overwrite=False): @operation() def copy(src: str, dest: str, overwrite=False): - if not host.get_fact(File, src): + src_is_dir = host.get_fact(Directory, src) + if not host.get_fact(File, src) and not src_is_dir: raise OperationError(f"src {src} does not exist") if not host.get_fact(Directory, dest): @@ -1322,15 +1323,14 @@ def copy(src: str, dest: str, overwrite=False): dest_file_path = os.path.join(dest, os.path.basename(src)) dest_file_exists = host.get_fact(File, dest_file_path) - if dest_file_exists and not overwrite: - raise OperationError( - f"dest {dest_file_path} already exists and `overwrite` is unset" - ) + raise OperationError(f"dest {dest_file_path} already exists and `overwrite` is unset") + + cp_cmd = ["cp -r"] - cp_cmd = ["cp"] if overwrite: cp_cmd.append("-f") + yield StringCommand(*cp_cmd, QuoteString(src), QuoteString(dest)) diff --git a/tests/operations/files.copy/copies_directory.json b/tests/operations/files.copy/copies_directory.json new file mode 100644 index 000000000..d09f899d2 --- /dev/null +++ b/tests/operations/files.copy/copies_directory.json @@ -0,0 +1,21 @@ +{ + "kwargs": { + "src": "/tmp/src_dir", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + + "path=/tmp/src_dir": null, + "path=/tmp/dest_dir/src_dir": null, + "path=/tmp/dest_dir/src_dir/file": null + }, + "files.Directory": { + "path=/tmp/src_dir": true, + "path=/tmp/dest_dir": true + } + }, + "commands": ["cp -r /tmp/src_dir /tmp/dest_dir"] +} diff --git a/tests/operations/files.copy/copies_directory_overwriting.json b/tests/operations/files.copy/copies_directory_overwriting.json new file mode 100644 index 000000000..91a93e663 --- /dev/null +++ b/tests/operations/files.copy/copies_directory_overwriting.json @@ -0,0 +1,21 @@ +{ + "kwargs": { + "src": "/tmp/src_dir", + "dest": "/tmp/dest_dir", + "overwrite": true + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + + "path=/tmp/src_dir": null, + "path=/tmp/dest_dir/src_dir": null, + "path=/tmp/dest_dir/src_dir/file": null + }, + "files.Directory": { + "path=/tmp/src_dir": true, + "path=/tmp/dest_dir": true + } + }, + "commands": ["cp -r -f /tmp/src_dir /tmp/dest_dir"] +} diff --git a/tests/operations/files.copy/copies_file.json b/tests/operations/files.copy/copies_file.json index 69578add2..4f46c02ee 100644 --- a/tests/operations/files.copy/copies_file.json +++ b/tests/operations/files.copy/copies_file.json @@ -10,8 +10,9 @@ "path=/tmp/dest_dir/file": null }, "files.Directory": { - "path=/tmp/dest_dir": true + "path=/tmp/dest_dir": true, + "path=/tmp/src_dir/file": null } }, - "commands": ["cp /tmp/src_dir/file /tmp/dest_dir"] + "commands": ["cp -r /tmp/src_dir/file /tmp/dest_dir"] } diff --git a/tests/operations/files.copy/copies_file_overwriting.json b/tests/operations/files.copy/copies_file_overwriting.json index 12fbff5f5..7b7494400 100644 --- a/tests/operations/files.copy/copies_file_overwriting.json +++ b/tests/operations/files.copy/copies_file_overwriting.json @@ -10,8 +10,9 @@ "path=/tmp/dest_dir/file": true }, "files.Directory": { - "path=/tmp/dest_dir": true + "path=/tmp/dest_dir": true, + "path=/tmp/src_dir/file": null } }, - "commands": ["cp -f /tmp/src_dir/file /tmp/dest_dir"] + "commands": ["cp -r -f /tmp/src_dir/file /tmp/dest_dir"] } diff --git a/tests/operations/files.copy/invalid_dest.json b/tests/operations/files.copy/invalid_dest.json index 077bce76f..63399aeff 100644 --- a/tests/operations/files.copy/invalid_dest.json +++ b/tests/operations/files.copy/invalid_dest.json @@ -10,7 +10,8 @@ "path=/tmp/dest_dir/file": null }, "files.Directory": { - "path=/tmp/dest_dir": null + "path=/tmp/dest_dir": null, + "path=/tmp/src_dir/file": null } }, "exception": { diff --git a/tests/operations/files.copy/invalid_overwrite.json b/tests/operations/files.copy/invalid_overwrite.json index fdf2a9668..d9be3cc8c 100644 --- a/tests/operations/files.copy/invalid_overwrite.json +++ b/tests/operations/files.copy/invalid_overwrite.json @@ -10,7 +10,8 @@ "path=/tmp/dest_dir/file": true }, "files.Directory": { - "path=/tmp/dest_dir": true + "path=/tmp/dest_dir": true, + "path=/tmp/src_dir/file": null } }, "exception": { diff --git a/tests/operations/files.copy/invalid_src.json b/tests/operations/files.copy/invalid_src.json index e70d7cc91..b77a44170 100644 --- a/tests/operations/files.copy/invalid_src.json +++ b/tests/operations/files.copy/invalid_src.json @@ -10,7 +10,8 @@ "path=/tmp/dest_dir/file": null }, "files.Directory": { - "path=/tmp/dest_dir": true + "path=/tmp/dest_dir": true, + "path=/tmp/src_dir/file": null } }, "exception": { From 6dd6cd262ddbad98b84a5ec3c0ce134befea11ca Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Sat, 6 Sep 2025 17:18:14 +0100 Subject: [PATCH 3/6] operations/files.copy: docs --- src/pyinfra/operations/files.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/pyinfra/operations/files.py b/src/pyinfra/operations/files.py index d37a1ec76..ad273784f 100644 --- a/src/pyinfra/operations/files.py +++ b/src/pyinfra/operations/files.py @@ -1314,6 +1314,13 @@ def move(src: str, dest: str, overwrite=False): @operation() def copy(src: str, dest: str, overwrite=False): + """ + Copy remote file/directory/link into remote directory + + + src: remote file/directory to copy + + dest: remote directory to copy `src` into + + overwrite: whether to overwrite dest, if present + """ src_is_dir = host.get_fact(Directory, src) if not host.get_fact(File, src) and not src_is_dir: raise OperationError(f"src {src} does not exist") From 823719dec2b29cff5fca95c9e78a4799cec6d257 Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Mon, 8 Dec 2025 09:24:58 +0000 Subject: [PATCH 4/6] operations/files.copy: noop if file already exists --- src/pyinfra/operations/files.py | 6 ++++- .../files.copy/invalid_overwrite.json | 11 ++++++++- .../files.copy/noop_dest_file_exists.json | 23 +++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 tests/operations/files.copy/noop_dest_file_exists.json diff --git a/src/pyinfra/operations/files.py b/src/pyinfra/operations/files.py index ad273784f..3b3698f4f 100644 --- a/src/pyinfra/operations/files.py +++ b/src/pyinfra/operations/files.py @@ -1331,7 +1331,11 @@ def copy(src: str, dest: str, overwrite=False): dest_file_path = os.path.join(dest, os.path.basename(src)) dest_file_exists = host.get_fact(File, dest_file_path) if dest_file_exists and not overwrite: - raise OperationError(f"dest {dest_file_path} already exists and `overwrite` is unset") + if _file_equal(src, dest_file_path): + host.noop(f"{dest_file_path} already exists") + return + else: + raise OperationError(f"{dest_file_path} already exists and is different than src") cp_cmd = ["cp -r"] diff --git a/tests/operations/files.copy/invalid_overwrite.json b/tests/operations/files.copy/invalid_overwrite.json index d9be3cc8c..ce476c314 100644 --- a/tests/operations/files.copy/invalid_overwrite.json +++ b/tests/operations/files.copy/invalid_overwrite.json @@ -4,6 +4,11 @@ "dest": "/tmp/dest_dir", "overwrite": false }, + "local_files": { + "files": { + "/tmp/src_dir/file": "some content" + } + }, "facts": { "files.File": { "path=/tmp/src_dir/file": true, @@ -12,10 +17,14 @@ "files.Directory": { "path=/tmp/dest_dir": true, "path=/tmp/src_dir/file": null + }, + "files.Sha1File": { + "path=/tmp/src_dir/file": "abc", + "path=/tmp/dest_dir/file": "xyz" } }, "exception": { "name": "OperationError", - "message": "dest /tmp/dest_dir/file already exists and `overwrite` is unset" + "message": "/tmp/dest_dir/file already exists and is different than src" } } diff --git a/tests/operations/files.copy/noop_dest_file_exists.json b/tests/operations/files.copy/noop_dest_file_exists.json new file mode 100644 index 000000000..c7d73a622 --- /dev/null +++ b/tests/operations/files.copy/noop_dest_file_exists.json @@ -0,0 +1,23 @@ +{ + "kwargs": { + "src": "/tmp/src_dir/file", + "dest": "/tmp/dest_dir", + "overwrite": false + }, + "facts": { + "files.File": { + "path=/tmp/src_dir/file": true, + "path=/tmp/dest_dir/file": true + }, + "files.Directory": { + "path=/tmp/dest_dir": true, + "path=/tmp/src_dir/file": null + }, + "files.Sha1File": { + "path=/tmp/src_dir/file": "94e66df8cd09d410c62d9e0dc59d3a884e458e05", + "path=/tmp/dest_dir/file": "94e66df8cd09d410c62d9e0dc59d3a884e458e05" + } + }, + "commands": [], + "noop_description": "/tmp/dest_dir/file already exists" +} From 3e9798b28050d32cb88e438160b2043108698e2f Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Mon, 8 Dec 2025 10:26:36 +0000 Subject: [PATCH 5/6] operations/files.copy: compare files locally --- src/pyinfra/operations/files.py | 11 ++++++++++- tests/operations/files.copy/invalid_overwrite.json | 5 ----- .../operations/files.copy/noop_dest_file_exists.json | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/pyinfra/operations/files.py b/src/pyinfra/operations/files.py index 3b3698f4f..ab5ba6cdc 100644 --- a/src/pyinfra/operations/files.py +++ b/src/pyinfra/operations/files.py @@ -751,6 +751,15 @@ def _file_equal(local_path: str | IO[Any] | None, remote_path: str) -> bool: return False +def _remote_file_equal(remote_path_a: str, remote_path_b: str) -> bool: + for fact in [Sha1File, Md5File, Sha256File]: + sum_a = host.get_fact(fact, path=remote_path_a) + sum_b = host.get_fact(fact, path=remote_path_b) + if sum_a and sum_b: + return sum_a == sum_b + return False + + @operation( # We don't (currently) cache the local state, so there's nothing we can # update to flag the local file as present. @@ -1331,7 +1340,7 @@ def copy(src: str, dest: str, overwrite=False): dest_file_path = os.path.join(dest, os.path.basename(src)) dest_file_exists = host.get_fact(File, dest_file_path) if dest_file_exists and not overwrite: - if _file_equal(src, dest_file_path): + if _remote_file_equal(src, dest_file_path): host.noop(f"{dest_file_path} already exists") return else: diff --git a/tests/operations/files.copy/invalid_overwrite.json b/tests/operations/files.copy/invalid_overwrite.json index ce476c314..edd81fe8e 100644 --- a/tests/operations/files.copy/invalid_overwrite.json +++ b/tests/operations/files.copy/invalid_overwrite.json @@ -4,11 +4,6 @@ "dest": "/tmp/dest_dir", "overwrite": false }, - "local_files": { - "files": { - "/tmp/src_dir/file": "some content" - } - }, "facts": { "files.File": { "path=/tmp/src_dir/file": true, diff --git a/tests/operations/files.copy/noop_dest_file_exists.json b/tests/operations/files.copy/noop_dest_file_exists.json index c7d73a622..6b31c0421 100644 --- a/tests/operations/files.copy/noop_dest_file_exists.json +++ b/tests/operations/files.copy/noop_dest_file_exists.json @@ -14,8 +14,8 @@ "path=/tmp/src_dir/file": null }, "files.Sha1File": { - "path=/tmp/src_dir/file": "94e66df8cd09d410c62d9e0dc59d3a884e458e05", - "path=/tmp/dest_dir/file": "94e66df8cd09d410c62d9e0dc59d3a884e458e05" + "path=/tmp/src_dir/file": "abc", + "path=/tmp/dest_dir/file": "abc" } }, "commands": [], From aa600db0a106758ae2c0536a1865bccec19a5ea1 Mon Sep 17 00:00:00 2001 From: Eric Driussi Date: Mon, 8 Dec 2025 10:49:36 +0000 Subject: [PATCH 6/6] operations/files.copy: skip win in tests --- tests/operations/files.copy/copies_directory.json | 1 + tests/operations/files.copy/copies_directory_overwriting.json | 2 +- tests/operations/files.copy/copies_file.json | 1 + tests/operations/files.copy/copies_file_overwriting.json | 1 + tests/operations/files.copy/invalid_overwrite.json | 1 + tests/operations/files.copy/noop_dest_file_exists.json | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/operations/files.copy/copies_directory.json b/tests/operations/files.copy/copies_directory.json index d09f899d2..19b5d0b92 100644 --- a/tests/operations/files.copy/copies_directory.json +++ b/tests/operations/files.copy/copies_directory.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir", "dest": "/tmp/dest_dir", diff --git a/tests/operations/files.copy/copies_directory_overwriting.json b/tests/operations/files.copy/copies_directory_overwriting.json index 91a93e663..0bc7c53f8 100644 --- a/tests/operations/files.copy/copies_directory_overwriting.json +++ b/tests/operations/files.copy/copies_directory_overwriting.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir", "dest": "/tmp/dest_dir", @@ -7,7 +8,6 @@ "facts": { "files.File": { "path=/tmp/src_dir/file": true, - "path=/tmp/src_dir": null, "path=/tmp/dest_dir/src_dir": null, "path=/tmp/dest_dir/src_dir/file": null diff --git a/tests/operations/files.copy/copies_file.json b/tests/operations/files.copy/copies_file.json index 4f46c02ee..f37991ff1 100644 --- a/tests/operations/files.copy/copies_file.json +++ b/tests/operations/files.copy/copies_file.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir/file", "dest": "/tmp/dest_dir", diff --git a/tests/operations/files.copy/copies_file_overwriting.json b/tests/operations/files.copy/copies_file_overwriting.json index 7b7494400..9c37cadb7 100644 --- a/tests/operations/files.copy/copies_file_overwriting.json +++ b/tests/operations/files.copy/copies_file_overwriting.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir/file", "dest": "/tmp/dest_dir", diff --git a/tests/operations/files.copy/invalid_overwrite.json b/tests/operations/files.copy/invalid_overwrite.json index edd81fe8e..9ff71a326 100644 --- a/tests/operations/files.copy/invalid_overwrite.json +++ b/tests/operations/files.copy/invalid_overwrite.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir/file", "dest": "/tmp/dest_dir", diff --git a/tests/operations/files.copy/noop_dest_file_exists.json b/tests/operations/files.copy/noop_dest_file_exists.json index 6b31c0421..5bc769ade 100644 --- a/tests/operations/files.copy/noop_dest_file_exists.json +++ b/tests/operations/files.copy/noop_dest_file_exists.json @@ -1,4 +1,5 @@ { + "require_platform": ["Darwin", "Linux"], "kwargs": { "src": "/tmp/src_dir/file", "dest": "/tmp/dest_dir",