From 772eba170a9062988ceae67e3576d466e31057dd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 1 Dec 2025 16:51:43 +0100 Subject: [PATCH 1/7] wip:ssh for gitsync --- .pre-commit-config.yaml | 16 +++---- Cargo.lock | 16 +++---- Cargo.nix | 44 +++++++++--------- Cargo.toml | 8 ++-- crate-hashes.json | 14 +++--- deploy/helm/airflow-operator/crds/crds.yaml | 27 ++++++++++- ...aml => example-airflow-gitsync-https.yaml} | 9 ++++ .../examples/example-airflow-gitsync-ssh.yaml | 19 ++++++++ .../pages/usage-guide/mounting-dags.adoc | 33 ++++++++++---- nix/sources.json | 6 +-- .../operator-binary/src/airflow_controller.rs | 3 ++ rust/operator-binary/src/crd/mod.rs | 4 ++ .../30-install-airflow-cluster.yaml.j2 | 45 ++++++++++++++----- .../mount-dags-gitsync/31-assert.yaml.j2 | 4 ++ tests/test-definition.yaml | 5 +++ 15 files changed, 180 insertions(+), 73 deletions(-) rename docs/modules/airflow/examples/{example-airflow-gitsync.yaml => example-airflow-gitsync-https.yaml} (87%) create mode 100644 docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5461297f..f88ad40f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # 6.0.0 + rev: v6.0.0 # 6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -16,26 +16,26 @@ repos: - repo: https://github.com/EmbarkStudios/cargo-deny # Failing to compile cargo-deny with version >0.18.3, needs rust 1.88.0 - rev: baa02b0a0c54e0578aae6bb7c7181ad00dc290af # 0.18.3 + rev: 0.18.6 # 0.18.3 hooks: - id: cargo-deny args: ["--all-features", "check", "advisories", "bans", "licenses", "sources"] - repo: https://github.com/adrienverge/yamllint - rev: 79a6b2b1392eaf49cdd32ac4f14be1a809bbd8f7 # 1.37.1 + rev: v1.37.1 # 1.37.1 hooks: - id: yamllint args: ["--strict"] - repo: https://github.com/igorshubovych/markdownlint-cli - rev: 192ad822316c3a22fb3d3cc8aa6eafa0b8488360 # 0.45.0 + rev: v0.46.0 # 0.45.0 hooks: - id: markdownlint types: [text] files: \.md(\.j2)*$ - repo: https://github.com/koalaman/shellcheck-precommit - rev: 99470f5e12208ff0fb17ab81c3c494f7620a1d8d # 0.11.0 + rev: v0.11.0 # 0.11.0 hooks: - id: shellcheck args: ["--severity=info"] @@ -44,7 +44,7 @@ repos: # If you do not, you will need to delete the cached ruff binary shown in the # error message - repo: https://github.com/astral-sh/ruff-pre-commit - rev: 3b4bc031619cde2e0a9f3c4441ac7cc8227245a4 # 0.14.1 + rev: v0.14.7 # 0.14.1 hooks: # Run the linter. - id: ruff-check @@ -52,12 +52,12 @@ repos: - id: ruff-format - repo: https://github.com/rhysd/actionlint - rev: e7d448ef7507c20fc4c88a95d0c448b848cd6127 # 1.7.8 + rev: v1.7.9 # 1.7.8 hooks: - id: actionlint - repo: https://github.com/hadolint/hadolint - rev: 57e1618d78fd469a92c1e584e8c9313024656623 # 2.14.0 + rev: v2.14.0 # 2.14.0 hooks: - id: hadolint diff --git a/Cargo.lock b/Cargo.lock index 12987668..e6318618 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "regex", @@ -2481,8 +2481,8 @@ dependencies = [ [[package]] name = "stackable-operator" -version = "0.100.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +version = "0.100.3" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "clap", @@ -2520,7 +2520,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "darling", "proc-macro2", @@ -2531,7 +2531,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "chrono", "k8s-openapi", @@ -2548,7 +2548,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "axum", "clap", @@ -2572,7 +2572,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "schemars", "serde", @@ -2585,7 +2585,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#184423db7409f853bd69db7eeecd1affbf97ef40" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index e04eb219..e06f37de 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4187,9 +4187,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "k8s_version"; authors = [ @@ -8089,13 +8089,13 @@ rec { }; "stackable-operator" = rec { crateName = "stackable-operator"; - version = "0.100.1"; + version = "0.100.3"; edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_operator"; authors = [ @@ -8262,9 +8262,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8297,9 +8297,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_shared"; authors = [ @@ -8379,9 +8379,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_telemetry"; authors = [ @@ -8489,9 +8489,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; libName = "stackable_versioned"; authors = [ @@ -8533,9 +8533,9 @@ rec { edition = "2024"; workspace_member = null; src = pkgs.fetchgit { - url = "https://github.com/stackabletech/operator-rs.git"; - rev = "184423db7409f853bd69db7eeecd1affbf97ef40"; - sha256 = "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l"; + url = "https://github.com/stackabletech//operator-rs.git"; + rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; + sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/Cargo.toml b/Cargo.toml index 8945301c..be57cf89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,7 @@ repository = "https://github.com/stackabletech/airflow-operator" [workspace.dependencies] product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.8.0" } -stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.1" } +stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", features = ["telemetry", "versioned"], tag = "stackable-operator-0.100.3" } anyhow = "1.0" base64 = "0.22" @@ -32,6 +32,6 @@ strum = { version = "0.27", features = ["derive"] } tokio = { version = "1.40", features = ["full"] } tracing = "0.1" -# [patch."https://github.com/stackabletech/operator-rs.git"] -# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" } -# stackable-operator = { path = "../operator-rs/crates/stackable-operator" } +[patch."https://github.com/stackabletech/operator-rs.git"] +stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "feat/gitsync-ssh" } +#stackable-operator = { path = "../operator-rs/crates/stackable-operator" } diff --git a/crate-hashes.json b/crate-hashes.json index 1c58f755..576d5189 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,10 @@ { - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#k8s-version@0.1.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator-derive@0.3.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-operator@0.100.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-shared@0.0.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-telemetry@0.6.1": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned-macros@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", - "git+https://github.com/stackabletech/operator-rs.git?tag=stackable-operator-0.100.1#stackable-versioned@0.8.3": "1a98klljvifnc168f1wc3d6szcry1lamxgjjdq89plr99p4b953l", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index b8e45ff8..e728bcd2 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -680,6 +680,7 @@ spec: The referenced Secret must include two fields: `user` and `password`. The `password` field can either be an actual password (not recommended) or a GitHub token, as described in the git-sync [documentation]. + This cannot be provided if `ssh_secret` is also provided. [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual nullable: true @@ -711,9 +712,19 @@ spec: [example]: https://docs.stackable.tech/home/nightly/airflow/usage-guide/mounting-dags#_example type: object repo: - description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator`.' + description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string + sshSecret: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + This cannot be provided if `credentials_secret` is also provided. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string wait: default: 20s description: |- @@ -1602,6 +1613,20 @@ spec: type: object x-kubernetes-preserve-unknown-fields: true type: object + objectOverrides: + default: [] + description: |- + A list of generic Kubernetes objects, which are merged into the objects that the operator + creates. + + List entries are arbitrary YAML objects, which need to be valid Kubernetes objects. + + Read the [Object overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#object-overrides) + for more information. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array schedulers: description: |- The `schedulers` is responsible for triggering jobs and persisting their metadata to the backend database. diff --git a/docs/modules/airflow/examples/example-airflow-gitsync.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml similarity index 87% rename from docs/modules/airflow/examples/example-airflow-gitsync.yaml rename to docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 6a4b5e46..49cabcfe 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -24,3 +24,12 @@ spec: --git-config: http.sslCAInfo:/tmp/ca-cert/ca.crt # <11> webservers: ... +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-credentials # <8> +type: Opaque +data: + user: c3Rh... + password: Z2l0a... diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml new file mode 100644 index 00000000..9ed7d7de --- /dev/null +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: +... + dagsGitSync: + - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> + sshSecret: git-sync-ssh # <2> +--- +apiVersion: v1 +kind: Secret +metadata: + name: git-sync-ssh # <2> +type: Opaque +data: + key: LS0tL... + knownHosts: Z2l0a... diff --git a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc index 4c1f3e1c..8b7aa8ff 100644 --- a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc +++ b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc @@ -1,6 +1,6 @@ = Mounting DAGs :description: Mount DAGs in Airflow via ConfigMap for single DAGs or use git-sync for multiple DAGs. git-sync pulls from a Git repo and handles updates automatically. -:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.1 +:git-sync: https://github.com/kubernetes/git-sync/tree/v4.2.4 DAGs can be mounted by using a ConfigMap or `git-sync`. This is best illustrated with an example of each, shown in the sections below. @@ -24,21 +24,31 @@ include::example$example-airflow-dags-configmap.yaml[] <7> The resource has to be defined using `subPath`: this is to prevent the versioning of ConfigMap elements which may cause a conflict with how Airflow propagates DAGs between its components. <8> If the mount path described above is anything other than the standard location (the default is `$AIRFLOW_HOME/dags`), then the location should be defined using the relevant environment variable. -WARNING: If a DAG mounted via ConfigMap consists of modularized files then using the standard location is mandatory as Python uses this as a "root" directory when looking for referenced files. +[WARNING] +-- +If a DAG mounted via ConfigMap consists of modularized files, Python uses this as a "root" directory when looking for referenced files. +If this is the case, then either the standard DAGs location should be used, or `PYTHONPATH` should be overriden to point to the new location (it is also necessary to include the logging configuration in the path) as shown below: +[source,yaml] +---- + envOverrides: &envOverrides + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + PYTHONPATH: "/stackable/app/log_config:/dags" +---- +-- The advantage of this approach is that DAGs are provided "in-line". However, handling multiple DAGs this way becomes cumbersome, as each must be mapped individually. -For multiple DAGs, it is easier to expose them via a mounted volume, as shown below. +For multiple DAGs, it is easier to expose them via `gitsync`, as shown below. == Via `git-sync` {git-sync}[git-sync] is a command that pulls a git repository into a local directory and is supplied as a sidecar container for use within Kubernetes. The Stackable Airflow images already ship with git-sync included, and the operator takes care of calling the tool and mounting volumes, so that only the repository and synchronization details are required: -.git-sync usage example +.git-sync usage example: https [source,yaml] ---- -include::example$example-airflow-gitsync.yaml[] +include::example$example-airflow-gitsync-https.yaml[] ---- <1> A Secret used for accessing database and admin user details (included here to illustrate where different credential secrets are defined) @@ -60,10 +70,17 @@ include::example$example-airflow-gitsync.yaml[] <11> Git-sync settings can be provided inline, although some of these (`--dest`, `--root`) are specified internally in the operator and are ignored if provided by the user. Git-config settings can also be specified, although a warning is logged if `safe.directory` is specified as this is defined internally, and should not be defined by the user. +.git-sync usage example: ssh +[source,yaml] +---- +include::example$example-airflow-gitsync-ssh.yaml[] +---- + +<1> The name of the Secret used to access the repository if it is not public. + This should include two fields: `key` and `knownHosts`, both of which can contain multiple entries. +<2> The secret referenced above. -IMPORTANT: The example shows a _list_ of git-sync definitions, with a single element. -This is to avoid breaking-changes in future releases. -Currently, only one such git-sync definition is considered and processed. +IMPORTANT: Gitsync will not allow both `credentialsSecret` and `sshSecret` to be declared and the operator will throw an exception if this is attempted. NOTE: git-sync can be used with DAGs that make use of Python modules, as Python is configured to use the git-sync target folder as the "root" location when looking for referenced files. See the xref:usage-guide/applying-custom-resources.adoc[] example for more details. diff --git a/nix/sources.json b/nix/sources.json index f79c2cf0..dcfe59fa 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -29,10 +29,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a7fc11be66bdfb5cdde611ee5ce381c183da8386", - "sha256": "0h3gvjbrlkvxhbxpy01n603ixv0pjy19n9kf73rdkchdvqcn70j2", + "rev": "5c46f3bd98147c8d82366df95bbef2cab3a967ea", + "sha256": "1q085irshssrpnryw9kixd0smc2c0rhqs20niklybaywpv2zlywx", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/a7fc11be66bdfb5cdde611ee5ce381c183da8386.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/5c46f3bd98147c8d82366df95bbef2cab3a967ea.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index e9e17d34..5f161213 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -436,6 +436,7 @@ pub async fn reconcile_airflow( AIRFLOW_CONTROLLER_NAME, &airflow.object_ref(&()), ClusterResourceApplyStrategy::from(&airflow.spec.cluster_operation), + airflow.spec.object_overrides.clone(), ) .context(CreateClusterResourcesSnafu)?; @@ -1452,6 +1453,8 @@ fn add_git_sync_resources( } pb.add_volumes(git_sync_resources.git_content_volumes.to_owned()) .context(AddVolumeSnafu)?; + pb.add_volumes(git_sync_resources.git_ssh_volumes.to_owned()) + .context(AddVolumeSnafu)?; cb.add_volume_mounts(git_sync_resources.git_content_volume_mounts.to_owned()) .context(AddVolumeMountSnafu)?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index aa1a234f..a7529513 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -20,6 +20,7 @@ use stackable_operator::{ merge::Merge, }, crd::git_sync, + deep_merger::ObjectOverrides, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, @@ -198,6 +199,9 @@ pub mod versioned { // no doc string - See ProductImage struct pub image: ProductImage, + #[serde(default)] + pub object_overrides: ObjectOverrides, + /// Configuration that applies to all roles and role groups. /// This includes settings for authentication, git sync, service exposition and volumes, among other things. pub cluster_config: v1alpha1::AirflowClusterConfig, diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 7d5d098f..d3bfa1ca 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -3,23 +3,20 @@ kind: TestStep metadata: name: install-airflow timeout: 480 +{% if test_scenario['values']['access'] == 'ssh' %} --- apiVersion: v1 kind: Secret metadata: - name: test-airflow-credentials + name: git-sync-ssh type: Opaque -stringData: - adminUser.username: airflow - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: airflow - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow -{% if test_scenario['values']['executor'] == 'celery' %} - connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow - connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +data: + # This is a combination of a private key and known hosts (github.com) for the owner of the repo (stackable-airflow/dags). + # Contact github users @razvan or @adwk67 for details. + key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBRFdmd0hlaHhaQXdqcEk0OXN0eEM2SmJNWVJLODEvRVdueFh2NnV4aUhLUUFBQUpCSUEzTWJTQU56Ckd3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQURXZndIZWh4WkF3anBJNDlzdHhDNkpiTVlSSzgxL0VXbnhYdjZ1eGlIS1EKQUFBRUFRcVZSOW9hNi91MzZTeC9DSFRxbExMa3A2dmVnc2dha3d1RHRJNFdJcHRnTlovQWQ2SEZrRENPa2pqMnkzRUxvbApzeGhFcnpYOFJhZkZlL3E3R0ljcEFBQUFEV2RwZEMxemVXNWpMWFJsYzNRPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K + knownHosts: Z2l0aHViLmNvbSBzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNqN25kTnhRb3dnY1FuanNoY0xycVBFaWlwaG50K1ZUVHZEUDZtSEJMOWoxYU5Va1k0VWUxZ3Z3bkdMVmxPaEdlWXJuWmFNZ1JLNitQS0NVWGFEYkM3cXRiVzhnSWtoTDdhR0NzT3IvQzU2U0pNeS9CQ1pmeGQxbld6QU94U0RQZ1ZzbWVyT0JZZk5xbHRWOS9oV0NxQnl3SU5JUis1ZElnNkpUSjcycGNFcEVqY1lnWGtFMllFRlhWMUpIbnNLZ2JMV05saFNjcWIyVW15UmtReXl0Ukx0TCszOFRHeGt4Q2ZsbU8rNVo4Q1NTTlk3R2lkak1JWjdRNHpNakEybjFuR3JsVERrendEQ3N3K3dxRlBHUUExNzljbmZHV09XUlZydWoxNno2WHl2eHZqSndiejB3UVo3NVhLNXRLU2I3Rk55ZUlFczRUVDRqaytTNGRoUGVBVUM1eStiRFlpcllnTTRHQzd1RW56dG5aeWFWV1E3QjM4MUFLNFFkcnd0NTFacUV4S2JRcFRVTm4rRWpxb1R3dnFOajRrcXg1UVVDSTBUaFMvWWtPeEpDWG1QVVdaYmhqcENnNTZpKzJhQjZDbUsySkdobjU3SzVtajBNTmRCWEE0L1dud0g2WG9QV0p6SzVOeXUyekIzbkFacCtTNWhwUXMrcDF2TjEvd3Nqaz0KZ2l0aHViLmNvbSBlY2RzYS1zaGEyLW5pc3RwMjU2IEFBQUFFMlZqWkhOaExYTm9ZVEl0Ym1semRIQXlOVFlBQUFBSWJtbHpkSEF5TlRZQUFBQkJCRW1LU0VOalFFZXpPbXhrWk15N29wS2d3RkI5bmt0NVlScllNak51RzVOODd1UmdnNkNMcmJvNXdBZFQveTZ2MG1LVjBVMncwV1oyWUIvKytUcG9ja2c9CmdpdGh1Yi5jb20gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9NcXFua1Z6cm0wU2RHNlVPb3FLTHNhYmdINUM5b2tXaTBkaDJsOUdLSmwK {% endif %} +{% if test_scenario['values']['access'] == 'https' %} --- apiVersion: v1 kind: Secret @@ -32,6 +29,24 @@ data: # This token doesn't expire. user: c3RhY2thYmxlLWFpcmZsb3c= password: Z2l0aHViX3BhdF8xMUJLUURCRVkwSk1EWlNVQk1RYTdoX0c2OGlhbWtpRkpFV1RMTTF0ajFwbHFTVFNyZ3p3dHZneXI5b2tubGRXaGpVRDZITFRFV0JJcm9yT0dXCg== +{% endif %} +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow +{% if test_scenario['values']['executor'] == 'celery' %} + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +{% endif %} --- apiVersion: v1 kind: ConfigMap @@ -60,13 +75,19 @@ spec: {% endif %} credentialsSecret: test-airflow-credentials dagsGitSync: +{% if test_scenario['values']['access'] == 'ssh' %} + - repo: ssh://git@github.com/stackable-airflow/dags.git + sshSecret: git-sync-ssh +{% endif %} +{% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags + credentialsSecret: git-credentials +{% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) branch: main wait: 5s {% endif %} - credentialsSecret: git-credentials gitSyncConf: # supply some config to check that safe.directory is correctly set --git-config: http.sslVerify:false diff --git a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 index 8c7e90a8..cbe76bf8 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/31-assert.yaml.j2 @@ -11,8 +11,10 @@ commands: # will expect 2 (two containers, base and gitsync) - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "AIRFLOW_TEST_VAR" | wc -l | grep 2 # will expect 1 (one container, gitsync) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_USERNAME" | wc -l | grep 1 - script: kubectl -n $NAMESPACE get cm airflow-executor-pod-template -o json | jq -r '.data."airflow_executor_pod_template.yaml"' | grep "GITSYNC_PASSWORD" | wc -l | grep 1 +{% endif %} {% else %} # check that the statefulset contains mounts and envs # will expect 6 (2 from from the volume declaration + mounts to 3 containers, base and 2 gitsyncs, plus configmap restarter) @@ -20,6 +22,8 @@ commands: # will expect 3 (two containers, base and gitsync-1, and one initContainer gitsync-0) - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "AIRFLOW_TEST_VAR" | wc -l | grep 3 # will expect 2 (one container, gitsync-1, and one initContainer gitsync-0) +{% if test_scenario['values']['access'] == 'https' %} - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_USERNAME" | wc -l | grep 2 - script: kubectl -n $NAMESPACE get sts airflow-worker-default -o json | grep "GITSYNC_PASSWORD" | wc -l | grep 2 {% endif %} +{% endif %} diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 39c632b3..367ddeae 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -33,6 +33,10 @@ dimensions: values: - celery - kubernetes + - name: access + values: + - https + - ssh tests: - name: smoke dimensions: @@ -49,6 +53,7 @@ tests: - airflow-latest - openshift - executor + - access - name: ldap dimensions: - airflow-latest From 2beef1d70f4953031c6d35210dd565fa761feff3 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 2 Dec 2025 10:25:08 +0100 Subject: [PATCH 2/7] linting, use deploy key --- .../modules/airflow/examples/example-airflow-gitsync-ssh.yaml | 4 +++- .../mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml index 9ed7d7de..52c0a0f9 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -4,10 +4,12 @@ kind: AirflowCluster metadata: name: airflow spec: -... + clusterConfig: dagsGitSync: - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> sshSecret: git-sync-ssh # <2> +... + --- apiVersion: v1 kind: Secret diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index d3bfa1ca..3897ad62 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -11,9 +11,9 @@ metadata: name: git-sync-ssh type: Opaque data: - # This is a combination of a private key and known hosts (github.com) for the owner of the repo (stackable-airflow/dags). + # This is a combination of a read-only deploy key and known hosts (github.com) for the repo (stackable-airflow/dags). # Contact github users @razvan or @adwk67 for details. - key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNBRFdmd0hlaHhaQXdqcEk0OXN0eEM2SmJNWVJLODEvRVdueFh2NnV4aUhLUUFBQUpCSUEzTWJTQU56Ckd3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQURXZndIZWh4WkF3anBJNDlzdHhDNkpiTVlSSzgxL0VXbnhYdjZ1eGlIS1EKQUFBRUFRcVZSOW9hNi91MzZTeC9DSFRxbExMa3A2dmVnc2dha3d1RHRJNFdJcHRnTlovQWQ2SEZrRENPa2pqMnkzRUxvbApzeGhFcnpYOFJhZkZlL3E3R0ljcEFBQUFEV2RwZEMxemVXNWpMWFJsYzNRPQotLS0tLUVORCBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0K + key: LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFBQUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhPUUFBQUNDTUpoLzdaUDFUWDBHem5sZG0zV3p3TW9Ed0xTVzh4dm9SUmpIZGVYTVViQUFBQUtDc3QzQ1NyTGR3CmtnQUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDQ01KaC83WlAxVFgwR3pubGRtM1d6d01vRHdMU1c4eHZvUlJqSGRlWE1VYkEKQUFBRUQwSUtCOG1wMWlScXFVSlVjTDh5Y0tKWkRwZkRnYWRXUHVhcUxYQkkvTVY0d21IL3RrL1ZOZlFiT2VWMmJkYlBBeQpnUEF0SmJ6RytoRkdNZDE1Y3hSc0FBQUFGbVJsY0d4dmVTMXJaWGt0Wm05eUxXZHBkSE41Ym1NQkFnTUVCUVlICi0tLS0tRU5EIE9QRU5TU0ggUFJJVkFURSBLRVktLS0tLQo= knownHosts: Z2l0aHViLmNvbSBzc2gtcnNhIEFBQUFCM056YUMxeWMyRUFBQUFEQVFBQkFBQUJnUUNqN25kTnhRb3dnY1FuanNoY0xycVBFaWlwaG50K1ZUVHZEUDZtSEJMOWoxYU5Va1k0VWUxZ3Z3bkdMVmxPaEdlWXJuWmFNZ1JLNitQS0NVWGFEYkM3cXRiVzhnSWtoTDdhR0NzT3IvQzU2U0pNeS9CQ1pmeGQxbld6QU94U0RQZ1ZzbWVyT0JZZk5xbHRWOS9oV0NxQnl3SU5JUis1ZElnNkpUSjcycGNFcEVqY1lnWGtFMllFRlhWMUpIbnNLZ2JMV05saFNjcWIyVW15UmtReXl0Ukx0TCszOFRHeGt4Q2ZsbU8rNVo4Q1NTTlk3R2lkak1JWjdRNHpNakEybjFuR3JsVERrendEQ3N3K3dxRlBHUUExNzljbmZHV09XUlZydWoxNno2WHl2eHZqSndiejB3UVo3NVhLNXRLU2I3Rk55ZUlFczRUVDRqaytTNGRoUGVBVUM1eStiRFlpcllnTTRHQzd1RW56dG5aeWFWV1E3QjM4MUFLNFFkcnd0NTFacUV4S2JRcFRVTm4rRWpxb1R3dnFOajRrcXg1UVVDSTBUaFMvWWtPeEpDWG1QVVdaYmhqcENnNTZpKzJhQjZDbUsySkdobjU3SzVtajBNTmRCWEE0L1dud0g2WG9QV0p6SzVOeXUyekIzbkFacCtTNWhwUXMrcDF2TjEvd3Nqaz0KZ2l0aHViLmNvbSBlY2RzYS1zaGEyLW5pc3RwMjU2IEFBQUFFMlZqWkhOaExYTm9ZVEl0Ym1semRIQXlOVFlBQUFBSWJtbHpkSEF5TlRZQUFBQkJCRW1LU0VOalFFZXpPbXhrWk15N29wS2d3RkI5bmt0NVlScllNak51RzVOODd1UmdnNkNMcmJvNXdBZFQveTZ2MG1LVjBVMncwV1oyWUIvKytUcG9ja2c9CmdpdGh1Yi5jb20gc3NoLWVkMjU1MTkgQUFBQUMzTnphQzFsWkRJMU5URTVBQUFBSU9NcXFua1Z6cm0wU2RHNlVPb3FLTHNhYmdINUM5b2tXaTBkaDJsOUdLSmwK {% endif %} {% if test_scenario['values']['access'] == 'https' %} From 5c6881e2e9b65881d0ece59666e5cd7605e350f1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 11:25:15 +0100 Subject: [PATCH 3/7] change credentials fields in line with operator-rs --- deploy/helm/airflow-operator/crds/crds.yaml | 101 +++++++----------- .../example-airflow-gitsync-https.yaml | 3 +- .../examples/example-airflow-gitsync-ssh.yaml | 3 +- .../pages/usage-guide/mounting-dags.adoc | 2 - .../30-install-airflow-cluster.yaml.j2 | 6 +- 5 files changed, 45 insertions(+), 70 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index e728bcd2..3dd59cbc 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,12 +115,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -143,7 +142,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -163,7 +161,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -385,12 +382,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -413,7 +409,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -433,7 +428,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -673,18 +667,34 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentialsSecret: - description: |- - The name of the Secret used to access the repository if it is not public. + credentials: + anyOf: + - required: + - basicAuthSecretName + - required: + - sshPrivateKeySecretName + description: An optional secret used for git access. + nullable: true + properties: + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - This cannot be provided if `ssh_secret` is also provided. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + type: object depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -715,16 +725,6 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string - sshSecret: - description: |- - The name of the Secret used for SSH access to the repository. - - The referenced Secret must include two fields: `key` and `knownHosts`. - This cannot be provided if `credentials_secret` is also provided. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string wait: default: 20s description: |- @@ -890,12 +890,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -918,7 +917,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -938,7 +936,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1160,12 +1157,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1188,7 +1184,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1208,7 +1203,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1458,12 +1452,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1486,7 +1479,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1506,7 +1498,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1707,12 +1698,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1735,7 +1725,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1755,7 +1744,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1977,12 +1965,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2005,7 +1992,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2025,7 +2011,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2220,12 +2205,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2248,7 +2232,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2268,7 +2251,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2490,12 +2472,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2518,7 +2499,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2538,7 +2518,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2733,12 +2712,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2761,7 +2739,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2781,7 +2758,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3008,12 +2984,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3036,7 +3011,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3056,7 +3030,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml index 49cabcfe..bfc5be08 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-https.yaml @@ -16,7 +16,8 @@ spec: gitFolder: "tests/templates/kuttl/mount-dags-gitsync/dags" # <5> depth: 10 # <6> wait: 20s # <7> - credentialsSecret: git-credentials # <8> + credentials: + basicAuthSecretName: git-credentials # <8> gitSyncConf: # <9> --rev: HEAD # <10> # --rev: git-sync-tag # N.B. tag must be covered by "depth" (the number of commits to clone) diff --git a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml index 52c0a0f9..f415f7c4 100644 --- a/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml +++ b/docs/modules/airflow/examples/example-airflow-gitsync-ssh.yaml @@ -7,7 +7,8 @@ spec: clusterConfig: dagsGitSync: - repo: ssh://git@github.com/stackable-airflow/dags.git # <1> - sshSecret: git-sync-ssh # <2> + credentials: + sshPrivateKeySecretName: git-sync-ssh # <2> ... --- diff --git a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc index 8b7aa8ff..cd03fc3b 100644 --- a/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc +++ b/docs/modules/airflow/pages/usage-guide/mounting-dags.adoc @@ -80,7 +80,5 @@ include::example$example-airflow-gitsync-ssh.yaml[] This should include two fields: `key` and `knownHosts`, both of which can contain multiple entries. <2> The secret referenced above. -IMPORTANT: Gitsync will not allow both `credentialsSecret` and `sshSecret` to be declared and the operator will throw an exception if this is attempted. - NOTE: git-sync can be used with DAGs that make use of Python modules, as Python is configured to use the git-sync target folder as the "root" location when looking for referenced files. See the xref:usage-guide/applying-custom-resources.adoc[] example for more details. diff --git a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 index 3897ad62..7b2b888b 100644 --- a/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/mount-dags-gitsync/30-install-airflow-cluster.yaml.j2 @@ -77,11 +77,13 @@ spec: dagsGitSync: {% if test_scenario['values']['access'] == 'ssh' %} - repo: ssh://git@github.com/stackable-airflow/dags.git - sshSecret: git-sync-ssh + credentials: + sshPrivateKeySecretName: git-sync-ssh {% endif %} {% if test_scenario['values']['access'] == 'https' %} - repo: https://github.com/stackable-airflow/dags - credentialsSecret: git-credentials + credentials: + basicAuthSecretName: git-credentials {% endif %} {% if test_scenario['values']['executor'] == 'celery' %} # Just setting some values to increase the test coverage (defaults should work just fine) From 436abeb9cee55ead70abf941ea926a1f7c25fe2c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 11:33:31 +0100 Subject: [PATCH 4/7] regenerate charts --- deploy/helm/airflow-operator/crds/crds.yaml | 101 +++++++++++++------- 1 file changed, 64 insertions(+), 37 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 3dd59cbc..e728bcd2 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,11 +115,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -142,6 +143,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -161,6 +163,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -382,11 +385,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -409,6 +413,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -428,6 +433,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -667,34 +673,18 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentials: - anyOf: - - required: - - basicAuthSecretName - - required: - - sshPrivateKeySecretName - description: An optional secret used for git access. - nullable: true - properties: - basicAuthSecretName: - description: |- - The name of the Secret used to access the repository via Basic Authentication if it is not public. - - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - sshPrivateKeySecretName: - description: |- - The name of the Secret used for SSH access to the repository. + credentialsSecret: + description: |- + The name of the Secret used to access the repository if it is not public. - The referenced Secret must include two fields: `key` and `knownHosts`. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. + This cannot be provided if `ssh_secret` is also provided. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - type: object + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -725,6 +715,16 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string + sshSecret: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + This cannot be provided if `credentials_secret` is also provided. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + nullable: true + type: string wait: default: 20s description: |- @@ -890,11 +890,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -917,6 +918,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -936,6 +938,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1157,11 +1160,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1184,6 +1188,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1203,6 +1208,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1452,11 +1458,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1479,6 +1486,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1498,6 +1506,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1698,11 +1707,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1725,6 +1735,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1744,6 +1755,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -1965,11 +1977,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1992,6 +2005,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2011,6 +2025,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2205,11 +2220,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2232,6 +2248,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2251,6 +2268,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2472,11 +2490,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2499,6 +2518,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2518,6 +2538,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2712,11 +2733,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2739,6 +2761,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2758,6 +2781,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -2984,11 +3008,12 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object custom: - description: Log configuration provided in a ConfigMap + description: Custom log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3011,6 +3036,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object @@ -3030,6 +3056,7 @@ spec: - ERROR - FATAL - NONE + - null nullable: true type: string type: object From e5a910ce41c0a85e8319e26ff2462accc1f8ddf0 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 12:21:20 +0100 Subject: [PATCH 5/7] bring enum in line with framework definition --- deploy/helm/airflow-operator/crds/crds.yaml | 111 +++++++++----------- 1 file changed, 47 insertions(+), 64 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index e728bcd2..f3352ae3 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -115,12 +115,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -143,7 +142,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -163,7 +161,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -385,12 +382,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -413,7 +409,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -433,7 +428,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -673,18 +667,44 @@ spec: Since git-sync v4.x.x this field is mapped to the flag `--ref`. type: string - credentialsSecret: - description: |- - The name of the Secret used to access the repository if it is not public. + credentials: + description: An optional secret used for git access. + nullable: true + oneOf: + - required: + - basicAuth + - required: + - ssh + properties: + basicAuth: + properties: + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. - This cannot be provided if `ssh_secret` is also provided. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + required: + - basicAuthSecretName + type: object + ssh: + properties: + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. + + The referenced Secret must include two fields: `key` and `knownHosts`. + + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + required: + - sshPrivateKeySecretName + type: object + type: object depth: default: 1 description: The depth of syncing, i.e. the number of commits to clone; defaults to 1. @@ -715,16 +735,6 @@ spec: description: 'The git repository URL that will be cloned, for example: `https://github.com/stackabletech/airflow-operator` or `ssh://git@github.com:stackable-airflow/dags.git`.' format: uri type: string - sshSecret: - description: |- - The name of the Secret used for SSH access to the repository. - - The referenced Secret must include two fields: `key` and `knownHosts`. - This cannot be provided if `credentials_secret` is also provided. - - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - nullable: true - type: string wait: default: 20s description: |- @@ -890,12 +900,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -918,7 +927,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -938,7 +946,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1160,12 +1167,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1188,7 +1194,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1208,7 +1213,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1458,12 +1462,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1486,7 +1489,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1506,7 +1508,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1707,12 +1708,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -1735,7 +1735,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1755,7 +1754,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -1977,12 +1975,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2005,7 +2002,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2025,7 +2021,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2220,12 +2215,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2248,7 +2242,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2268,7 +2261,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2490,12 +2482,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2518,7 +2509,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2538,7 +2528,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2733,12 +2722,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -2761,7 +2749,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -2781,7 +2768,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3008,12 +2994,11 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object custom: - description: Custom log configuration provided in a ConfigMap + description: Log configuration provided in a ConfigMap properties: configMap: description: ConfigMap containing the log configuration files @@ -3036,7 +3021,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object @@ -3056,7 +3040,6 @@ spec: - ERROR - FATAL - NONE - - null nullable: true type: string type: object From 576baf9c26d0b04d54ac481caf40855f33fad93d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 12:50:27 +0100 Subject: [PATCH 6/7] bring enum in line with framework definition: correction --- deploy/helm/airflow-operator/crds/crds.yaml | 42 ++++++++------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index f3352ae3..76c505e3 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -672,38 +672,28 @@ spec: nullable: true oneOf: - required: - - basicAuth + - basicAuthSecretName - required: - - ssh + - sshPrivateKeySecretName properties: - basicAuth: - properties: - basicAuthSecretName: - description: |- - The name of the Secret used to access the repository via Basic Authentication if it is not public. + basicAuthSecretName: + description: |- + The name of the Secret used to access the repository via Basic Authentication if it is not public. - The referenced Secret must include two fields: `user` and `password`. - The `password` field can either be an actual password (not recommended) or a GitHub token, - as described in the git-sync [documentation]. + The referenced Secret must include two fields: `user` and `password`. + The `password` field can either be an actual password (not recommended) or a GitHub token, + as described in the git-sync [documentation]. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - required: - - basicAuthSecretName - type: object - ssh: - properties: - sshPrivateKeySecretName: - description: |- - The name of the Secret used for SSH access to the repository. + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string + sshPrivateKeySecretName: + description: |- + The name of the Secret used for SSH access to the repository. - The referenced Secret must include two fields: `key` and `knownHosts`. + The referenced Secret must include two fields: `key` and `knownHosts`. - [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual - type: string - required: - - sshPrivateKeySecretName - type: object + [documentation]: https://github.com/kubernetes/git-sync/tree/v4.2.4?tab=readme-ov-file#manual + type: string type: object depth: default: 1 From 5e4d83879db551982c18e054af92aac24b6c771b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 18 Dec 2025 14:08:08 +0100 Subject: [PATCH 7/7] missed files --- Cargo.lock | 29 +++++++++------------- Cargo.nix | 63 +++++++++++++++++++++++++++++++++-------------- crate-hashes.json | 19 ++++++++------ 3 files changed, 68 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6318618..188416e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "k8s-version" version = "0.1.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "darling", "regex", @@ -1348,8 +1348,7 @@ dependencies = [ [[package]] name = "kube" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e7bb0b6a46502cc20e4575b6ff401af45cfea150b34ba272a3410b78aa014e" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "k8s-openapi", "kube-client", @@ -1361,8 +1360,7 @@ dependencies = [ [[package]] name = "kube-client" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4987d57a184d2b5294fdad3d7fc7f278899469d21a4da39a8f6ca16426567a36" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "base64", "bytes", @@ -1397,8 +1395,7 @@ dependencies = [ [[package]] name = "kube-core" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914bbb770e7bb721a06e3538c0edd2babed46447d128f7c21caa68747060ee73" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "chrono", "derive_more", @@ -1416,8 +1413,7 @@ dependencies = [ [[package]] name = "kube-derive" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03dee8252be137772a6ab3508b81cd797dee62ee771112a2453bc85cbbe150d2" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "darling", "proc-macro2", @@ -1430,8 +1426,7 @@ dependencies = [ [[package]] name = "kube-runtime" version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea4de4b562c5cc89ab10300bb63474ae1fa57ff5a19275f2e26401a323e3fd" +source = "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#26543e85dc7daaf82d8f7dbd902b26775798879e" dependencies = [ "ahash", "async-broadcast", @@ -2482,7 +2477,7 @@ dependencies = [ [[package]] name = "stackable-operator" version = "0.100.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "chrono", "clap", @@ -2520,7 +2515,7 @@ dependencies = [ [[package]] name = "stackable-operator-derive" version = "0.3.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "darling", "proc-macro2", @@ -2531,7 +2526,7 @@ dependencies = [ [[package]] name = "stackable-shared" version = "0.0.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "chrono", "k8s-openapi", @@ -2548,7 +2543,7 @@ dependencies = [ [[package]] name = "stackable-telemetry" version = "0.6.1" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "axum", "clap", @@ -2572,7 +2567,7 @@ dependencies = [ [[package]] name = "stackable-versioned" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "schemars", "serde", @@ -2585,7 +2580,7 @@ dependencies = [ [[package]] name = "stackable-versioned-macros" version = "0.8.3" -source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a" +source = "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#ec980f4862a9f2c4529cc1058056c53440268ac9" dependencies = [ "convert_case", "darling", diff --git a/Cargo.nix b/Cargo.nix index e06f37de..9aa43a86 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -4188,8 +4188,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "k8s_version"; authors = [ @@ -4220,7 +4220,12 @@ rec { crateName = "kube"; version = "2.0.1"; edition = "2024"; - sha256 = "0kh1m9w0nhd3fai4pcshl7z5rx0s83zvcxa51v12ql26d85vprs8"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; authors = [ "clux " "Natalie Klestrup Röijezon " @@ -4293,7 +4298,12 @@ rec { crateName = "kube-client"; version = "2.0.1"; edition = "2024"; - sha256 = "0dksaqk698bciyda6k8ss9lr92bqyb3pygddzna54asd31xdb1s9"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_client"; authors = [ "clux " @@ -4526,7 +4536,12 @@ rec { crateName = "kube-core"; version = "2.0.1"; edition = "2024"; - sha256 = "0wzfc1q78s5a3k1gfa6i8xjd9gmssbnw0f1mdsh23dvv1rvvnjwi"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_core"; authors = [ "clux " @@ -4608,7 +4623,12 @@ rec { crateName = "kube-derive"; version = "2.0.1"; edition = "2024"; - sha256 = "1ljhw6xmrj1v8ni144bpxrifwzbrrn0qnl5kd8m7fdz15cjyiph3"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; procMacro = true; libName = "kube_derive"; authors = [ @@ -4657,7 +4677,12 @@ rec { crateName = "kube-runtime"; version = "2.0.1"; edition = "2024"; - sha256 = "1zg34fih2r72y9sr58gmgyjizbkl6jv0nc0hmf4wrib2npj4vska"; + workspace_member = null; + src = pkgs.fetchgit { + url = "https://github.com/stackabletech/kube-rs"; + rev = "26543e85dc7daaf82d8f7dbd902b26775798879e"; + sha256 = "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf"; + }; libName = "kube_runtime"; authors = [ "clux " @@ -8094,8 +8119,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_operator"; authors = [ @@ -8263,8 +8288,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; procMacro = true; libName = "stackable_operator_derive"; @@ -8298,8 +8323,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_shared"; authors = [ @@ -8380,8 +8405,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_telemetry"; authors = [ @@ -8490,8 +8515,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; libName = "stackable_versioned"; authors = [ @@ -8534,8 +8559,8 @@ rec { workspace_member = null; src = pkgs.fetchgit { url = "https://github.com/stackabletech//operator-rs.git"; - rev = "3ab7a5f9aeb594a3bceb3220f970bccb7aac4c1a"; - sha256 = "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g"; + rev = "ec980f4862a9f2c4529cc1058056c53440268ac9"; + sha256 = "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8"; }; procMacro = true; libName = "stackable_versioned_macros"; diff --git a/crate-hashes.json b/crate-hashes.json index 576d5189..79f6e88b 100644 --- a/crate-hashes.json +++ b/crate-hashes.json @@ -1,10 +1,15 @@ { - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", - "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "1nm3w9pqn36s4hjhbprj284crxs4sq9srfqpn6fypjh2s8d0qx0g", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#k8s-version@0.1.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator-derive@0.3.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-operator@0.100.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-shared@0.0.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-telemetry@0.6.1": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned-macros@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech//operator-rs.git?branch=feat%2Fgitsync-ssh#stackable-versioned@0.8.3": "09rljv23m8rgi8142wa62md2vxm1b2d7rafj8ml6p2ndc02c79m8", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-client@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-core@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-derive@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube-runtime@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", + "git+https://github.com/stackabletech/kube-rs?branch=2.0.1-fix-schema-hoisting#kube@2.0.1": "1a7bcl0w1jg71jc4iml0vjp8dpzy71mhxl012grxcy2xp5i6xvgf", "git+https://github.com/stackabletech/product-config.git?tag=0.8.0#product-config@0.8.0": "1dz70kapm2wdqcr7ndyjji0lhsl98bsq95gnb2lw487wf6yr7987" } \ No newline at end of file