diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 420fd9c..472c1af 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -24,7 +24,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 25b7ad2..d704cc1 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install ShellCheck run: sudo apt-get install -y shellcheck @@ -26,7 +26,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install hadolint run: | @@ -45,7 +45,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Install yamllint run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d6e1b79..cc3a71c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 @@ -131,7 +131,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 diff --git a/Dockerfile b/Dockerfile index 34c101b..3321dad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -50,8 +50,8 @@ RUN apt-get update && \ unzip=6.0-28ubuntu6 \ nano=8.3-1 \ vim=2:9.1.0967-1ubuntu4.1 \ - python3.13=3.13.3-1ubuntu0.3 \ - python3.13-venv=3.13.3-1ubuntu0.3 \ + python3.13=3.13.3-1ubuntu0.4 \ + python3.13-venv=3.13.3-1ubuntu0.4 \ python3-pip=25.0+dfsg-1ubuntu0.2 \ supervisor=4.2.5-3 && \ # Install Azure CLI in venv with optimizations for scanning @@ -76,16 +76,16 @@ RUN echo $TZ > /etc/timezone && \ # Install yq (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then ARCH="amd64"; elif [ "$ARCH" = "aarch64" ]; then ARCH="arm64"; fi && \ - curl -sL https://github.com/mikefarah/yq/releases/download/v4.48.2/yq_linux_${ARCH}.tar.gz | tar xz && \ + curl -sL https://github.com/mikefarah/yq/releases/download/v4.49.2/yq_linux_${ARCH}.tar.gz | tar xz && \ mv yq_linux_${ARCH} /usr/bin/yq && \ rm -rf /tmp/* # Install Google Cloud SDK (architecture-aware) RUN ARCH=$(uname -m) && \ if [ "$ARCH" = "x86_64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-547.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-548.0.0-linux-x86_64.tar.gz" -o google-cloud-sdk.tar.gz; \ elif [ "$ARCH" = "aarch64" ]; then \ - curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-547.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ + curl -sSL "https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-548.0.0-linux-arm.tar.gz" -o google-cloud-sdk.tar.gz; \ fi && \ tar -xzf google-cloud-sdk.tar.gz && \ ./google-cloud-sdk/install.sh -q && \ diff --git a/deploy.yml b/deploy.yml new file mode 100644 index 0000000..e4453fb --- /dev/null +++ b/deploy.yml @@ -0,0 +1,27 @@ +# npm install -g @udx/worker-deployment +# gcloud auth login +# gcloud auth application-default login +# worker-run + +--- +kind: workerDeployConfig +version: udx.io/worker-v1/deploy +config: + # Docker image + image: "usabilitydynamics/udx-worker:latest" + + env: + TEST_ENV_SECRET: "gcp/rabbit-ci-dev/worker-secret-test" + + # Mount volumes + # volumes: + # - "./worker.yaml:/home/udx/.config/worker/worker.yaml" + + # ports: + # - "80:80" + + # Command to run + # command: "/usr/local/bin/init.sh" + + service_account: + email: "worker-site@rabbit-ci-dev.iam.gserviceaccount.com" diff --git a/docs/config.md b/docs/config.md index 2b29e1e..6f043fa 100644 --- a/docs/config.md +++ b/docs/config.md @@ -3,6 +3,7 @@ ## Overview The UDX Worker uses `worker.yaml` as its primary configuration file, allowing you to: + - Define environment variables - Reference secrets from various providers - Configure worker behavior @@ -15,12 +16,12 @@ The UDX Worker uses `worker.yaml` as its primary configuration file, allowing yo ## Configuration Structure -| Section | Purpose | Required | -|---------|----------|----------| -| `kind` | Configuration type identifier | Yes | -| `version` | Schema version | Yes | -| `config.env` | Environment variables | No | -| `config.secrets` | Secret references | No | +| Section | Purpose | Required | +| ---------------- | ----------------------------- | -------- | +| `kind` | Configuration type identifier | Yes | +| `version` | Schema version | Yes | +| `config.env` | Environment variables | No | +| `config.secrets` | Secret references | No | ## Basic Example @@ -72,6 +73,10 @@ secrets: ## Environment Variables +Environment variables can be defined in two ways: + +### 1. Direct Values + ```yaml config: env: @@ -79,26 +84,50 @@ config: AZURE_TENANT_ID: "tenant-id" AWS_REGION: "us-west-2" GCP_PROJECT: "my-project" - + # Application Settings LOG_LEVEL: "info" MAX_WORKERS: "5" ENABLE_METRICS: "true" ``` +### 2. Secret References + +Environment variables can also reference secrets using the same provider format as the `secrets` section: + +```yaml +config: + env: + # Reference secrets directly in env variables + DATABASE_URL: "gcp/my-project/db-connection-string" + API_TOKEN: "azure/kv-prod/api-token" + AWS_SECRET_KEY: "aws/prod/secret-access-key" + VAULT_PASSWORD: "bitwarden/prod/vault-pass" + + # Mix with regular values + LOG_LEVEL: "info" +``` + +The worker will automatically detect secret references (format: `provider/vault/secret`) in environment variables and resolve them at runtime. + ## Best Practices 1. **Secret Management** - - Never store sensitive values directly in `env` - - Use `secrets` section for sensitive data - - Reference secrets from appropriate providers + + - Never store sensitive values as plain text + - Use either `config.secrets` section OR secret references in `config.env` + - Both methods support the same provider format: `provider/vault/secret` + - Choose based on your preference: + - `config.secrets`: Explicit separation of secrets + - `config.env` with references: Unified configuration 2. **Environment Variables** - - Use `env` for non-sensitive configuration + + - Use `env` for non-sensitive configuration OR secret references - Keep values consistent across environments - Document any required variables 3. **File Handling** - Keep configuration in version control (without sensitive data) - Use different files for different environments - - Validate configuration before deployment \ No newline at end of file + - Validate configuration before deployment diff --git a/lib/environment.sh b/lib/environment.sh index 58f570c..0da68d4 100644 --- a/lib/environment.sh +++ b/lib/environment.sh @@ -66,6 +66,13 @@ configure_environment() { log_info "No secrets defined in the configuration." fi + # Fetch secrets from environment variables with provider prefixes + log_info "Checking for secret references in environment variables..." + if ! fetch_secrets_from_env_vars; then + log_error "Environment" "Failed to fetch secrets from environment variables." + return 1 + fi + # Perform cleanup log_info "Cleaning up sensitive data..." if ! cleanup_actors; then diff --git a/lib/secrets.sh b/lib/secrets.sh index 69ce54f..39b8d62 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -5,6 +5,33 @@ source "${WORKER_LIB_DIR}/utils.sh" # shellcheck source=${WORKER_LIB_DIR}/env_handler.sh disable=SC1091 source "${WORKER_LIB_DIR}/env_handler.sh" +# System variables to skip when scanning for secret references +# Only declare if not already defined (prevents errors when sourced multiple times) +if [[ -z "${SYSTEM_VARS+x}" ]]; then + readonly SYSTEM_VARS=( + "HOME" "USER" "PATH" "SHELL" "TERM" "LANG" "PWD" "SHLVL" "_" + "PS1" "HOSTNAME" "UID" "GID" "OLDPWD" "LS_COLORS" "DEBIAN_FRONTEND" + "LOGNAME" "MAIL" "TMPDIR" "SSH_CONNECTION" "SSH_CLIENT" "SSH_TTY" + ) +fi + +# Worker internal variables to skip (exact matches and prefixes) +if [[ -z "${WORKER_INTERNAL_VARS+x}" ]]; then + readonly WORKER_INTERNAL_VARS=( + "AZURE_CONFIG_DIR" + "AWS_CONFIG_FILE" + "GCP_CREDS" + "TZ" + ) +fi + +if [[ -z "${WORKER_INTERNAL_PREFIXES+x}" ]]; then + readonly WORKER_INTERNAL_PREFIXES=( + "WORKER_" + "CLOUDSDK_" + ) +fi + # Dynamically source the required provider-specific modules source_provider_module() { local provider="$1" @@ -129,5 +156,151 @@ resolve_secret_by_name() { fi } +# Function to detect if a value is a secret reference +is_secret_reference() { + local value="$1" + + # Check if value matches pattern: provider/vault/secret + # Supported providers: gcp, azure, aws, bitwarden + if [[ "$value" =~ ^(gcp|azure|aws|bitwarden)/.+/.+ ]]; then + return 0 + fi + return 1 +} + +# Function to check if a variable should be skipped +should_skip_variable() { + local var_name="$1" + + # Check against system variables + for sys_var in "${SYSTEM_VARS[@]}"; do + if [[ "$var_name" == "$sys_var" ]]; then + return 0 + fi + done + + # Check against worker internal variables + for internal_var in "${WORKER_INTERNAL_VARS[@]}"; do + if [[ "$var_name" == "$internal_var" ]]; then + return 0 + fi + done + + # Check against worker internal prefixes + for prefix in "${WORKER_INTERNAL_PREFIXES[@]}"; do + if [[ "$var_name" == ${prefix}* ]]; then + return 0 + fi + done + + return 1 +} + +# Function to fetch secrets from environment variables +fetch_secrets_from_env_vars() { + local processed_vars=() + local -a secret_keys=() + local -a secret_values=() + + # Helper function to collect a secret reference + collect_secret() { + local var_name="$1" + local var_value="$2" + + # Check if the value is a secret reference + if ! is_secret_reference "$var_value"; then + return 0 + fi + + log_info "Found secret reference in $var_name: $var_value" + + # Collect key-value pair + secret_keys+=("$var_name") + secret_values+=("$var_value") + } + + # 1. Process environment variables from worker.yaml (in WORKER_ENV_FILE) + if [[ -f "$WORKER_ENV_FILE" ]]; then + while IFS= read -r line; do + # Skip comments and empty lines + [[ "$line" =~ ^#.*$ || -z "$line" ]] && continue + + # Extract variable name and value + if [[ "$line" =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then + local var_name="${BASH_REMATCH[1]}" + local var_value="${BASH_REMATCH[2]}" + + # Track that we've processed this variable + processed_vars+=("$var_name") + + # Collect if it's a secret reference + collect_secret "$var_name" "$var_value" + fi + done < "$WORKER_ENV_FILE" + fi + + # 2. Process deployment environment variables (from container environment) + while IFS='=' read -r key value; do + # Skip if already processed from worker config + local already_processed=false + for processed_var in "${processed_vars[@]}"; do + if [[ "$key" == "$processed_var" ]]; then + already_processed=true + break + fi + done + if [[ "$already_processed" == "true" ]]; then + continue + fi + + # Skip system and worker internal variables + if should_skip_variable "$key"; then + continue + fi + + # Collect if it's a secret reference + collect_secret "$key" "$value" + done < <(env) + + # Build JSON from collected secrets in a single jq invocation + local secrets_json="{}" + if [[ ${#secret_keys[@]} -gt 0 ]]; then + # Build jq arguments for all key-value pairs + local jq_args=() + for i in "${!secret_keys[@]}"; do + jq_args+=(--arg "key$i" "${secret_keys[$i]}" --arg "val$i" "${secret_values[$i]}") + done + + # Build jq filter to construct object with all pairs + local jq_filter="." + for i in "${!secret_keys[@]}"; do + jq_filter="$jq_filter | . + {\$key$i: \$val$i}" + done + + # Construct JSON in single jq invocation + secrets_json=$(echo '{}' | jq "${jq_args[@]}" "$jq_filter") + fi + + # If no secrets found, return early + if [[ "$secrets_json" == "{}" ]]; then + log_info "No secret references found in environment variables." + return 0 + fi + + # Validate JSON (should always be valid since jq built it) + if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then + log_error "Secrets" "Invalid JSON format for collected secrets" + return 1 + fi + + # Use existing append_resolved_secrets function to resolve and append + if ! append_resolved_secrets "$secrets_json"; then + log_error "Secrets" "Failed to resolve secrets from environment variables" + return 1 + fi + + return 0 +} + # Example usage: # fetch_secrets '{"TEST": "gcp/new_relic_api_key"}'