Enhanced Secret Fetching with Environment Variable Support

README.md

@@ -12,9 +12,9 @@ UDX Worker is a containerized solution that simplifies DevSecOps by providing:
 
 - 🔒 **Secure Environment**: Built on zero-trust principles
 - 🤖 **Automation Support**: Streamlined task execution
-- 🔑 **Secret Management**: Secure handling of sensitive data
+- 🔑 **Secret Management**: Automatic detection and resolution from multiple providers
 - 📦 **12-Factor Compliance**: Modern application practices
-- ♾️ **CI/CD Ready**: Seamless pipeline integration
+- ♾️ **CI/CD Ready**: Seamless pipeline integration with environment-based overrides
 
 ## 🏃 Quick Start
 

docs/config.md

@@ -110,24 +110,88 @@ config:
 
 The worker will automatically detect secret references (format: `provider/vault/secret`) in environment variables and resolve them at runtime.
 
+## Deployment Environment Variables
+
+Environment variables can also be set at the deployment level (e.g., Kubernetes, Docker Compose) and will **take priority** over `worker.yaml` configuration.
+
+### Priority Order
+
+1. **Deployment environment variables** (highest priority)
+2. `worker.yaml` config.secrets
+3. `worker.yaml` config.env
+
+### Example: Environment Override
+
+```yaml
+# worker.yaml (production defaults)
+config:
+  secrets:
+    ES_PASSWORD: "gcp/prod-project/es-password"
+```
+
+```yaml
+# Kubernetes deployment (staging override)
+spec:
+  containers:
+    - name: worker
+      env:
+        - name: ES_PASSWORD
+          value: "gcp/staging-project/es-password"
+```
+
+**Result**: The staging secret reference from Kubernetes will be used, not the production one from `worker.yaml`.
+
+### Use Cases
+
+- **Environment-specific overrides**: Different secrets per environment (dev/staging/prod)
+- **Sensitive values**: Keep secrets out of config files entirely
+- **Dynamic configuration**: Runtime values that change per deployment
+- **Testing**: Override config values without modifying files
+
+### Behavior
+
+- Deployment env vars with secret references are automatically detected and resolved
+- Deployment env vars with static values are used as-is
+- If a deployment env var exists, the corresponding `worker.yaml` entry is skipped
+
 ## Best Practices
 
 1. **Secret Management**
 
    - Never store sensitive values as plain text
-   - Use either `config.secrets` section OR secret references in `config.env`
-   - Both methods support the same provider format: `provider/vault/secret`
-   - Choose based on your preference:
+   - Use secret references in any of these locations:
      - `config.secrets`: Explicit separation of secrets
-     - `config.env` with references: Unified configuration
-
-2. **Environment Variables**
-
-   - Use `env` for non-sensitive configuration OR secret references
-   - Keep values consistent across environments
+     - `config.env`: Unified configuration with secret references
+     - Deployment env vars: Runtime overrides (highest priority)
+   - All methods support the same provider format: `provider/vault/secret`
+
+2. **Environment-Specific Configuration**
+
+   - Use `worker.yaml` for shared/default configuration
+   - Use deployment env vars for environment-specific overrides
+   - Example pattern:
+     ```yaml
+     # worker.yaml: production defaults
+     config:
+       secrets:
+         DB_PASSWORD: "gcp/prod/db-pass"
+
+     # K8s staging: override with deployment env
+     env:
+       - name: DB_PASSWORD
+         value: "gcp/staging/db-pass"
+     ```
+
+3. **Environment Variables**
+
+   - Use `config.env` for non-sensitive configuration OR secret references
+   - Use deployment env vars to override per environment
    - Document any required variables
+   - Remember: deployment env vars always win
+
+4. **File Handling**
 
-3. **File Handling**
-   - Keep configuration in version control (without sensitive data)
-   - Use different files for different environments
+   - Keep `worker.yaml` in version control (without sensitive data)
+   - Use secret references instead of plain text values
    - Validate configuration before deployment
+   - Use deployment env vars for truly sensitive overrides

lib/env_handler.sh

@@ -51,9 +51,13 @@ generate_env_file() {
     done < <(echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' -)
 }
 
-# Append resolved secrets to environment file
-append_resolved_secrets() {
+# Internal function to resolve and append secrets
+# Parameters:
+#   $1 - secrets_json: JSON object of secrets to resolve
+#   $2 - respect_deployment_env: if "true", skip secrets that exist in deployment environment
+_resolve_and_append_secrets() {
     local secrets_json="$1"
+    local respect_deployment_env="${2:-false}"
     local has_failures=false
 
     if [ -z "$secrets_json" ]; then
@@ -78,6 +82,18 @@ append_resolved_secrets() {
         local name value
         name=$(echo "$secret" | jq -r '.key')
 
+        # Check if variable exists in deployment environment (only if respect_deployment_env is true)
+        if [[ "$respect_deployment_env" == "true" ]] && printenv "$name" > /dev/null 2>&1; then
+            local deploy_value
+            deploy_value="$(printenv "$name")"
+            if [[ "$deploy_value" =~ ^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+ ]]; then
+                log_info "Environment" "Skipping [$name] from config.secrets - will be resolved from deployment environment secret reference"
+            else
+                log_info "Environment" "Skipping [$name] from config.secrets - using deployment environment static value"
+            fi
+            continue
+        fi
+
         # Create config JSON for resolve_secret_by_name
         local config_json
         config_json="{ \"config\": { \"secrets\": { \"$name\": $(echo "$secret" | jq '.value') } } }"
@@ -105,6 +121,18 @@ append_resolved_secrets() {
     rm -f "$temp_file"
 }
 
+# Resolve secrets from worker.yaml config.secrets section
+# Respects deployment environment - skips secrets that exist in deployment env
+append_resolved_secrets() {
+    _resolve_and_append_secrets "$1" "true"
+}
+
+# Resolve secrets detected in environment variables
+# Always resolves - deployment env vars take precedence by being processed here
+resolve_env_var_secrets() {
+    _resolve_and_append_secrets "$1" "false"
+}
+
 # Load environment variables and secrets
 load_environment() {
     if [ -f "$WORKER_ENV_FILE" ]; then

lib/secrets.sh

@@ -161,8 +161,7 @@ is_secret_reference() {
     local value="$1"
 
     # Check if value matches pattern: provider/vault/secret
-    # Supported providers: gcp, azure, aws, bitwarden
-    if [[ "$value" =~ ^(gcp|azure|aws|bitwarden)/.+/.+ ]]; then
+    if [[ "$value" =~ ^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+ ]]; then
         return 0
     fi
     return 1
@@ -198,7 +197,6 @@ should_skip_variable() {
 
 # Function to fetch secrets from environment variables
 fetch_secrets_from_env_vars() {
-    local processed_vars=()
     local -a secret_keys=()
     local -a secret_values=()
 
@@ -219,40 +217,9 @@ fetch_secrets_from_env_vars() {
         secret_values+=("$var_value")
     }
 
-    # 1. Process environment variables from worker.yaml (in WORKER_ENV_FILE)
-    if [[ -f "$WORKER_ENV_FILE" ]]; then
-        while IFS= read -r line; do
-            # Skip comments and empty lines
-            [[ "$line" =~ ^#.*$ || -z "$line" ]] && continue
-
-            # Extract variable name and value
-            if [[ "$line" =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then
-                local var_name="${BASH_REMATCH[1]}"
-                local var_value="${BASH_REMATCH[2]}"
-
-                # Track that we've processed this variable
-                processed_vars+=("$var_name")
-
-                # Collect if it's a secret reference
-                collect_secret "$var_name" "$var_value"
-            fi
-        done < "$WORKER_ENV_FILE"
-    fi
-
-    # 2. Process deployment environment variables (from container environment)
+    # Scan all environment variables for secret references
+    # This includes both worker.yaml config.env and deployment env vars
     while IFS='=' read -r key value; do
-        # Skip if already processed from worker config
-        local already_processed=false
-        for processed_var in "${processed_vars[@]}"; do
-            if [[ "$key" == "$processed_var" ]]; then
-                already_processed=true
-                break
-            fi
-        done
-        if [[ "$already_processed" == "true" ]]; then
-            continue
-        fi
-
         # Skip system and worker internal variables
         if should_skip_variable "$key"; then
             continue
@@ -264,7 +231,10 @@ fetch_secrets_from_env_vars() {
 
     # Build JSON from collected secrets in a single jq invocation
     local secrets_json="{}"
+    local secret_count=0
     if [[ ${#secret_keys[@]} -gt 0 ]]; then
+        secret_count=${#secret_keys[@]}
+
         # Build jq arguments for all key-value pairs
         local jq_args=()
         for i in "${!secret_keys[@]}"; do
@@ -287,14 +257,16 @@ fetch_secrets_from_env_vars() {
         return 0
     fi
 
+    log_info "Found $secret_count secret reference(s) in environment variables"
+
     # Validate JSON (should always be valid since jq built it)
     if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then
         log_error "Secrets" "Invalid JSON format for collected secrets"
         return 1
     fi
 
-    # Use existing append_resolved_secrets function to resolve and append
-    if ! append_resolved_secrets "$secrets_json"; then
+    # Resolve secrets from environment variables (always resolves, no skipping)
+    if ! resolve_env_var_secrets "$secrets_json"; then
         log_error "Secrets" "Failed to resolve secrets from environment variables"
         return 1
     fi

lib/utils.sh

@@ -1,5 +1,11 @@
 #!/bin/bash
 
+# Supported secret providers (used for secret reference detection)
+# Only declare if not already defined (prevents errors when sourced multiple times)
+if [[ -z "${SUPPORTED_SECRET_PROVIDERS+x}" ]]; then
+    readonly SUPPORTED_SECRET_PROVIDERS="gcp|azure|aws|bitwarden"
+fi
+
 # Function to resolve placeholders with environment variables
 resolve_env_vars() {
     local value="$1"