diff --git a/README.md b/README.md index 4c99d92..0427be2 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ UDX Worker is a containerized solution that simplifies DevSecOps by providing: - πŸ”’ **Secure Environment**: Built on zero-trust principles - πŸ€– **Automation Support**: Streamlined task execution -- πŸ”‘ **Secret Management**: Secure handling of sensitive data +- πŸ”‘ **Secret Management**: Automatic detection and resolution from multiple providers - πŸ“¦ **12-Factor Compliance**: Modern application practices -- ♾️ **CI/CD Ready**: Seamless pipeline integration +- ♾️ **CI/CD Ready**: Seamless pipeline integration with environment-based overrides ## πŸƒ Quick Start diff --git a/docs/config.md b/docs/config.md index 6f043fa..48bfbe3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -110,24 +110,88 @@ config: The worker will automatically detect secret references (format: `provider/vault/secret`) in environment variables and resolve them at runtime. +## Deployment Environment Variables + +Environment variables can also be set at the deployment level (e.g., Kubernetes, Docker Compose) and will **take priority** over `worker.yaml` configuration. + +### Priority Order + +1. **Deployment environment variables** (highest priority) +2. `worker.yaml` config.secrets +3. `worker.yaml` config.env + +### Example: Environment Override + +```yaml +# worker.yaml (production defaults) +config: + secrets: + ES_PASSWORD: "gcp/prod-project/es-password" +``` + +```yaml +# Kubernetes deployment (staging override) +spec: + containers: + - name: worker + env: + - name: ES_PASSWORD + value: "gcp/staging-project/es-password" +``` + +**Result**: The staging secret reference from Kubernetes will be used, not the production one from `worker.yaml`. + +### Use Cases + +- **Environment-specific overrides**: Different secrets per environment (dev/staging/prod) +- **Sensitive values**: Keep secrets out of config files entirely +- **Dynamic configuration**: Runtime values that change per deployment +- **Testing**: Override config values without modifying files + +### Behavior + +- Deployment env vars with secret references are automatically detected and resolved +- Deployment env vars with static values are used as-is +- If a deployment env var exists, the corresponding `worker.yaml` entry is skipped + ## Best Practices 1. **Secret Management** - Never store sensitive values as plain text - - Use either `config.secrets` section OR secret references in `config.env` - - Both methods support the same provider format: `provider/vault/secret` - - Choose based on your preference: + - Use secret references in any of these locations: - `config.secrets`: Explicit separation of secrets - - `config.env` with references: Unified configuration - -2. **Environment Variables** - - - Use `env` for non-sensitive configuration OR secret references - - Keep values consistent across environments + - `config.env`: Unified configuration with secret references + - Deployment env vars: Runtime overrides (highest priority) + - All methods support the same provider format: `provider/vault/secret` + +2. **Environment-Specific Configuration** + + - Use `worker.yaml` for shared/default configuration + - Use deployment env vars for environment-specific overrides + - Example pattern: + ```yaml + # worker.yaml: production defaults + config: + secrets: + DB_PASSWORD: "gcp/prod/db-pass" + + # K8s staging: override with deployment env + env: + - name: DB_PASSWORD + value: "gcp/staging/db-pass" + ``` + +3. **Environment Variables** + + - Use `config.env` for non-sensitive configuration OR secret references + - Use deployment env vars to override per environment - Document any required variables + - Remember: deployment env vars always win + +4. **File Handling** -3. **File Handling** - - Keep configuration in version control (without sensitive data) - - Use different files for different environments + - Keep `worker.yaml` in version control (without sensitive data) + - Use secret references instead of plain text values - Validate configuration before deployment + - Use deployment env vars for truly sensitive overrides diff --git a/lib/env_handler.sh b/lib/env_handler.sh index efeec78..d439c35 100644 --- a/lib/env_handler.sh +++ b/lib/env_handler.sh @@ -51,9 +51,13 @@ generate_env_file() { done < <(echo "$config" | yq eval '.config.env | to_entries | .[] | "export " + .key + "=\"" + .value + "\""' -) } -# Append resolved secrets to environment file -append_resolved_secrets() { +# Internal function to resolve and append secrets +# Parameters: +# $1 - secrets_json: JSON object of secrets to resolve +# $2 - respect_deployment_env: if "true", skip secrets that exist in deployment environment +_resolve_and_append_secrets() { local secrets_json="$1" + local respect_deployment_env="${2:-false}" local has_failures=false if [ -z "$secrets_json" ]; then @@ -78,6 +82,18 @@ append_resolved_secrets() { local name value name=$(echo "$secret" | jq -r '.key') + # Check if variable exists in deployment environment (only if respect_deployment_env is true) + if [[ "$respect_deployment_env" == "true" ]] && printenv "$name" > /dev/null 2>&1; then + local deploy_value + deploy_value="$(printenv "$name")" + if [[ "$deploy_value" =~ ^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+ ]]; then + log_info "Environment" "Skipping [$name] from config.secrets - will be resolved from deployment environment secret reference" + else + log_info "Environment" "Skipping [$name] from config.secrets - using deployment environment static value" + fi + continue + fi + # Create config JSON for resolve_secret_by_name local config_json config_json="{ \"config\": { \"secrets\": { \"$name\": $(echo "$secret" | jq '.value') } } }" @@ -105,6 +121,18 @@ append_resolved_secrets() { rm -f "$temp_file" } +# Resolve secrets from worker.yaml config.secrets section +# Respects deployment environment - skips secrets that exist in deployment env +append_resolved_secrets() { + _resolve_and_append_secrets "$1" "true" +} + +# Resolve secrets detected in environment variables +# Always resolves - deployment env vars take precedence by being processed here +resolve_env_var_secrets() { + _resolve_and_append_secrets "$1" "false" +} + # Load environment variables and secrets load_environment() { if [ -f "$WORKER_ENV_FILE" ]; then diff --git a/lib/secrets.sh b/lib/secrets.sh index 39b8d62..6968fed 100644 --- a/lib/secrets.sh +++ b/lib/secrets.sh @@ -161,8 +161,7 @@ is_secret_reference() { local value="$1" # Check if value matches pattern: provider/vault/secret - # Supported providers: gcp, azure, aws, bitwarden - if [[ "$value" =~ ^(gcp|azure|aws|bitwarden)/.+/.+ ]]; then + if [[ "$value" =~ ^(${SUPPORTED_SECRET_PROVIDERS})/.+/.+ ]]; then return 0 fi return 1 @@ -198,7 +197,6 @@ should_skip_variable() { # Function to fetch secrets from environment variables fetch_secrets_from_env_vars() { - local processed_vars=() local -a secret_keys=() local -a secret_values=() @@ -219,40 +217,9 @@ fetch_secrets_from_env_vars() { secret_values+=("$var_value") } - # 1. Process environment variables from worker.yaml (in WORKER_ENV_FILE) - if [[ -f "$WORKER_ENV_FILE" ]]; then - while IFS= read -r line; do - # Skip comments and empty lines - [[ "$line" =~ ^#.*$ || -z "$line" ]] && continue - - # Extract variable name and value - if [[ "$line" =~ ^export[[:space:]]+([^=]+)=\"([^\"]*)\" ]]; then - local var_name="${BASH_REMATCH[1]}" - local var_value="${BASH_REMATCH[2]}" - - # Track that we've processed this variable - processed_vars+=("$var_name") - - # Collect if it's a secret reference - collect_secret "$var_name" "$var_value" - fi - done < "$WORKER_ENV_FILE" - fi - - # 2. Process deployment environment variables (from container environment) + # Scan all environment variables for secret references + # This includes both worker.yaml config.env and deployment env vars while IFS='=' read -r key value; do - # Skip if already processed from worker config - local already_processed=false - for processed_var in "${processed_vars[@]}"; do - if [[ "$key" == "$processed_var" ]]; then - already_processed=true - break - fi - done - if [[ "$already_processed" == "true" ]]; then - continue - fi - # Skip system and worker internal variables if should_skip_variable "$key"; then continue @@ -264,7 +231,10 @@ fetch_secrets_from_env_vars() { # Build JSON from collected secrets in a single jq invocation local secrets_json="{}" + local secret_count=0 if [[ ${#secret_keys[@]} -gt 0 ]]; then + secret_count=${#secret_keys[@]} + # Build jq arguments for all key-value pairs local jq_args=() for i in "${!secret_keys[@]}"; do @@ -287,14 +257,16 @@ fetch_secrets_from_env_vars() { return 0 fi + log_info "Found $secret_count secret reference(s) in environment variables" + # Validate JSON (should always be valid since jq built it) if ! echo "$secrets_json" | jq empty > /dev/null 2>&1; then log_error "Secrets" "Invalid JSON format for collected secrets" return 1 fi - # Use existing append_resolved_secrets function to resolve and append - if ! append_resolved_secrets "$secrets_json"; then + # Resolve secrets from environment variables (always resolves, no skipping) + if ! resolve_env_var_secrets "$secrets_json"; then log_error "Secrets" "Failed to resolve secrets from environment variables" return 1 fi diff --git a/lib/utils.sh b/lib/utils.sh index 06c84cf..ceeba1b 100644 --- a/lib/utils.sh +++ b/lib/utils.sh @@ -1,5 +1,11 @@ #!/bin/bash +# Supported secret providers (used for secret reference detection) +# Only declare if not already defined (prevents errors when sourced multiple times) +if [[ -z "${SUPPORTED_SECRET_PROVIDERS+x}" ]]; then + readonly SUPPORTED_SECRET_PROVIDERS="gcp|azure|aws|bitwarden" +fi + # Function to resolve placeholders with environment variables resolve_env_vars() { local value="$1"