feat: template provider

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ permissions:
 
 jobs:
   test:
-    name: Run End-to-End Tests
+    name: Run short test
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -24,150 +24,22 @@ jobs:
         with:
           go-version-file: go.mod
 
-      - name: Setup env via sstart
-        uses: dirathea/setup-sstart-env@main
-        env:
-          INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET }}
-          INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID }}
-          INFISICAL_SITE_URL: ${{ secrets.INFISICAL_SITE_URL }}
-        with:
-          config: |
-            providers:
-              - kind: infisical
-                project_id: 8aded323-e110-4f48-9c7f-24c275358609
-                environment: prod
-                path: /github
-
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
-
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v2
-
-      - name: Install Bitwarden CLI
-        run: |
-          BW_DOWNLOAD_URL=$(curl -s https://api.github.com/repos/bitwarden/cli/releases/latest | grep '"browser_download_url".*linux.*zip' | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
-          curl -L "${BW_DOWNLOAD_URL}" -o bw.zip
-          unzip -q bw.zip
-          chmod +x bw
-          sudo mv bw /usr/local/bin/
-          rm bw.zip
-          bw --version
-
-      - name: Determine which tests to run
-        id: test_filter
-        uses: actions/github-script@v7
-        with:
-          script: |
-            try {
-              const { data: files } = await github.rest.pulls.listFiles({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                pull_number: context.issue.number,
-              });
-
-              const changedFiles = files.map(f => f.filename);
-              console.log('Changed files:', changedFiles);
-
-            // Map provider directories to test name prefixes
-            const providerTestMap = {
-              'internal/provider/aws/': ['TestE2E_AWSSecretsManager'],
-              'internal/provider/azurekeyvault/': ['TestE2E_AzureKeyVault'],
-              'internal/provider/bitwarden/': ['TestE2E_Bitwarden', 'TestE2E_BitwardenSM'],
-              'internal/provider/doppler/': ['TestE2E_Doppler'],
-              'internal/provider/gcsm/': ['TestE2E_GCSM'],
-              'internal/provider/infisical/': ['TestE2E_Infisical'],
-              'internal/provider/onepassword/': ['TestE2E_OnePassword'],
-              'internal/provider/vault/': ['TestE2E_Vault', 'TestE2E_OpenBao'],
-              'internal/oidc/': ['TestE2E_SSO'],
-            };
-
-              // Core files that require all tests
-              const corePaths = [
-                'internal/secrets/',
-                'internal/config/',
-                'internal/app/',
-                'internal/cli/',
-                'tests/end2end/',
-                'cmd/',
-              ];
-
-              // Check if any core files changed
-              const hasCoreChanges = changedFiles.some(file => 
-                corePaths.some(path => file.startsWith(path))
-              );
-
-              if (hasCoreChanges) {
-                console.log('Core files changed, running all tests');
-                core.setOutput('test_filter', '');
-                core.setOutput('run_all_tests', 'true');
-                core.setOutput('skip_tests', 'false');
-                return;
-              }
-
-              // Find which providers changed
-              const affectedTests = new Set();
-
-              for (const file of changedFiles) {
-                for (const [providerPath, testNames] of Object.entries(providerTestMap)) {
-                  if (file.startsWith(providerPath)) {
-                    testNames.forEach(test => affectedTests.add(test));
-                  }
-                }
-              }
-
-              if (affectedTests.size === 0) {
-                console.log('No provider changes detected, skipping end2end tests');
-                core.setOutput('test_filter', '');
-                core.setOutput('run_all_tests', 'false');
-                core.setOutput('skip_tests', 'true');
-                return;
-              }
-
-              // Construct test filter regex (matches any of the affected tests)
-              // Format: TestE2E_(Provider1|Provider2) for go test -run flag
-              const testFilter = Array.from(affectedTests).join('|');
-              console.log('Running selective tests:', testFilter);
-              core.setOutput('test_filter', testFilter);
-              core.setOutput('run_all_tests', 'false');
-              core.setOutput('skip_tests', 'false');
-            } catch (error) {
-              console.log('Error determining test filter, running all tests:', error.message);
-              core.setOutput('test_filter', '');
-              core.setOutput('run_all_tests', 'true');
-              core.setOutput('skip_tests', 'false');
-            }
-
-      - name: Run end-to-end tests
-        if: steps.test_filter.outputs.skip_tests != 'true'
+      - name: Run tests in short mode
         env:
           CGO_LDFLAGS: -lm
-          # the rest of env supplied by setup-sstart-env
         run: |
           go install gotest.tools/gotestsum
-          if [ "${{ steps.test_filter.outputs.run_all_tests }}" = "true" ]; then
-            echo "Running all end-to-end tests"
-            gotestsum --junitfile test-results.xml --format testname -- ./tests/end2end/...
-          else
-            echo "Running selective tests: ${{ steps.test_filter.outputs.test_filter }}"
-            gotestsum --junitfile test-results.xml --format testname -- -run "${{ steps.test_filter.outputs.test_filter }}" ./tests/end2end/...
-          fi
+          echo "Running tests in short mode"
+          gotestsum --junitfile test-results.xml --format testname -- -short ./tests/end2end/...
 
       - name: Publish test results
-        if: always() && steps.test_filter.outputs.skip_tests != 'true'
+        if: always()
         uses: EnricoMi/publish-unit-test-result-action@v2
         with:
           files: test-results.xml
-          check_name: End-to-End Test Results
+          check_name: CI Test Results
           fail_on: 'nothing'
           comment_mode: off
-
-      - name: Skip end-to-end tests
-        if: steps.test_filter.outputs.skip_tests == 'true'
-        run: |
-          echo "No provider changes detected. Skipping end-to-end tests."
 
   build:
     name: Build

.github/workflows/end2end.yml

@@ -0,0 +1,77 @@
+name: End-to-End Tests
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+    branches:
+      - main
+
+permissions:
+  contents: read
+  checks: write
+  pull-requests: write
+
+jobs:
+  test:
+    name: Run End-to-End Tests
+    runs-on: ubuntu-latest
+    environment: end2end
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Setup env via sstart
+        uses: dirathea/setup-sstart-env@main
+        env:
+          INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_SECRET }}
+          INFISICAL_UNIVERSAL_AUTH_CLIENT_ID: ${{ secrets.INFISICAL_UNIVERSAL_AUTH_CLIENT_ID }}
+          INFISICAL_SITE_URL: https://eu.infisical.com
+        with:
+          config: |
+            providers:
+              - kind: infisical
+                project_id: 8aded323-e110-4f48-9c7f-24c275358609
+                environment: prod
+                path: /github
+
+      - name: Authenticate to Google Cloud
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: ${{ env.GOOGLE_APPLICATION_CREDENTIALS }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Install Bitwarden CLI
+        run: |
+          BW_DOWNLOAD_URL=$(curl -s https://api.github.com/repos/bitwarden/cli/releases/latest | grep '"browser_download_url".*linux.*zip' | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
+          curl -L "${BW_DOWNLOAD_URL}" -o bw.zip
+          unzip -q bw.zip
+          chmod +x bw
+          sudo mv bw /usr/local/bin/
+          rm bw.zip
+          bw --version
+
+      - name: Run all end-to-end tests
+        env:
+          CGO_LDFLAGS: -lm
+          # the rest of env supplied by setup-sstart-env
+        run: |
+          go install gotest.tools/gotestsum
+          echo "Running all end-to-end tests (including tests that require real services)"
+          gotestsum --junitfile test-results.xml --format testname -- ./tests/end2end/...
+
+      - name: Publish test results
+        if: always()
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          files: test-results.xml
+          check_name: End-to-End Test Results
+          fail_on: 'nothing'
+          comment_mode: off
+

CONFIGURATION.md

@@ -34,6 +34,7 @@ providers:
 | `dotenv` | Stable |
 | `gcloud_secretmanager` | Stable |
 | `infisical` | Stable |
+| `template` | Stable |
 | `vault` | Stable |
 
 ## Provider Configuration
@@ -590,6 +591,100 @@ You can also use simple environment variable expansion with `${VAR}` or `$VAR` s
     path: ${HOME}/.config/myapp/.env
 ```
 
+## Template Providers
+
+The template provider allows you to construct new secrets by combining values from other providers using Go template syntax. This is useful when your application needs secrets in a different format than how they're stored (e.g., building connection URIs from separate credentials).
+
+**Configuration:**
+- `uses` (required): List of provider IDs that this template provider depends on. The template provider can only access secrets from providers explicitly listed here (principle of least privilege).
+- `templates` (required): Map of output secret keys to template expressions. Each template expression is evaluated using Go's `text/template` package.
+
+**Template Syntax:**
+- Use `{{.<provider_id>.<secret_key>}}` to reference secrets from other providers
+- The syntax is similar to Helm templates and uses Go's text/template package
+- You can use all Go template functions (e.g., `{{if}}`, `{{range}}`, `{{index}}`, etc.)
+- Provider IDs and secret keys are case-sensitive
+
+**Security Model:**
+The template provider follows the principle of least privilege:
+- Only providers listed in the `uses` field are accessible
+- If a provider is not in `uses`, references to it will resolve to empty values
+- This ensures templates can only access secrets they explicitly declare as dependencies
+
+**Provider Order:**
+Template providers must be defined after the providers they depend on. Providers are processed in the order they appear in the configuration file, so ensure all source providers are listed before the template provider.
+
+**Example - Building a Database URI:**
+```yaml
+providers:
+  # Fetch database host configuration
+  - kind: aws_secretsmanager
+    id: db_config
+    secret_id: rds/credentials
+    # Returns: DB_HOST, DB_PORT, DB_NAME
+
+  # Fetch database credentials
+  - kind: aws_secretsmanager
+    id: db_creds
+    secret_id: rds/prod/credentials
+    # Returns: DB_USER, DB_PASSWORD
+
+  # Build database URI using template provider
+  - kind: template
+    uses:
+      - db_config
+      - db_creds
+    templates:
+      DATABASE_URI: postgresql://{{.db_creds.DB_USER}}:{{.db_creds.DB_PASSWORD}}@{{.db_config.DB_HOST}}:{{.db_config.DB_PORT}}/{{.db_config.DB_NAME}}
+```
+
+**Example - Multiple Templates:**
+```yaml
+providers:
+  - kind: aws_secretsmanager
+    id: api_config
+    secret_id: api/config
+    # Returns: API_HOST, API_PORT
+
+  - kind: aws_secretsmanager
+    id: api_creds
+    secret_id: api/credentials
+    # Returns: API_KEY, API_SECRET
+
+  - kind: template
+    uses:
+      - api_config
+      - api_creds
+    templates:
+      API_BASE_URL: https://{{.api_config.API_HOST}}:{{.api_config.API_PORT}}
+      API_AUTH_HEADER: Bearer {{.api_creds.API_KEY}}
+      API_FULL_URL: https://{{.api_config.API_HOST}}:{{.api_config.API_PORT}}/v1?key={{.api_creds.API_KEY}}
+```
+
+**Example - Using Template Functions:**
+```yaml
+providers:
+  - kind: aws_secretsmanager
+    id: config
+    secret_id: app/config
+    # Returns: ENV (e.g., "production")
+
+  - kind: template
+    uses:
+      - config
+    templates:
+      # Use conditional logic based on secret values
+      LOG_LEVEL: {{if eq .config.ENV "production"}}error{{else}}debug{{end}}
+      # Combine multiple template expressions
+      APP_ENV: {{.config.ENV}}
+```
+
+**Error Handling:**
+- If a referenced provider ID doesn't exist, the template will fail with an error
+- If a referenced secret key doesn't exist in a provider, it will resolve to an empty value
+- If `uses` is not specified or empty, all provider references will resolve to empty values
+- Template parsing errors will be reported with the specific template expression that failed
+
 ## Multiple Providers
 
 Each provider loads from a single source. To load multiple secrets from the same provider type, create multiple provider instances: