From 85d10c9ac3616074672aefa6e31179c1425d832a Mon Sep 17 00:00:00 2001 From: Anmol1696 Date: Wed, 31 Dec 2025 12:14:39 +0400 Subject: [PATCH 1/2] add secrets-meta package --- packages/secrets-meta/README.md | 11 +++ .../procedures/get_job_secrets_metadata.sql | 80 +++++++++++++++++ .../procedures/secret_metadata.sql | 89 +++++++++++++++++++ .../tables/secret_providers/table.sql | 33 +++++++ .../meta_public/tables/secrets/table.sql | 75 ++++++++++++++++ .../tables/secrets/triggers/invariants.sql | 46 ++++++++++ packages/secrets-meta/pgpm.plan | 15 ++++ .../procedures/get_job_secrets_metadata.sql | 8 ++ .../procedures/secret_metadata.sql | 18 ++++ .../tables/secret_providers/table.sql | 8 ++ .../meta_public/tables/secrets/table.sql | 8 ++ .../tables/secrets/triggers/invariants.sql | 9 ++ packages/secrets-meta/secrets-meta.control | 5 ++ 13 files changed, 405 insertions(+) create mode 100644 packages/secrets-meta/README.md create mode 100644 packages/secrets-meta/deploy/schemas/meta_private/procedures/get_job_secrets_metadata.sql create mode 100644 packages/secrets-meta/deploy/schemas/meta_public/procedures/secret_metadata.sql create mode 100644 packages/secrets-meta/deploy/schemas/meta_public/tables/secret_providers/table.sql create mode 100644 packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/table.sql create mode 100644 packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/triggers/invariants.sql create mode 100644 packages/secrets-meta/pgpm.plan create mode 100644 packages/secrets-meta/revert/schemas/meta_private/procedures/get_job_secrets_metadata.sql create mode 100644 packages/secrets-meta/revert/schemas/meta_public/procedures/secret_metadata.sql create mode 100644 packages/secrets-meta/revert/schemas/meta_public/tables/secret_providers/table.sql create mode 100644 packages/secrets-meta/revert/schemas/meta_public/tables/secrets/table.sql create mode 100644 packages/secrets-meta/revert/schemas/meta_public/tables/secrets/triggers/invariants.sql create mode 100644 packages/secrets-meta/secrets-meta.control diff --git a/packages/secrets-meta/README.md b/packages/secrets-meta/README.md new file mode 100644 index 00000000..9097417a --- /dev/null +++ b/packages/secrets-meta/README.md @@ -0,0 +1,11 @@ +# secrets-meta + +Meta-level secrets metadata module for PGPM: + +- `meta_public.secret_providers` — registry of secret backends (OpenBao, k8s, etc.) +- `meta_public.secrets` — per-owner/app secret metadata (no values) +- helper functions for metadata management and job→secret metadata lookup. + +Values are stored in an external provider (e.g. OpenBao KV v2); this +module stores only the routing and ownership information. + diff --git a/packages/secrets-meta/deploy/schemas/meta_private/procedures/get_job_secrets_metadata.sql b/packages/secrets-meta/deploy/schemas/meta_private/procedures/get_job_secrets_metadata.sql new file mode 100644 index 00000000..900c4df8 --- /dev/null +++ b/packages/secrets-meta/deploy/schemas/meta_private/procedures/get_job_secrets_metadata.sql @@ -0,0 +1,80 @@ +-- Deploy schemas/meta_private/procedures/get_job_secrets_metadata to pg +-- requires: schemas/meta_private/schema +-- requires: pgpm-database-jobs:schemas/app_jobs/tables/jobs/table +-- requires: schemas/meta_public/tables/secrets/table +-- requires: schemas/meta_public/tables/secret_providers/table +-- requires: db-meta-schema:schemas/meta_public/tables/apps/table + +BEGIN; + +CREATE OR REPLACE FUNCTION meta_private.get_job_secrets_metadata( + in_job_id bigint +) +RETURNS TABLE ( + secret_id uuid, + key text, + provider_type text, + provider_config jsonb, + provider_ref text, + app_id uuid, + database_id uuid, + task_identifier text +) +LANGUAGE sql +SECURITY DEFINER +AS $$ + WITH job_row AS ( + SELECT + j.id, + j.database_id, + j.task_identifier, + j.payload + FROM app_jobs.jobs j + WHERE j.id = in_job_id + ), + refs AS ( + SELECT + jr.database_id, + jr.task_identifier, + (each_ref).key AS ref_key, + (each_ref).value AS ref_value + FROM job_row jr, + LATERAL json_each(jr.payload -> 'secretRefs') AS each_ref(key, value) + ), + resolved AS ( + SELECT + s.id AS secret_id, + s.key, + sp.provider_type, + sp.config AS provider_config, + s.provider_ref, + s.app_id, + a.database_id, + r.task_identifier + FROM refs r + JOIN meta_public.secrets s + ON s.owner_type = (r.ref_value ->> 'ownerType') + AND s.owner_id = (r.ref_value ->> 'ownerId')::uuid + AND s.app_id = (r.ref_value ->> 'appId')::uuid + AND s.key_normalized = lower(r.ref_value ->> 'key') + JOIN meta_public.secret_providers sp + ON sp.id = s.provider_id + JOIN meta_public.apps a + ON a.id = s.app_id + AND a.database_id = r.database_id + WHERE sp.is_active + ) + SELECT + secret_id, + key, + provider_type, + provider_config, + provider_ref, + app_id, + database_id, + task_identifier + FROM resolved; +$$; + +COMMIT; + diff --git a/packages/secrets-meta/deploy/schemas/meta_public/procedures/secret_metadata.sql b/packages/secrets-meta/deploy/schemas/meta_public/procedures/secret_metadata.sql new file mode 100644 index 00000000..f574452e --- /dev/null +++ b/packages/secrets-meta/deploy/schemas/meta_public/procedures/secret_metadata.sql @@ -0,0 +1,89 @@ +-- Deploy schemas/meta_public/procedures/secret_metadata to pg +-- requires: schemas/meta_public/tables/secrets/table + +BEGIN; + +CREATE OR REPLACE FUNCTION meta_public.create_secret_metadata( + in_owner_type text, + in_owner_id uuid, + in_app_id uuid, + in_key text, + in_provider_id uuid, + in_provider_ref text, + in_description text DEFAULT NULL +) +RETURNS meta_public.secrets +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +DECLARE + v_secret meta_public.secrets; +BEGIN + IF in_owner_type NOT IN ('user', 'org', 'app', 'site') THEN + RAISE EXCEPTION 'invalid owner_type %', in_owner_type; + END IF; + + INSERT INTO meta_public.secrets ( + owner_type, + owner_id, + app_id, + key, + provider_id, + provider_ref, + description + ) VALUES ( + in_owner_type, + in_owner_id, + in_app_id, + in_key, + in_provider_id, + in_provider_ref, + in_description + ) + RETURNING * INTO v_secret; + + RETURN v_secret; +END; +$$; + + +CREATE OR REPLACE FUNCTION meta_public.rotate_secret_metadata( + in_secret_id uuid +) +RETURNS meta_public.secrets +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +DECLARE + v_secret meta_public.secrets; +BEGIN + UPDATE meta_public.secrets s + SET rotated_at = current_timestamp, + updated_at = current_timestamp + WHERE s.id = in_secret_id + RETURNING * INTO v_secret; + + IF NOT FOUND THEN + RAISE EXCEPTION 'secret % not found', in_secret_id; + END IF; + + RETURN v_secret; +END; +$$; + + +CREATE OR REPLACE FUNCTION meta_public.delete_secret_metadata( + in_secret_id uuid +) +RETURNS void +LANGUAGE plpgsql +SECURITY DEFINER +AS $$ +BEGIN + DELETE FROM meta_public.secrets s + WHERE s.id = in_secret_id; +END; +$$; + +COMMIT; + diff --git a/packages/secrets-meta/deploy/schemas/meta_public/tables/secret_providers/table.sql b/packages/secrets-meta/deploy/schemas/meta_public/tables/secret_providers/table.sql new file mode 100644 index 00000000..50cf4119 --- /dev/null +++ b/packages/secrets-meta/deploy/schemas/meta_public/tables/secret_providers/table.sql @@ -0,0 +1,33 @@ +-- Deploy schemas/meta_public/tables/secret_providers/table to pg +-- requires: schemas/meta_public/schema + +BEGIN; + +CREATE TABLE IF NOT EXISTS meta_public.secret_providers ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + name text NOT NULL, + provider_type text NOT NULL, + config jsonb NOT NULL DEFAULT '{}'::jsonb, + description text, + is_active boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp +); + +COMMENT ON TABLE meta_public.secret_providers IS + 'Registry of secret provider backends (OpenBao, k8s, etc).'; + +COMMENT ON COLUMN meta_public.secret_providers.name IS + 'Human-readable name for this secret provider.'; + +COMMENT ON COLUMN meta_public.secret_providers.provider_type IS + 'Provider type identifier (e.g. openbao, k8s, aws_secrets_manager).'; + +COMMENT ON COLUMN meta_public.secret_providers.config IS + 'Provider-specific configuration (JSON), such as endpoints, mounts, roles.'; + +COMMENT ON COLUMN meta_public.secret_providers.is_active IS + 'Whether this provider is currently active/usable.'; + +COMMIT; + diff --git a/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/table.sql b/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/table.sql new file mode 100644 index 00000000..58d60493 --- /dev/null +++ b/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/table.sql @@ -0,0 +1,75 @@ +-- Deploy schemas/meta_public/tables/secrets/table to pg +-- requires: schemas/meta_public/schema +-- requires: schemas/meta_public/tables/apps/table +-- requires: schemas/meta_public/tables/secret_providers/table + +BEGIN; + +CREATE TABLE IF NOT EXISTS meta_public.secrets ( + id uuid PRIMARY KEY DEFAULT uuid_generate_v4(), + + -- Ownership / scope + owner_type text NOT NULL, -- user | org | app | site + owner_id uuid NOT NULL, + app_id uuid NOT NULL, + + -- Logical key + key text NOT NULL, + + -- Normalized key for uniqueness (lowercased or citext) + key_normalized text NOT NULL, + + -- Provider linkage + provider_id uuid NOT NULL, + provider_ref text NOT NULL, + + description text, + + is_active boolean NOT NULL DEFAULT true, + + created_at timestamptz NOT NULL DEFAULT current_timestamp, + updated_at timestamptz NOT NULL DEFAULT current_timestamp, + rotated_at timestamptz +); + +COMMENT ON TABLE meta_public.secrets IS + 'Metadata for user/org/app secrets; values live in external providers.'; + +COMMENT ON COLUMN meta_public.secrets.owner_type IS + 'Owner type for the secret: user, org, app, or site.'; + +COMMENT ON COLUMN meta_public.secrets.owner_id IS + 'ID of the owning user/org/app/site.'; + +COMMENT ON COLUMN meta_public.secrets.app_id IS + 'Logical app/database this secret is associated with.'; + +COMMENT ON COLUMN meta_public.secrets.key IS + 'Logical secret key name (e.g. MAILGUN_API_KEY).'; + +COMMENT ON COLUMN meta_public.secrets.key_normalized IS + 'Normalized form of key used for uniqueness (e.g. lower(key)).'; + +COMMENT ON COLUMN meta_public.secrets.provider_id IS + 'Foreign key to meta_public.secret_providers.'; + +COMMENT ON COLUMN meta_public.secrets.provider_ref IS + 'Opaque provider-specific reference/path (e.g. OpenBao KV path).'; + +ALTER TABLE meta_public.secrets + ADD CONSTRAINT secrets_app_fkey + FOREIGN KEY (app_id) + REFERENCES meta_public.apps (id) + ON DELETE CASCADE; + +ALTER TABLE meta_public.secrets + ADD CONSTRAINT secrets_provider_fkey + FOREIGN KEY (provider_id) + REFERENCES meta_public.secret_providers (id) + ON DELETE RESTRICT; + +CREATE UNIQUE INDEX IF NOT EXISTS secrets_owner_app_key_norm_uniq + ON meta_public.secrets (owner_type, owner_id, app_id, key_normalized); + +COMMIT; + diff --git a/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/triggers/invariants.sql b/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/triggers/invariants.sql new file mode 100644 index 00000000..570e93f7 --- /dev/null +++ b/packages/secrets-meta/deploy/schemas/meta_public/tables/secrets/triggers/invariants.sql @@ -0,0 +1,46 @@ +-- Deploy schemas/meta_public/tables/secrets/triggers/invariants to pg +-- requires: schemas/meta_public/tables/secrets/table + +BEGIN; + +CREATE OR REPLACE FUNCTION meta_public.secrets_invariants_tg() +RETURNS trigger AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + -- Normalize key + NEW.key_normalized := lower(NEW.key); + + -- Timestamps + IF NEW.created_at IS NULL THEN + NEW.created_at := current_timestamp; + END IF; + IF NEW.updated_at IS NULL THEN + NEW.updated_at := current_timestamp; + END IF; + + ELSIF TG_OP = 'UPDATE' THEN + -- Normalize key + NEW.key_normalized := lower(NEW.key); + + -- Prevent provider_ref changes + IF NEW.provider_ref IS DISTINCT FROM OLD.provider_ref THEN + RAISE EXCEPTION 'provider_ref is immutable for secret %', OLD.id; + END IF; + + -- Maintain updated_at + NEW.updated_at := current_timestamp; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql VOLATILE; + +DROP TRIGGER IF EXISTS secrets_invariants_before_tg ON meta_public.secrets; + +CREATE TRIGGER secrets_invariants_before_tg + BEFORE INSERT OR UPDATE ON meta_public.secrets + FOR EACH ROW + EXECUTE PROCEDURE meta_public.secrets_invariants_tg(); + +COMMIT; + diff --git a/packages/secrets-meta/pgpm.plan b/packages/secrets-meta/pgpm.plan new file mode 100644 index 00000000..6260e0ff --- /dev/null +++ b/packages/secrets-meta/pgpm.plan @@ -0,0 +1,15 @@ +%syntax-version=1.0.0 +%project=secrets-meta +%uri=secrets-meta + +# Meta-level secrets metadata and helpers + +schemas/meta_public/tables/secret_providers/table [db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # add secret_providers registry +schemas/meta_public/tables/secrets/table [schemas/meta_public/tables/secret_providers/table db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # add secrets metadata table + +schemas/meta_public/tables/secrets/triggers/invariants [schemas/meta_public/tables/secrets/table] 2025-01-01T00:00:00Z secrets-meta # enforce invariants on secrets + +schemas/meta_public/procedures/secret_metadata [schemas/meta_public/tables/secrets/table] 2025-01-01T00:00:00Z secrets-meta # helpers for create/rotate/delete metadata + +schemas/meta_private/procedures/get_job_secrets_metadata [pgpm-database-jobs:schemas/app_jobs/tables/jobs/table schemas/meta_public/tables/secrets/table schemas/meta_public/tables/secret_providers/table db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # helper for jobId→secretRefs→metadata + diff --git a/packages/secrets-meta/revert/schemas/meta_private/procedures/get_job_secrets_metadata.sql b/packages/secrets-meta/revert/schemas/meta_private/procedures/get_job_secrets_metadata.sql new file mode 100644 index 00000000..0bc059b2 --- /dev/null +++ b/packages/secrets-meta/revert/schemas/meta_private/procedures/get_job_secrets_metadata.sql @@ -0,0 +1,8 @@ +-- Revert schemas/meta_private/procedures/get_job_secrets_metadata from pg + +BEGIN; + +DROP FUNCTION IF EXISTS meta_private.get_job_secrets_metadata(bigint); + +COMMIT; + diff --git a/packages/secrets-meta/revert/schemas/meta_public/procedures/secret_metadata.sql b/packages/secrets-meta/revert/schemas/meta_public/procedures/secret_metadata.sql new file mode 100644 index 00000000..ac5a5728 --- /dev/null +++ b/packages/secrets-meta/revert/schemas/meta_public/procedures/secret_metadata.sql @@ -0,0 +1,18 @@ +-- Revert schemas/meta_public/procedures/secret_metadata from pg + +BEGIN; + +DROP FUNCTION IF EXISTS meta_public.delete_secret_metadata(uuid); +DROP FUNCTION IF EXISTS meta_public.rotate_secret_metadata(uuid); +DROP FUNCTION IF EXISTS meta_public.create_secret_metadata( + text, + uuid, + uuid, + text, + uuid, + text, + text +); + +COMMIT; + diff --git a/packages/secrets-meta/revert/schemas/meta_public/tables/secret_providers/table.sql b/packages/secrets-meta/revert/schemas/meta_public/tables/secret_providers/table.sql new file mode 100644 index 00000000..abd6d2ae --- /dev/null +++ b/packages/secrets-meta/revert/schemas/meta_public/tables/secret_providers/table.sql @@ -0,0 +1,8 @@ +-- Revert schemas/meta_public/tables/secret_providers/table from pg + +BEGIN; + +DROP TABLE IF EXISTS meta_public.secret_providers CASCADE; + +COMMIT; + diff --git a/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/table.sql b/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/table.sql new file mode 100644 index 00000000..8035117f --- /dev/null +++ b/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/table.sql @@ -0,0 +1,8 @@ +-- Revert schemas/meta_public/tables/secrets/table from pg + +BEGIN; + +DROP TABLE IF EXISTS meta_public.secrets CASCADE; + +COMMIT; + diff --git a/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/triggers/invariants.sql b/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/triggers/invariants.sql new file mode 100644 index 00000000..c2695595 --- /dev/null +++ b/packages/secrets-meta/revert/schemas/meta_public/tables/secrets/triggers/invariants.sql @@ -0,0 +1,9 @@ +-- Revert schemas/meta_public/tables/secrets/triggers/invariants from pg + +BEGIN; + +DROP TRIGGER IF EXISTS secrets_invariants_before_tg ON meta_public.secrets; +DROP FUNCTION IF EXISTS meta_public.secrets_invariants_tg(); + +COMMIT; + diff --git a/packages/secrets-meta/secrets-meta.control b/packages/secrets-meta/secrets-meta.control new file mode 100644 index 00000000..047be06c --- /dev/null +++ b/packages/secrets-meta/secrets-meta.control @@ -0,0 +1,5 @@ +comment = 'Meta-level secrets metadata (providers + per-app/user keys)' +default_version = '0.0.1' +module_pathname = '$libdir/secrets-meta' +relocatable = true + From 6ea64f66935357bd5783caf72e2b5a3959dbeb6f Mon Sep 17 00:00:00 2001 From: Anmol1696 Date: Wed, 31 Dec 2025 15:55:55 +0400 Subject: [PATCH 2/2] update pgpm pland and package.json file --- packages/secrets-meta/package.json | 37 ++++++++++++++++++++++++++++++ packages/secrets-meta/pgpm.plan | 3 ++- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 packages/secrets-meta/package.json diff --git a/packages/secrets-meta/package.json b/packages/secrets-meta/package.json new file mode 100644 index 00000000..486a7b8a --- /dev/null +++ b/packages/secrets-meta/package.json @@ -0,0 +1,37 @@ +{ + "name": "@pgpm/secrets-meta", + "version": "0.0.1", + "description": "Meta-level secrets registry (providers + secret metadata, no values)", + "author": "Constructive ", + "keywords": [ + "postgresql", + "pgpm", + "secrets", + "metadata" + ], + "publishConfig": { + "access": "public" + }, + "scripts": { + "bundle": "pgpm package", + "test": "jest", + "test:watch": "jest --watch" + }, + "dependencies": { + "@pgpm/database-jobs": "workspace:*", + "@pgpm/db-meta-schema": "workspace:*", + "@pgpm/verify": "workspace:*" + }, + "devDependencies": { + "pgpm": "^1.3.0" + }, + "repository": { + "type": "git", + "url": "https://github.com/constructive-io/pgpm-modules" + }, + "homepage": "https://github.com/constructive-io/pgpm-modules", + "bugs": { + "url": "https://github.com/constructive-io/pgpm-modules/issues" + } +} + diff --git a/packages/secrets-meta/pgpm.plan b/packages/secrets-meta/pgpm.plan index 6260e0ff..6c2db297 100644 --- a/packages/secrets-meta/pgpm.plan +++ b/packages/secrets-meta/pgpm.plan @@ -4,6 +4,8 @@ # Meta-level secrets metadata and helpers +schemas/meta_public/schema [db-meta-schema:schemas/meta_public/schema] 2025-01-01T00:00:00Z secrets-meta # meta_public schema alias + schemas/meta_public/tables/secret_providers/table [db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # add secret_providers registry schemas/meta_public/tables/secrets/table [schemas/meta_public/tables/secret_providers/table db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # add secrets metadata table @@ -12,4 +14,3 @@ schemas/meta_public/tables/secrets/triggers/invariants [schemas/meta_public/tabl schemas/meta_public/procedures/secret_metadata [schemas/meta_public/tables/secrets/table] 2025-01-01T00:00:00Z secrets-meta # helpers for create/rotate/delete metadata schemas/meta_private/procedures/get_job_secrets_metadata [pgpm-database-jobs:schemas/app_jobs/tables/jobs/table schemas/meta_public/tables/secrets/table schemas/meta_public/tables/secret_providers/table db-meta-schema:schemas/meta_public/tables/apps/table] 2025-01-01T00:00:00Z secrets-meta # helper for jobId→secretRefs→metadata -