From 7380f18f155245ee0be7a67768613615d773399a Mon Sep 17 00:00:00 2001 From: Denozordec Date: Sun, 15 Nov 2026 20:36:08 +0700 Subject: [PATCH] =?UTF-8?q?feat(mcp):=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80=D0=B0=D0=BA?= =?UTF-8?q?=D1=82=D1=8B,=20=D1=81=D1=85=D0=B5=D0=BC=D1=83=20=D0=91=D0=94?= =?UTF-8?q?=20=D0=B8=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20?= =?UTF-8?q?MCP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 5.5 --- backend/drizzle/0015_mcp.sql | 101 ++ .../drizzle/0016_mcp_routing_observations.sql | 12 + backend/drizzle/0017_mcp_controller.sql | 7 + backend/drizzle/0018_mcp_owner_validation.sql | 3 + backend/drizzle/0019_mcp_recovery.sql | 7 + backend/drizzle/0020_mcp_management.sql | 8 + backend/drizzle/0021_mcp_path_checks.sql | 8 + backend/drizzle/0022_mcp_recovery_intents.sql | 7 + backend/drizzle/0023_mcp_protocol_limits.sql | 6 + backend/drizzle/0024_user_interface_types.sql | 4 + .../drizzle/0025_mcp_portal_revocation.sql | 11 + .../drizzle/0026_mcp_owner_claims_issued.sql | 3 + backend/drizzle/0027_mcp_readiness.sql | 8 + backend/drizzle/0028_mcp_artifact_scope.sql | 2 + backend/drizzle/0029_mcp_routing_sources.sql | 23 + ...30_mcp_scoped_configuration_checkpoint.sql | 6 + backend/drizzle/0031_mcp_scoped_recovery.sql | 24 + backend/drizzle/0032_flow_write_receipts.sql | 10 + .../drizzle/0033_flow_settings_intents.sql | 11 + .../drizzle/0034_vrf_production_tickets.sql | 19 + backend/drizzle/0035_server_probe_journal.sql | 67 ++ .../drizzle/0036_diagnostic_cancellations.sql | 14 + .../drizzle/0037_server_probe_storage_v2.sql | 226 +++++ ..._server_poll_journal_and_writer_guards.sql | 59 ++ backend/drizzle/0039_server_poll_storage.sql | 175 ++++ .../drizzle/0040_mcp_routing_reservations.sql | 56 ++ .../0041_server_poll_terminal_archives.sql | 333 +++++++ .../0042_server_poll_settings_epoch.sql | 49 + .../0043_mcp_routing_forward_charges.sql | 62 ++ ...044_mcp_routing_manual_inverse_charges.sql | 93 ++ .../0045_mcp_operation_replay_tombstones.sql | 13 + .../0046_server_poll_operation_retirement.sql | 186 ++++ .../0047_server_poll_retirement_claims.sql | 40 + .../0048_mcp_ordinary_retirement_claims.sql | 27 + ...0049_mcp_ordinary_telemetry_retirement.sql | 18 + .../0050_server_probe_retired_sources.sql | 22 + .../0051_server_probe_retirement_claims.sql | 52 + .../drizzle/0052_server_probe_retired_gc.sql | 96 ++ .../0053_server_probe_terminal_proofs.sql | 74 ++ backend/drizzle/0054_mcp_geoip_retirement.sql | 18 + .../0055_mcp_backup_schedule_retirement.sql | 18 + .../0056_mcp_manual_inverse_recovery_v8.sql | 43 + .../0057_mcp_routing_charge_audit_index.sql | 8 + .../db/application-operator-transactions.ts | 194 ++++ .../application-writer-profile-canonical.ts | 9 + .../application-writer-profile-format.test.ts | 16 + .../db/application-writer-profile-format.ts | 56 ++ ...profile-registry-five-query-oracle.test.ts | 52 + ...writer-profile-registry-read-model.test.ts | 55 ++ ...riter-profile-registry-read-parity.test.ts | 117 +++ ...ation-writer-profile-registry-read.test.ts | 14 + ...pplication-writer-profile-registry-read.ts | 128 +++ ...-writer-profile-registry-structure.test.ts | 55 ++ ...ation-writer-profile-registry-structure.ts | 137 +++ backend/src/db/bootstrap.ts | 117 ++- .../src/db/controlled-migration-history.ts | 75 ++ .../src/db/controlled-migrations.sql.test.ts | 107 ++ backend/src/db/controlled-migrations.test.ts | 322 ++++++ backend/src/db/controlled-migrations.ts | 146 +++ .../db/controlled-schema-reference-core.ts | 254 +++++ .../controlled-schema-reference.sql.test.ts | 164 ++++ backend/src/db/controlled-schema-reference.ts | 9 + .../db/controlled-schema-verification.test.ts | 157 +++ .../src/db/controlled-schema-verification.ts | 142 +++ backend/src/db/database-restore-data.ts | 44 + .../db/database-restore-quarantine-profile.ts | 70 ++ .../db/database-restore-quarantine.test.ts | 85 ++ backend/src/db/database-restore-quarantine.ts | 141 +++ backend/src/db/database-runtime-schema.ts | 78 ++ backend/src/db/database-runtime-source.ts | 257 +++++ .../database-runtime-writer-profile.test.ts | 37 + .../src/db/database-runtime-writer-profile.ts | 47 + .../db/database-schema-control-model.test.ts | 54 + .../src/db/database-schema-control-model.ts | 115 +++ .../db/database-schema-control.sql.test.ts | 197 ++++ .../src/db/database-schema-control.test.ts | 197 ++++ backend/src/db/database-schema-control.ts | 170 ++++ .../db/database-source-activation-record.ts | 32 + .../src/db/database-source-binding.test.ts | 180 ++++ backend/src/db/database-source-binding.ts | 112 +++ ...abase-source-capture-adopted.sql-helper.ts | 137 +++ ...atabase-source-capture-adopted.sql.test.ts | 180 ++++ .../database-source-observation.sql-helper.ts | 51 + .../db/database-source-observation.test.ts | 118 +++ backend/src/db/database-source-observation.ts | 120 +++ backend/src/db/index.ts | 76 +- backend/src/db/legacy-poll-storage-fixture.ts | 26 + backend/src/db/legacy-probe-0037-body.ts | 17 + backend/src/db/legacy-probe-fixture.ts | 34 + backend/src/db/legacy-source58-transition.ts | 95 ++ .../db/migrate-partition-admission.test.ts | 59 ++ backend/src/db/migrate.ts | 75 +- .../db/migration-data-inventory.sql.test.ts | 155 +++ .../src/db/migration-data-inventory.test.ts | 241 +++++ backend/src/db/migration-data-inventory.ts | 266 +++++ backend/src/db/migration-manifest.test.ts | 247 +++++ backend/src/db/migration-manifest.ts | 927 ++++++++++++++++++ .../db/migration-schema-inventory.sql.test.ts | 119 +++ .../src/db/migration-schema-inventory.test.ts | 360 +++++++ backend/src/db/migration-schema-inventory.ts | 474 +++++++++ ...ration-schema-restore-layout.sql-helper.ts | 114 +++ backend/src/db/operation-lock.ts | 36 +- backend/src/db/partition-guard-inheritance.ts | 186 ++++ backend/src/db/partitions-writer.test.ts | 80 ++ backend/src/db/partitions.ts | 159 ++- backend/src/db/pg-schema.test.ts | 2 + .../proposals/server-poll-settings-epoch.sql | 46 + .../src/db/proposals/server-poll-storage.sql | 170 ++++ .../server-probe-legacy-0037-owned.ts | 113 +++ .../db/proposals/server-probe-storage-v2.sql | 255 +++++ .../routing-reservation-schema.owned.test.ts | 121 +++ backend/src/db/schema.ts | 93 +- ...server-poll-settings-epoch.owned-helper.ts | 140 +++ .../server-poll-terminal-archives.sql.test.ts | 109 ++ .../sqlite-import-control-writes.sql.test.ts | 167 ++++ .../src/db/sqlite-import-control-writes.ts | 61 ++ .../sqlite-import-partition-admission.test.ts | 52 + backend/src/db/sqlite-import.ts | 46 +- packages/contracts/package.json | 12 + packages/contracts/src/bridge-vlan.ts | 23 + packages/contracts/src/index.ts | 1 + packages/contracts/src/ipsec.ts | 22 +- packages/contracts/src/logs.ts | 9 +- packages/contracts/src/mcp-routing.test.ts | 16 + packages/contracts/src/mcp-routing.ts | 63 ++ packages/contracts/src/mcp.ts | 174 ++++ packages/contracts/src/servers.test.ts | 8 + packages/contracts/src/servers.ts | 11 +- packages/contracts/src/traffic-flow.ts | 33 + packages/contracts/src/uptime-speed.ts | 1 + packages/contracts/src/users.test.ts | 9 + packages/contracts/src/users.ts | 9 +- 132 files changed, 12164 insertions(+), 125 deletions(-) create mode 100644 backend/drizzle/0015_mcp.sql create mode 100644 backend/drizzle/0016_mcp_routing_observations.sql create mode 100644 backend/drizzle/0017_mcp_controller.sql create mode 100644 backend/drizzle/0018_mcp_owner_validation.sql create mode 100644 backend/drizzle/0019_mcp_recovery.sql create mode 100644 backend/drizzle/0020_mcp_management.sql create mode 100644 backend/drizzle/0021_mcp_path_checks.sql create mode 100644 backend/drizzle/0022_mcp_recovery_intents.sql create mode 100644 backend/drizzle/0023_mcp_protocol_limits.sql create mode 100644 backend/drizzle/0024_user_interface_types.sql create mode 100644 backend/drizzle/0025_mcp_portal_revocation.sql create mode 100644 backend/drizzle/0026_mcp_owner_claims_issued.sql create mode 100644 backend/drizzle/0027_mcp_readiness.sql create mode 100644 backend/drizzle/0028_mcp_artifact_scope.sql create mode 100644 backend/drizzle/0029_mcp_routing_sources.sql create mode 100644 backend/drizzle/0030_mcp_scoped_configuration_checkpoint.sql create mode 100644 backend/drizzle/0031_mcp_scoped_recovery.sql create mode 100644 backend/drizzle/0032_flow_write_receipts.sql create mode 100644 backend/drizzle/0033_flow_settings_intents.sql create mode 100644 backend/drizzle/0034_vrf_production_tickets.sql create mode 100644 backend/drizzle/0035_server_probe_journal.sql create mode 100644 backend/drizzle/0036_diagnostic_cancellations.sql create mode 100644 backend/drizzle/0037_server_probe_storage_v2.sql create mode 100644 backend/drizzle/0038_server_poll_journal_and_writer_guards.sql create mode 100644 backend/drizzle/0039_server_poll_storage.sql create mode 100644 backend/drizzle/0040_mcp_routing_reservations.sql create mode 100644 backend/drizzle/0041_server_poll_terminal_archives.sql create mode 100644 backend/drizzle/0042_server_poll_settings_epoch.sql create mode 100644 backend/drizzle/0043_mcp_routing_forward_charges.sql create mode 100644 backend/drizzle/0044_mcp_routing_manual_inverse_charges.sql create mode 100644 backend/drizzle/0045_mcp_operation_replay_tombstones.sql create mode 100644 backend/drizzle/0046_server_poll_operation_retirement.sql create mode 100644 backend/drizzle/0047_server_poll_retirement_claims.sql create mode 100644 backend/drizzle/0048_mcp_ordinary_retirement_claims.sql create mode 100644 backend/drizzle/0049_mcp_ordinary_telemetry_retirement.sql create mode 100644 backend/drizzle/0050_server_probe_retired_sources.sql create mode 100644 backend/drizzle/0051_server_probe_retirement_claims.sql create mode 100644 backend/drizzle/0052_server_probe_retired_gc.sql create mode 100644 backend/drizzle/0053_server_probe_terminal_proofs.sql create mode 100644 backend/drizzle/0054_mcp_geoip_retirement.sql create mode 100644 backend/drizzle/0055_mcp_backup_schedule_retirement.sql create mode 100644 backend/drizzle/0056_mcp_manual_inverse_recovery_v8.sql create mode 100644 backend/drizzle/0057_mcp_routing_charge_audit_index.sql create mode 100644 backend/src/db/application-operator-transactions.ts create mode 100644 backend/src/db/application-writer-profile-canonical.ts create mode 100644 backend/src/db/application-writer-profile-format.test.ts create mode 100644 backend/src/db/application-writer-profile-format.ts create mode 100644 backend/src/db/application-writer-profile-registry-five-query-oracle.test.ts create mode 100644 backend/src/db/application-writer-profile-registry-read-model.test.ts create mode 100644 backend/src/db/application-writer-profile-registry-read-parity.test.ts create mode 100644 backend/src/db/application-writer-profile-registry-read.test.ts create mode 100644 backend/src/db/application-writer-profile-registry-read.ts create mode 100644 backend/src/db/application-writer-profile-registry-structure.test.ts create mode 100644 backend/src/db/application-writer-profile-registry-structure.ts create mode 100644 backend/src/db/controlled-migration-history.ts create mode 100644 backend/src/db/controlled-migrations.sql.test.ts create mode 100644 backend/src/db/controlled-migrations.test.ts create mode 100644 backend/src/db/controlled-migrations.ts create mode 100644 backend/src/db/controlled-schema-reference-core.ts create mode 100644 backend/src/db/controlled-schema-reference.sql.test.ts create mode 100644 backend/src/db/controlled-schema-reference.ts create mode 100644 backend/src/db/controlled-schema-verification.test.ts create mode 100644 backend/src/db/controlled-schema-verification.ts create mode 100644 backend/src/db/database-restore-data.ts create mode 100644 backend/src/db/database-restore-quarantine-profile.ts create mode 100644 backend/src/db/database-restore-quarantine.test.ts create mode 100644 backend/src/db/database-restore-quarantine.ts create mode 100644 backend/src/db/database-runtime-schema.ts create mode 100644 backend/src/db/database-runtime-source.ts create mode 100644 backend/src/db/database-runtime-writer-profile.test.ts create mode 100644 backend/src/db/database-runtime-writer-profile.ts create mode 100644 backend/src/db/database-schema-control-model.test.ts create mode 100644 backend/src/db/database-schema-control-model.ts create mode 100644 backend/src/db/database-schema-control.sql.test.ts create mode 100644 backend/src/db/database-schema-control.test.ts create mode 100644 backend/src/db/database-schema-control.ts create mode 100644 backend/src/db/database-source-activation-record.ts create mode 100644 backend/src/db/database-source-binding.test.ts create mode 100644 backend/src/db/database-source-binding.ts create mode 100644 backend/src/db/database-source-capture-adopted.sql-helper.ts create mode 100644 backend/src/db/database-source-capture-adopted.sql.test.ts create mode 100644 backend/src/db/database-source-observation.sql-helper.ts create mode 100644 backend/src/db/database-source-observation.test.ts create mode 100644 backend/src/db/database-source-observation.ts create mode 100644 backend/src/db/legacy-poll-storage-fixture.ts create mode 100644 backend/src/db/legacy-probe-0037-body.ts create mode 100644 backend/src/db/legacy-probe-fixture.ts create mode 100644 backend/src/db/legacy-source58-transition.ts create mode 100644 backend/src/db/migrate-partition-admission.test.ts create mode 100644 backend/src/db/migration-data-inventory.sql.test.ts create mode 100644 backend/src/db/migration-data-inventory.test.ts create mode 100644 backend/src/db/migration-data-inventory.ts create mode 100644 backend/src/db/migration-manifest.test.ts create mode 100644 backend/src/db/migration-manifest.ts create mode 100644 backend/src/db/migration-schema-inventory.sql.test.ts create mode 100644 backend/src/db/migration-schema-inventory.test.ts create mode 100644 backend/src/db/migration-schema-inventory.ts create mode 100644 backend/src/db/migration-schema-restore-layout.sql-helper.ts create mode 100644 backend/src/db/partition-guard-inheritance.ts create mode 100644 backend/src/db/partitions-writer.test.ts create mode 100644 backend/src/db/proposals/server-poll-settings-epoch.sql create mode 100644 backend/src/db/proposals/server-poll-storage.sql create mode 100644 backend/src/db/proposals/server-probe-legacy-0037-owned.ts create mode 100644 backend/src/db/proposals/server-probe-storage-v2.sql create mode 100644 backend/src/db/routing-reservation-schema.owned.test.ts create mode 100644 backend/src/db/server-poll-settings-epoch.owned-helper.ts create mode 100644 backend/src/db/server-poll-terminal-archives.sql.test.ts create mode 100644 backend/src/db/sqlite-import-control-writes.sql.test.ts create mode 100644 backend/src/db/sqlite-import-control-writes.ts create mode 100644 backend/src/db/sqlite-import-partition-admission.test.ts create mode 100644 packages/contracts/src/bridge-vlan.ts create mode 100644 packages/contracts/src/mcp-routing.test.ts create mode 100644 packages/contracts/src/mcp-routing.ts create mode 100644 packages/contracts/src/mcp.ts create mode 100644 packages/contracts/src/servers.test.ts create mode 100644 packages/contracts/src/users.test.ts diff --git a/backend/drizzle/0015_mcp.sql b/backend/drizzle/0015_mcp.sql new file mode 100644 index 00000000..abafc705 --- /dev/null +++ b/backend/drizzle/0015_mcp.sql @@ -0,0 +1,101 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_settings ( + id integer PRIMARY KEY CHECK(id=1), enabled boolean NOT NULL DEFAULT false, + requests_retention_days integer NOT NULL DEFAULT 30 CHECK(requests_retention_days BETWEEN 1 AND 365), + audit_retention_days integer NOT NULL DEFAULT 180 CHECK(audit_retention_days BETWEEN 30 AND 3650), + updated_at timestamptz NOT NULL DEFAULT now() +); +INSERT INTO mcp_settings(id) VALUES(1) ON CONFLICT DO NOTHING; +CREATE TABLE IF NOT EXISTS mcp_connections ( + id uuid PRIMARY KEY, name text NOT NULL, owner_id text NOT NULL, owner_permissions jsonb NOT NULL, + policy jsonb NOT NULL, generation text NOT NULL, created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), last_used_at timestamptz +); +CREATE TABLE IF NOT EXISTS mcp_credentials ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id) ON DELETE CASCADE, + digest text NOT NULL UNIQUE, kind text NOT NULL CHECK(kind IN ('bearer','access','refresh')), + expires_at timestamptz NOT NULL, revoked_at timestamptz, created_at timestamptz NOT NULL DEFAULT now(), + family_id uuid, client_id text, audience text +); +CREATE INDEX IF NOT EXISTS mcp_credentials_connection ON mcp_credentials(connection_id); +CREATE TABLE IF NOT EXISTS mcp_audit ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, actor_id text NOT NULL, action text NOT NULL, + connection_id uuid, payload jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS mcp_audit_time ON mcp_audit(created_at,id); +CREATE TABLE IF NOT EXISTS mcp_requests ( + id uuid PRIMARY KEY, connection_id uuid REFERENCES mcp_connections(id) ON DELETE SET NULL, + actor_id text, tool text NOT NULL, server_ids bigint[] NOT NULL DEFAULT '{}', + status text NOT NULL, input jsonb NOT NULL DEFAULT '{}', result jsonb, error_code text, + duration_ms integer, created_at timestamptz NOT NULL DEFAULT now(), finished_at timestamptz +); +CREATE INDEX IF NOT EXISTS mcp_requests_connection_time ON mcp_requests(connection_id,created_at,id); +CREATE INDEX IF NOT EXISTS mcp_requests_server ON mcp_requests USING gin(server_ids); +CREATE TABLE IF NOT EXISTS mcp_protections ( + id uuid PRIMARY KEY, object_ref jsonb NOT NULL, reason text NOT NULL DEFAULT '', + created_by text NOT NULL, created_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_plans ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id), operation_id text NOT NULL, + operation_version integer NOT NULL, policy_version integer NOT NULL, generation text NOT NULL, + payload jsonb NOT NULL, evidence jsonb NOT NULL, state text NOT NULL, expires_at timestamptz NOT NULL, + approved_by text, approved_at timestamptz, created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS mcp_plans_connection ON mcp_plans(connection_id,created_at); +CREATE TABLE IF NOT EXISTS mcp_operations ( + id uuid PRIMARY KEY, plan_id uuid NOT NULL UNIQUE REFERENCES mcp_plans(id), + connection_id uuid NOT NULL REFERENCES mcp_connections(id), idempotency_key text NOT NULL, + state text NOT NULL DEFAULT 'pending', verification text NOT NULL DEFAULT 'pending', + result jsonb, error_code text, created_at timestamptz NOT NULL DEFAULT now(), finished_at timestamptz, + UNIQUE(connection_id,idempotency_key) +); +CREATE TABLE IF NOT EXISTS mcp_operation_steps ( + operation_id uuid NOT NULL REFERENCES mcp_operations(id), step_index integer NOT NULL, + server_id bigint, state text NOT NULL, before_state jsonb, intended_state jsonb, result jsonb, + created_at timestamptz NOT NULL DEFAULT now(), finished_at timestamptz, + PRIMARY KEY(operation_id,step_index) +); +CREATE TABLE IF NOT EXISTS mcp_budget_buckets ( + connection_id uuid NOT NULL REFERENCES mcp_connections(id) ON DELETE CASCADE, + kind text NOT NULL, bucket timestamptz NOT NULL, used integer NOT NULL CHECK(used>=0), + PRIMARY KEY(connection_id,kind,bucket) +); +CREATE TABLE IF NOT EXISTS mcp_read_leases ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); +CREATE TABLE IF NOT EXISTS mcp_grants ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id) ON DELETE CASCADE, + payload jsonb NOT NULL, revoked_at timestamptz +); +CREATE TABLE IF NOT EXISTS mcp_secrets ( + id uuid PRIMARY KEY, purpose text NOT NULL, server_id bigint, ciphertext text NOT NULL, + created_by text NOT NULL, created_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_artifacts ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id), + filename text NOT NULL, media_type text NOT NULL, ciphertext text NOT NULL, + expires_at timestamptz NOT NULL, created_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_oauth_clients ( + id text PRIMARY KEY, name text NOT NULL, redirect_uris jsonb NOT NULL, enabled boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_oauth_requests ( + id uuid PRIMARY KEY, client_id text NOT NULL REFERENCES mcp_oauth_clients(id), + redirect_uri text NOT NULL, challenge text NOT NULL, state text, resource text NOT NULL, + scope text NOT NULL, expires_at timestamptz NOT NULL, consumed_at timestamptz, + connection_id uuid REFERENCES mcp_connections(id), code_digest text UNIQUE +); +CREATE TABLE IF NOT EXISTS mcp_routing_profiles ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id), + version integer NOT NULL DEFAULT 1, config jsonb NOT NULL, state jsonb NOT NULL DEFAULT '{}', + enabled boolean NOT NULL DEFAULT false, grant_expires_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_drift ( + id uuid PRIMARY KEY, operation_id uuid NOT NULL REFERENCES mcp_operations(id), + server_id bigint, expected jsonb NOT NULL, observed jsonb NOT NULL, + acknowledged_at timestamptz, detected_at timestamptz NOT NULL DEFAULT now() +); +COMMIT; diff --git a/backend/drizzle/0016_mcp_routing_observations.sql b/backend/drizzle/0016_mcp_routing_observations.sql new file mode 100644 index 00000000..b7301661 --- /dev/null +++ b/backend/drizzle/0016_mcp_routing_observations.sql @@ -0,0 +1,12 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_routing_observations ( + id uuid PRIMARY KEY, src_server_id bigint NOT NULL, dst_server_id bigint NOT NULL, + src_interface text NOT NULL, dst_interface text NOT NULL, family text NOT NULL CHECK(family IN ('ipv4','ipv6')), + routing_table text NOT NULL DEFAULT 'main', ping_ms double precision, loss_pct double precision, + dl_mbps double precision, ul_mbps double precision, ping_at timestamptz, bandwidth_at timestamptz, + context_verified boolean NOT NULL DEFAULT false, source text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), CHECK(ping_ms IS NULL OR ping_ms>=0), + CHECK(loss_pct IS NULL OR loss_pct BETWEEN 0 AND 100), CHECK(dl_mbps IS NULL OR dl_mbps>=0), CHECK(ul_mbps IS NULL OR ul_mbps>=0) +); +CREATE INDEX IF NOT EXISTS mcp_routing_observations_leg ON mcp_routing_observations(src_server_id,dst_server_id,src_interface,family,routing_table,created_at DESC); +COMMIT; diff --git a/backend/drizzle/0017_mcp_controller.sql b/backend/drizzle/0017_mcp_controller.sql new file mode 100644 index 00000000..988405bd --- /dev/null +++ b/backend/drizzle/0017_mcp_controller.sql @@ -0,0 +1,7 @@ +BEGIN; +ALTER TABLE mcp_routing_profiles ADD COLUMN IF NOT EXISTS credential_id uuid REFERENCES mcp_credentials(id); +ALTER TABLE mcp_routing_profiles ADD COLUMN IF NOT EXISTS generation text; +ALTER TABLE mcp_routing_profiles ADD COLUMN IF NOT EXISTS policy_version integer; +CREATE INDEX IF NOT EXISTS mcp_routing_profiles_enabled ON mcp_routing_profiles(updated_at) WHERE enabled; +CREATE UNIQUE INDEX IF NOT EXISTS mcp_drift_open_step ON mcp_drift(operation_id,(expected->>'stepIndex')) WHERE acknowledged_at IS NULL; +COMMIT; diff --git a/backend/drizzle/0018_mcp_owner_validation.sql b/backend/drizzle/0018_mcp_owner_validation.sql new file mode 100644 index 00000000..fd91b038 --- /dev/null +++ b/backend/drizzle/0018_mcp_owner_validation.sql @@ -0,0 +1,3 @@ +BEGIN; +ALTER TABLE mcp_connections ADD COLUMN IF NOT EXISTS owner_valid_until timestamptz NOT NULL DEFAULT now(); +COMMIT; diff --git a/backend/drizzle/0019_mcp_recovery.sql b/backend/drizzle/0019_mcp_recovery.sql new file mode 100644 index 00000000..11d803b6 --- /dev/null +++ b/backend/drizzle/0019_mcp_recovery.sql @@ -0,0 +1,7 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_operation_checkpoints ( + operation_id uuid NOT NULL REFERENCES mcp_operations(id), step_index integer NOT NULL, + before_sealed text NOT NULL, after_sealed text, created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY(operation_id,step_index) +); +COMMIT; diff --git a/backend/drizzle/0020_mcp_management.sql b/backend/drizzle/0020_mcp_management.sql new file mode 100644 index 00000000..e83b8806 --- /dev/null +++ b/backend/drizzle/0020_mcp_management.sql @@ -0,0 +1,8 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_management_channels ( + id uuid PRIMARY KEY, server_id bigint NOT NULL UNIQUE REFERENCES servers(id) ON DELETE CASCADE, + config_sealed text NOT NULL, identity_hash text NOT NULL, attestation text NOT NULL, + created_by text NOT NULL, generation text NOT NULL, expires_at timestamptz NOT NULL, + verified_at timestamptz NOT NULL DEFAULT now(), revoked_at timestamptz +); +COMMIT; diff --git a/backend/drizzle/0021_mcp_path_checks.sql b/backend/drizzle/0021_mcp_path_checks.sql new file mode 100644 index 00000000..6396545f --- /dev/null +++ b/backend/drizzle/0021_mcp_path_checks.sql @@ -0,0 +1,8 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_path_checks ( + id uuid PRIMARY KEY, connection_id uuid NOT NULL REFERENCES mcp_connections(id), + profile_hash text NOT NULL, candidate_id text NOT NULL, status text NOT NULL CHECK(status IN ('passed','failed','unknown')), + evidence jsonb NOT NULL, created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS mcp_path_checks_latest ON mcp_path_checks(connection_id,profile_hash,candidate_id,created_at DESC); +COMMIT; diff --git a/backend/drizzle/0022_mcp_recovery_intents.sql b/backend/drizzle/0022_mcp_recovery_intents.sql new file mode 100644 index 00000000..2fdc29c6 --- /dev/null +++ b/backend/drizzle/0022_mcp_recovery_intents.sql @@ -0,0 +1,7 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_recovery_intents ( + operation_id uuid PRIMARY KEY REFERENCES mcp_operations(id), state text NOT NULL, + authority_profile_id uuid NOT NULL REFERENCES mcp_routing_profiles(id), + created_at timestamptz NOT NULL DEFAULT now(), finished_at timestamptz +); +COMMIT; diff --git a/backend/drizzle/0023_mcp_protocol_limits.sql b/backend/drizzle/0023_mcp_protocol_limits.sql new file mode 100644 index 00000000..1ec810d4 --- /dev/null +++ b/backend/drizzle/0023_mcp_protocol_limits.sql @@ -0,0 +1,6 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_unauthorized_buckets ( + bucket timestamptz NOT NULL, client_hash text NOT NULL, reason text NOT NULL, + attempts integer NOT NULL DEFAULT 1, PRIMARY KEY(bucket,client_hash,reason) +); +COMMIT; diff --git a/backend/drizzle/0024_user_interface_types.sql b/backend/drizzle/0024_user_interface_types.sql new file mode 100644 index 00000000..0d209333 --- /dev/null +++ b/backend/drizzle/0024_user_interface_types.sql @@ -0,0 +1,4 @@ +BEGIN; +ALTER TABLE user_interface_bindings DROP CONSTRAINT IF EXISTS user_interface_bindings_interface_type_check; +ALTER TABLE user_interface_bindings ADD CONSTRAINT user_interface_bindings_interface_type_check CHECK(interface_type IN ('ether','bridge','vlan','vxlan','gre','wg','ipsec','other')); +COMMIT; diff --git a/backend/drizzle/0025_mcp_portal_revocation.sql b/backend/drizzle/0025_mcp_portal_revocation.sql new file mode 100644 index 00000000..b0e5f9a4 --- /dev/null +++ b/backend/drizzle/0025_mcp_portal_revocation.sql @@ -0,0 +1,11 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_owner_revocations ( + owner_id text PRIMARY KEY, invalid_before timestamptz NOT NULL, + updated_at timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS mcp_portal_events ( + id uuid PRIMARY KEY, owner_id text NOT NULL, payload_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS mcp_portal_events_time ON mcp_portal_events(created_at); +COMMIT; diff --git a/backend/drizzle/0026_mcp_owner_claims_issued.sql b/backend/drizzle/0026_mcp_owner_claims_issued.sql new file mode 100644 index 00000000..c55e64e9 --- /dev/null +++ b/backend/drizzle/0026_mcp_owner_claims_issued.sql @@ -0,0 +1,3 @@ +BEGIN; +ALTER TABLE mcp_connections ADD COLUMN IF NOT EXISTS owner_claims_issued_at timestamptz NOT NULL DEFAULT to_timestamp(0); +COMMIT; diff --git a/backend/drizzle/0027_mcp_readiness.sql b/backend/drizzle/0027_mcp_readiness.sql new file mode 100644 index 00000000..a1713014 --- /dev/null +++ b/backend/drizzle/0027_mcp_readiness.sql @@ -0,0 +1,8 @@ +CREATE INDEX IF NOT EXISTS mcp_audit_handshake_cooldown + ON mcp_audit(connection_id, created_at DESC) + WHERE action = 'connection.handshake.start'; +CREATE INDEX IF NOT EXISTS mcp_credentials_readiness_expired + ON mcp_credentials(connection_id, expires_at) + WHERE client_id = 'mcp-readiness-probe'; +CREATE INDEX IF NOT EXISTS mcp_grants_credential + ON mcp_grants(connection_id, (payload->>'credentialId')); diff --git a/backend/drizzle/0028_mcp_artifact_scope.sql b/backend/drizzle/0028_mcp_artifact_scope.sql new file mode 100644 index 00000000..7da202f5 --- /dev/null +++ b/backend/drizzle/0028_mcp_artifact_scope.sql @@ -0,0 +1,2 @@ +ALTER TABLE mcp_artifacts ADD COLUMN IF NOT EXISTS access_scope jsonb; +CREATE INDEX IF NOT EXISTS mcp_artifacts_expiry ON mcp_artifacts(expires_at); diff --git a/backend/drizzle/0029_mcp_routing_sources.sql b/backend/drizzle/0029_mcp_routing_sources.sql new file mode 100644 index 00000000..688aa804 --- /dev/null +++ b/backend/drizzle/0029_mcp_routing_sources.sql @@ -0,0 +1,23 @@ +BEGIN; +ALTER TABLE servers ADD COLUMN IF NOT EXISTS routing_endpoint_epoch uuid NOT NULL DEFAULT gen_random_uuid(); +CREATE OR REPLACE FUNCTION mm_routing_endpoint_epoch() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF ROW(NEW.host,NEW.port,NEW.username,NEW.password,NEW.use_ssl,NEW.verify_ssl,NEW.enabled) + IS DISTINCT FROM ROW(OLD.host,OLD.port,OLD.username,OLD.password,OLD.use_ssl,OLD.verify_ssl,OLD.enabled) THEN + NEW.routing_endpoint_epoch := gen_random_uuid(); + ELSE + NEW.routing_endpoint_epoch := OLD.routing_endpoint_epoch; + END IF; + RETURN NEW; +END $$; +DROP TRIGGER IF EXISTS mm_routing_endpoint_epoch ON servers; +CREATE TRIGGER mm_routing_endpoint_epoch BEFORE UPDATE ON servers FOR EACH ROW EXECUTE FUNCTION mm_routing_endpoint_epoch(); +ALTER TABLE mcp_routing_observations ADD COLUMN IF NOT EXISTS source_proof text; +DO $$ BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname='mcp_routing_observation_source_proof' AND conrelid='mcp_routing_observations'::regclass) THEN + ALTER TABLE mcp_routing_observations ADD CONSTRAINT mcp_routing_observation_source_proof CHECK(source_proof IS NULL OR source_proof ~ '^[a-f0-9]{64}$'); + END IF; +END $$; +-- Existing measurements stay readable but unsigned. Never attest old samples +-- using today's endpoint: that would invent capture-time provenance. +COMMIT; diff --git a/backend/drizzle/0030_mcp_scoped_configuration_checkpoint.sql b/backend/drizzle/0030_mcp_scoped_configuration_checkpoint.sql new file mode 100644 index 00000000..3356853d --- /dev/null +++ b/backend/drizzle/0030_mcp_scoped_configuration_checkpoint.sql @@ -0,0 +1,6 @@ +BEGIN; +-- Service-owned exact read-back must survive a subsequent network probe failure. +-- Generic after_sealed remains an array of resource snapshots. +ALTER TABLE mcp_operation_checkpoints ADD COLUMN IF NOT EXISTS configuration_sealed text; +ALTER TABLE mcp_operation_checkpoints ADD COLUMN IF NOT EXISTS configuration_proof text; +COMMIT; diff --git a/backend/drizzle/0031_mcp_scoped_recovery.sql b/backend/drizzle/0031_mcp_scoped_recovery.sql new file mode 100644 index 00000000..08b40e2b --- /dev/null +++ b/backend/drizzle/0031_mcp_scoped_recovery.sql @@ -0,0 +1,24 @@ +BEGIN; +CREATE TABLE IF NOT EXISTS mcp_scoped_recovery_intents ( + source_operation_id uuid NOT NULL REFERENCES mcp_operations(id), + source_step_index integer NOT NULL CHECK(source_step_index>=0), + source_receipt_id uuid NOT NULL, + source_configuration_proof text NOT NULL, + recovery_receipt_id uuid NOT NULL UNIQUE, + recovery_operation_id uuid REFERENCES mcp_operations(id), + mode text NOT NULL CHECK(mode IN ('manual','reserved')), + authority_profile_id uuid REFERENCES mcp_routing_profiles(id), + authority_profile_version integer, + intent_sealed text NOT NULL, + intent_proof text NOT NULL, + state text NOT NULL CHECK(state IN ('intent','verified','unknown')), + deadline timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + verified_sealed text, + verified_proof text, + PRIMARY KEY(source_operation_id,source_step_index), + CHECK(mode='manual' OR authority_profile_id IS NOT NULL AND authority_profile_version IS NOT NULL), + CHECK((state='verified')=(verified_sealed IS NOT NULL AND verified_proof IS NOT NULL)) +); +COMMIT; diff --git a/backend/drizzle/0032_flow_write_receipts.sql b/backend/drizzle/0032_flow_write_receipts.sql new file mode 100644 index 00000000..051a7504 --- /dev/null +++ b/backend/drizzle/0032_flow_write_receipts.sql @@ -0,0 +1,10 @@ +-- Immutable commit evidence shared by ingestion and REST/MCP fact rebuild. +-- No raw flow payload, IP addresses, credentials or notification contents belong here. +CREATE TABLE IF NOT EXISTS flow_write_receipts ( + batch_id uuid PRIMARY KEY, + kind text NOT NULL CHECK (kind IN ('ingest', 'facts', 'rebuild')), + source_proof text NOT NULL CHECK (source_proof ~ '^[a-f0-9]{64}$'), + result jsonb NOT NULL CHECK (jsonb_typeof(result) = 'object' AND octet_length(result::text) <= 65536), + committed_at timestamptz NOT NULL DEFAULT now() +); +-- Receipts are intentionally not pruned: an owned unknown batch may still need them. diff --git a/backend/drizzle/0033_flow_settings_intents.sql b/backend/drizzle/0033_flow_settings_intents.sql new file mode 100644 index 00000000..c2bec1b9 --- /dev/null +++ b/backend/drizzle/0033_flow_settings_intents.sql @@ -0,0 +1,11 @@ +-- A claim survives unknown results and is never recycled for another attempt. +CREATE TABLE IF NOT EXISTS flow_settings_intents ( + receipt_id uuid NOT NULL, + phase text NOT NULL CHECK (phase IN ('configuration', 'listener')), + request_proof text NOT NULL, + state text NOT NULL DEFAULT 'claimed' CHECK (state IN ('claimed', 'completed')), + result jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (receipt_id, phase) +); +INSERT INTO traffic_flow_settings(id) VALUES(1) ON CONFLICT DO NOTHING; diff --git a/backend/drizzle/0034_vrf_production_tickets.sql b/backend/drizzle/0034_vrf_production_tickets.sql new file mode 100644 index 00000000..600a9cbe --- /dev/null +++ b/backend/drizzle/0034_vrf_production_tickets.sql @@ -0,0 +1,19 @@ +-- Precondition receipts only. No executor capability is granted by this table. +CREATE TABLE IF NOT EXISTS vrf_production_tickets ( + ticket_id uuid PRIMARY KEY, + plan_id uuid NOT NULL UNIQUE REFERENCES mcp_plans(id), + connection_id uuid NOT NULL REFERENCES mcp_connections(id), + server_id bigint NOT NULL REFERENCES servers(id), + owner_id text NOT NULL, + action_id text NOT NULL CHECK (action_id IN ('ros.vrf.create','ros.vrf.update','ros.vrf.delete')), + action_version integer NOT NULL CHECK (action_version > 0), + source_proof text NOT NULL CHECK (source_proof ~ '^[a-f0-9]{64}$'), + digest text NOT NULL CHECK (digest ~ '^[a-f0-9]{64}$'), + ciphertext text NOT NULL, + state text NOT NULL DEFAULT 'issued' CHECK (state IN ('issued','claimed')), + issued_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + claimed_at timestamptz, + CHECK (expires_at > issued_at AND expires_at <= issued_at + interval '5 minutes'), + CHECK ((state='issued' AND claimed_at IS NULL) OR (state='claimed' AND claimed_at IS NOT NULL)) +); diff --git a/backend/drizzle/0035_server_probe_journal.sql b/backend/drizzle/0035_server_probe_journal.sql new file mode 100644 index 00000000..06ee09de --- /dev/null +++ b/backend/drizzle/0035_server_probe_journal.sql @@ -0,0 +1,67 @@ +-- Durable diagnostic attempts; this migration registers no MCP action. +BEGIN; + +CREATE TABLE server_probe_executions ( + receipt_id uuid PRIMARY KEY, binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')), + execution_id uuid NOT NULL, actor_id text NOT NULL CHECK(length(actor_id) BETWEEN 1 AND 200), + authentication text NOT NULL CHECK(authentication IN ('mcp','jwt','local')), + mcp_connection_id uuid REFERENCES mcp_connections(id), mcp_credential_id uuid REFERENCES mcp_credentials(id), request_id text, + authority_issued_at bigint NOT NULL CHECK(authority_issued_at>=0), authority_expires_at timestamptz NOT NULL, + source_server_id integer NOT NULL CHECK(source_server_id>0), destination_server_id integer CHECK(destination_server_id>0), + tool text NOT NULL CHECK(tool IN ('ping','traceroute','bandwidth','dns','route','mtu')), + state text NOT NULL DEFAULT 'claimed' CHECK(state IN ('claimed','completed','failed','cancelled')), + attempted_stages integer CHECK(attempted_stages BETWEEN 0 AND 11), completed_stages integer CHECK(completed_stages BETWEEN 0 AND attempted_stages), + receipt_ciphertext text CHECK(octet_length(receipt_ciphertext)<=262144), receipt_proof text CHECK(receipt_proof ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), finished_at timestamptz, + UNIQUE(identity_kind,execution_id), + CHECK((identity_kind='mcp' AND authentication='mcp' AND mcp_connection_id IS NOT NULL AND mcp_credential_id IS NOT NULL AND request_id IS NULL) + OR (identity_kind='rest' AND authentication IN ('jwt','local') AND mcp_connection_id IS NULL AND mcp_credential_id IS NULL AND request_id IS NOT NULL AND length(request_id) BETWEEN 1 AND 200)), + CHECK((tool='bandwidth')=(destination_server_id IS NOT NULL)), + CHECK((state='claimed' AND receipt_ciphertext IS NULL AND receipt_proof IS NULL AND finished_at IS NULL AND attempted_stages IS NULL AND completed_stages IS NULL) + OR (state<>'claimed' AND receipt_ciphertext IS NOT NULL AND receipt_proof IS NOT NULL AND finished_at IS NOT NULL AND attempted_stages IS NOT NULL AND completed_stages IS NOT NULL)) +); +CREATE TABLE server_probe_execution_stages ( + receipt_id uuid NOT NULL REFERENCES server_probe_executions(receipt_id), stage_index integer NOT NULL CHECK(stage_index BETWEEN 1 AND 11), + kind text NOT NULL CHECK(kind IN ('source-dns-A','native-ping','native-traceroute','native-bandwidth','native-route-read', + 'manager-dns-A','manager-dns-AAAA','manager-dns-MX','manager-dns-NS','manager-dns-TXT','manager-dns-CNAME','manager-dns-PTR', + 'native-mtu-1500','native-mtu-1492','native-mtu-1480','native-mtu-1476','native-mtu-1472','native-mtu-1468','native-mtu-1400','native-mtu-1280','native-mtu-1024','native-mtu-576')), + timeout_ms integer NOT NULL CHECK(timeout_ms BETWEEN 1 AND 120000), + state text NOT NULL DEFAULT 'intent' CHECK(state IN ('intent','received')), + outcome text CHECK(outcome IN ('received','source-unavailable')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), received_at timestamptz, + PRIMARY KEY(receipt_id,stage_index), + CHECK((state='intent' AND outcome IS NULL AND received_at IS NULL) OR (state='received' AND outcome IS NOT NULL AND received_at IS NOT NULL)) +); +CREATE TABLE server_probe_execution_quarantines ( + receipt_id uuid PRIMARY KEY, binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')), execution_id uuid NOT NULL, + identity_proof text NOT NULL CHECK(identity_proof ~ '^[a-f0-9]{64}$'), + error_code text NOT NULL CHECK(error_code ~ '^[A-Z_]{1,64}$'), + attempted_stages integer NOT NULL CHECK(attempted_stages BETWEEN 0 AND 11), completed_stages integer NOT NULL CHECK(completed_stages BETWEEN 0 AND attempted_stages), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), UNIQUE(identity_kind,execution_id) +); +CREATE FUNCTION server_probe_execution_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'probe execution journal is append only'; END IF; + IF OLD.state<>'claimed' OR (to_jsonb(OLD)-ARRAY['state','attempted_stages','completed_stages','receipt_ciphertext','receipt_proof','finished_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['state','attempted_stages','completed_stages','receipt_ciphertext','receipt_proof','finished_at']) OR NEW.state='claimed' + THEN RAISE EXCEPTION 'probe execution identity or terminal receipt is immutable'; END IF; + IF EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=OLD.receipt_id) THEN RAISE EXCEPTION 'probe execution quarantined'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_execution_immutable BEFORE UPDATE OR DELETE ON server_probe_executions FOR EACH ROW EXECUTE FUNCTION server_probe_execution_immutable(); +CREATE FUNCTION server_probe_stage_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN RAISE EXCEPTION 'probe stage journal is append only'; END IF; + PERFORM receipt_id FROM server_probe_executions WHERE receipt_id=NEW.receipt_id AND state='claimed' FOR UPDATE; + IF NOT FOUND OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=NEW.receipt_id) THEN RAISE EXCEPTION 'probe attempt is not executable'; END IF; + IF TG_OP='INSERT' AND (NEW.stage_index<>(SELECT COALESCE(max(stage_index),0)+1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id) + OR EXISTS(SELECT 1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id AND state<>'received')) + THEN RAISE EXCEPTION 'probe stages must be serial and contiguous'; END IF; + IF TG_OP='UPDATE' AND (OLD.state<>'intent' OR NEW.state<>'received' OR (to_jsonb(OLD)-ARRAY['state','outcome','received_at']) IS DISTINCT FROM (to_jsonb(NEW)-ARRAY['state','outcome','received_at'])) + THEN RAISE EXCEPTION 'probe stage terminal state is immutable'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_stage_immutable BEFORE INSERT OR UPDATE OR DELETE ON server_probe_execution_stages FOR EACH ROW EXECUTE FUNCTION server_probe_stage_immutable(); +CREATE FUNCTION server_probe_quarantine_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'probe quarantine is immutable'; END $$; +CREATE TRIGGER server_probe_quarantine_immutable BEFORE UPDATE OR DELETE ON server_probe_execution_quarantines FOR EACH ROW EXECUTE FUNCTION server_probe_quarantine_immutable(); + +INSERT INTO schema_migrations(id) VALUES ('0035_server_probe_journal') ON CONFLICT (id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0036_diagnostic_cancellations.sql b/backend/drizzle/0036_diagnostic_cancellations.sql new file mode 100644 index 00000000..3c678241 --- /dev/null +++ b/backend/drizzle/0036_diagnostic_cancellations.sql @@ -0,0 +1,14 @@ +-- Durable stop requests for opt-in diagnostics, not configuration rollback. +BEGIN; +CREATE TABLE mcp_diagnostic_cancellations ( + operation_id uuid PRIMARY KEY REFERENCES mcp_operations(id), + connection_id uuid NOT NULL REFERENCES mcp_connections(id), + actor_id text NOT NULL CHECK (length(actor_id) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE FUNCTION mcp_diagnostic_cancellation_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN RAISE EXCEPTION 'diagnostic cancellation request is immutable'; END $$; +CREATE TRIGGER mcp_diagnostic_cancellation_immutable BEFORE UPDATE OR DELETE ON mcp_diagnostic_cancellations + FOR EACH ROW EXECUTE FUNCTION mcp_diagnostic_cancellation_immutable(); +INSERT INTO schema_migrations(id) VALUES ('0036_diagnostic_cancellations') ON CONFLICT (id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0037_server_probe_storage_v2.sql b/backend/drizzle/0037_server_probe_storage_v2.sql new file mode 100644 index 00000000..9c922f78 --- /dev/null +++ b/backend/drizzle/0037_server_probe_storage_v2.sql @@ -0,0 +1,226 @@ +-- Fresh-only enrollment of empty diagnostic tables. No historical backfill, +-- source-generation claim, public registration or automatic cleanup is granted. +BEGIN; +LOCK TABLE server_probe_executions,server_probe_execution_stages,server_probe_execution_quarantines,mcp_diagnostic_cancellations IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM server_probe_executions) + OR EXISTS(SELECT 1 FROM server_probe_execution_stages) + OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines) + OR EXISTS(SELECT 1 FROM mcp_diagnostic_cancellations) + THEN RAISE EXCEPTION 'nonempty legacy diagnostic history prevents fresh storage enrollment'; END IF; + IF EXISTS(SELECT 1 FROM pg_catalog.pg_class WHERE relname IN + ('server_probe_storage_usage','server_probe_storage_admissions','server_probe_execution_payloads')) + THEN RAISE EXCEPTION 'unmarked diagnostic storage relation prevents fresh storage enrollment'; END IF; + IF NOT EXISTS(SELECT 1 FROM mcp_settings WHERE id=1 AND requests_retention_days BETWEEN 1 AND 365) + THEN RAISE EXCEPTION 'authoritative probe retention setting unavailable'; END IF; +END $$; + +CREATE TABLE server_probe_storage_usage ( + scope text PRIMARY KEY CHECK(scope='global' OR (scope LIKE 'owner:%' AND length(scope) BETWEEN 7 AND 206)), + admissions bigint NOT NULL DEFAULT 0 CHECK(admissions>=0), + reserved_files bigint NOT NULL DEFAULT 0 CHECK(reserved_files>=0 AND reserved_files<=admissions), + reserved_bytes bigint NOT NULL DEFAULT 0 CHECK(reserved_bytes>=0), + CHECK(reserved_bytes=admissions*32768+reserved_files*262144) +); +CREATE TABLE server_probe_storage_admissions ( + receipt_id uuid PRIMARY KEY, + owner_id text NOT NULL CHECK(length(owner_id) BETWEEN 1 AND 200), + identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')), execution_id uuid NOT NULL, + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + authority_expires_at timestamptz NOT NULL, + claim_token uuid, claim_attempted_at timestamptz, + purged_at timestamptz, cancelled_at timestamptz, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + retention_days integer NOT NULL CHECK(retention_days BETWEEN 1 AND 365), + safety_ms integer NOT NULL DEFAULT 60000 CHECK(safety_ms BETWEEN 60000 AND 86400000), + gc_not_before timestamptz NOT NULL DEFAULT 'infinity', + CHECK(gc_not_before>=authority_expires_at+make_interval(days=>retention_days)+safety_ms*interval '1 millisecond'), + UNIQUE(identity_kind,execution_id), + CHECK(claim_token IS NULL OR (claim_attempted_at IS NULL AND purged_at IS NULL AND cancelled_at IS NULL)), + CHECK(cancelled_at IS NULL OR (claim_token IS NULL AND claim_attempted_at IS NOT NULL)) +); +CREATE INDEX server_probe_storage_admissions_owner_age ON server_probe_storage_admissions(owner_id,authority_expires_at,receipt_id); +CREATE TABLE server_probe_execution_payloads ( + receipt_id uuid PRIMARY KEY REFERENCES server_probe_executions(receipt_id), + receipt_ciphertext text NOT NULL CHECK(octet_length(receipt_ciphertext) BETWEEN 1 AND 262144), + receipt_proof text NOT NULL CHECK(receipt_proof ~ '^[a-f0-9]{64}$') +); +INSERT INTO server_probe_storage_usage(scope,admissions,reserved_files,reserved_bytes) VALUES('global',0,0,0); + +-- PostgreSQL drops table CHECKs dependent on this column together with it. +-- The locked empty-table guard precedes this column removal; no data is copied. +ALTER TABLE server_probe_executions DROP COLUMN receipt_ciphertext; +ALTER TABLE server_probe_executions ADD CONSTRAINT server_probe_storage_terminal_state CHECK( + (state='claimed' AND receipt_proof IS NULL AND finished_at IS NULL AND attempted_stages IS NULL AND completed_stages IS NULL) + OR(state<>'claimed' AND receipt_proof IS NOT NULL AND finished_at IS NOT NULL AND attempted_stages IS NOT NULL AND completed_stages IS NOT NULL)); + +-- Predicate only: it performs no DELETE. Every guarded exception below must +-- independently satisfy the persisted lifetime, actual current retention and +-- original terminal operation. The GUC alone never grants cleanup authority. +CREATE FUNCTION server_probe_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; activity timestamptz; current_retention integer; terminal_at timestamptz; +BEGIN + IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.gc_not_before>=clock_timestamp() OR a.authority_expires_at>=clock_timestamp() THEN RETURN false; END IF; + SELECT settings.requests_retention_days INTO current_retention FROM mcp_settings settings WHERE settings.id=1; + IF current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at,COALESCE(cancellation.created_at,o.finished_at)) INTO terminal_at + FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections owner_connection ON owner_connection.id=o.connection_id + LEFT JOIN mcp_diagnostic_cancellations cancellation ON cancellation.operation_id=o.id + WHERE o.id=a.execution_id AND owner_connection.owner_id=a.owner_id AND p.operation_id='servers.probes.run' + AND o.state IN ('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + AND (cancellation.operation_id IS NULL OR(cancellation.connection_id=o.connection_id AND cancellation.actor_id=a.owner_id)) + FOR UPDATE OF o,p,owner_connection; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.created_at,a.authority_expires_at,terminal_at,COALESCE(a.cancelled_at,a.created_at), + COALESCE((SELECT finished_at FROM server_probe_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_probe_execution_quarantines WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(GREATEST(created_at,COALESCE(received_at,created_at))) FROM server_probe_execution_stages WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,current_retention))+a.safety_ms*interval '1 millisecond'clock_timestamp() + THEN RETURN NEW; END IF; + IF OLD.purged_at IS NULL AND NEW.purged_at IS NOT NULL AND NEW.claim_token IS NULL + AND NEW.claim_attempted_at IS NOT DISTINCT FROM OLD.claim_attempted_at THEN + SELECT GREATEST(OLD.authority_expires_at,COALESCE(e.finished_at,OLD.authority_expires_at),COALESCE(q.created_at,OLD.authority_expires_at)) INTO activity + FROM (SELECT 1) x LEFT JOIN server_probe_executions e ON e.receipt_id=OLD.receipt_id + LEFT JOIN server_probe_execution_quarantines q ON q.receipt_id=OLD.receipt_id; + IF activityGREATEST(OLD.retention_days,current_retention)) THEN RETURN NEW; END IF; + END IF; + RAISE EXCEPTION 'probe admission transition refused'; +END $$; +CREATE TRIGGER server_probe_storage_admission_guard BEFORE INSERT OR UPDATE OR DELETE ON server_probe_storage_admissions + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_admission_guard(); + +CREATE FUNCTION server_probe_storage_anchor_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR a.owner_id<>NEW.actor_id OR a.identity_kind<>NEW.identity_kind OR a.execution_id<>NEW.execution_id + OR a.binding_proof<>NEW.binding_proof OR a.authority_expires_at<>NEW.authority_expires_at + OR a.purged_at IS NOT NULL OR a.cancelled_at IS NOT NULL OR a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'probe anchor has no live consumed admission'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_probe_storage_anchor_guard BEFORE INSERT OR UPDATE ON server_probe_executions + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_anchor_guard(); + +CREATE OR REPLACE FUNCTION server_probe_execution_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN + IF server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe execution lifetime has not elapsed'; + END IF; + IF OLD.state<>'claimed' OR (to_jsonb(OLD)-ARRAY['state','attempted_stages','completed_stages','receipt_proof','finished_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['state','attempted_stages','completed_stages','receipt_proof','finished_at']) OR NEW.state='claimed' + THEN RAISE EXCEPTION 'probe execution identity or terminal receipt is immutable'; END IF; + IF EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=OLD.receipt_id) + THEN RAISE EXCEPTION 'probe execution quarantined'; END IF; + RETURN NEW; END $$; + +CREATE FUNCTION server_probe_storage_stage_guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id AND purged_at IS NULL AND cancelled_at IS NULL + AND claim_token IS NULL AND claim_attempted_at IS NOT NULL AND authority_expires_at>clock_timestamp() FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'probe stage has no live admission'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_storage_stage_guard BEFORE INSERT OR UPDATE ON server_probe_execution_stages + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_stage_guard(); + +CREATE FUNCTION server_probe_storage_quarantine_guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id AND identity_kind=NEW.identity_kind + AND execution_id=NEW.execution_id AND binding_proof=NEW.binding_proof FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'probe quarantine has no committed matching admission'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_storage_quarantine_guard BEFORE INSERT ON server_probe_execution_quarantines + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_quarantine_guard(); + +CREATE FUNCTION server_probe_storage_payload_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; e server_probe_executions%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'probe payload is immutable'; END IF; + IF TG_OP='DELETE' THEN + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=OLD.receipt_id AND purged_at IS NOT NULL FOR UPDATE; + IF NOT FOUND AND NOT server_probe_storage_gc_allowed(OLD.receipt_id) THEN RAISE EXCEPTION 'probe payload is not purged'; END IF; + RETURN OLD; + END IF; + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR a.purged_at IS NOT NULL OR a.cancelled_at IS NOT NULL OR a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'probe payload has no live admission'; END IF; + SELECT * INTO e FROM server_probe_executions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR e.state='claimed' OR e.receipt_proof<>NEW.receipt_proof OR e.actor_id<>a.owner_id OR e.binding_proof<>a.binding_proof + OR e.identity_kind<>a.identity_kind OR e.execution_id<>a.execution_id OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines q + WHERE q.receipt_id=NEW.receipt_id OR(q.identity_kind=a.identity_kind AND q.execution_id=a.execution_id)) + THEN RAISE EXCEPTION 'probe payload terminal identity mismatch or quarantine'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_probe_storage_payload_guard BEFORE INSERT OR UPDATE OR DELETE ON server_probe_execution_payloads + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_payload_guard(); +-- Preserve every original INSERT/UPDATE rule; only the already-validated +-- lifetime predicate can admit DELETE; no cleanup is scheduled by migration. +CREATE OR REPLACE FUNCTION server_probe_stage_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN + IF server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe stage lifetime has not elapsed'; + END IF; + PERFORM receipt_id FROM server_probe_executions WHERE receipt_id=NEW.receipt_id AND state='claimed' FOR UPDATE; + IF NOT FOUND OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=NEW.receipt_id) THEN RAISE EXCEPTION 'probe attempt is not executable'; END IF; + IF TG_OP='INSERT' AND (NEW.stage_index<>(SELECT COALESCE(max(stage_index),0)+1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id) + OR EXISTS(SELECT 1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id AND state<>'received')) + THEN RAISE EXCEPTION 'probe stages must be serial and contiguous'; END IF; + IF TG_OP='UPDATE' AND (OLD.state<>'intent' OR NEW.state<>'received' OR (to_jsonb(OLD)-ARRAY['state','outcome','received_at']) IS DISTINCT FROM (to_jsonb(NEW)-ARRAY['state','outcome','received_at'])) + THEN RAISE EXCEPTION 'probe stage terminal state is immutable'; END IF; + RETURN NEW; END $$; +CREATE OR REPLACE FUNCTION server_probe_quarantine_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' AND server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe quarantine remains immutable'; END $$; +CREATE OR REPLACE FUNCTION mcp_diagnostic_cancellation_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE original_receipt uuid; +BEGIN + IF TG_OP='DELETE' THEN + SELECT a.receipt_id INTO original_receipt FROM server_probe_storage_admissions a + JOIN mcp_operations o ON o.id=a.execution_id AND o.connection_id=OLD.connection_id + WHERE a.identity_kind='mcp' AND a.execution_id=OLD.operation_id AND a.owner_id=OLD.actor_id; + IF FOUND AND server_probe_storage_gc_allowed(original_receipt) THEN RETURN OLD; END IF; + END IF; + RAISE EXCEPTION 'diagnostic cancellation lifetime has not elapsed'; +END $$; +INSERT INTO schema_migrations(id) VALUES ('0037_server_probe_storage_v2') ON CONFLICT (id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0038_server_poll_journal_and_writer_guards.sql b/backend/drizzle/0038_server_poll_journal_and_writer_guards.sql new file mode 100644 index 00000000..0f072ccb --- /dev/null +++ b/backend/drizzle/0038_server_poll_journal_and_writer_guards.sql @@ -0,0 +1,59 @@ +-- Initial poll journal and adopted partition-writer backstop. +-- Mixed old replicas MUST be drained before activation: triggers alone run +-- after relation access and cannot replace pre-DML shared writer admission. +BEGIN; +DO $$ BEGIN + PERFORM pg_catalog.pg_advisory_xact_lock(8723110::bigint); + IF EXISTS(SELECT 1 FROM pg_catalog.pg_class WHERE relname=ANY(ARRAY['server_poll_executions','server_poll_intents','server_poll_acks','server_poll_receipts','server_poll_quarantines'])) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_proc WHERE proname=ANY(ARRAY['server_poll_journal_immutable','server_partition_writer_guard'])) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_trigger WHERE tgname='server_partition_writer_guard') + THEN RAISE EXCEPTION 'unmarked poll journal or partition guard requires explicit reconciliation'; END IF; +END $$; +CREATE TABLE server_poll_executions ( + receipt_id uuid PRIMARY KEY,identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')),execution_id uuid NOT NULL, + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'),context_ciphertext text NOT NULL CHECK(octet_length(context_ciphertext)<=2097152), + context_proof text NOT NULL CHECK(context_proof ~ '^[a-f0-9]{64}$'),created_at timestamptz NOT NULL DEFAULT clock_timestamp(),UNIQUE(identity_kind,execution_id)); +CREATE TABLE server_poll_intents ( + receipt_id uuid REFERENCES server_poll_executions(receipt_id),stage text NOT NULL CHECK(stage ~ '^(native\.(identity|resource|interfaces|addresses)|publish\.[0-9]{1,4})$'), + before_proof text NOT NULL CHECK(before_proof ~ '^[a-f0-9]{64}$'),effect_proof text NOT NULL CHECK(effect_proof ~ '^[a-f0-9]{64}$'), + intent_proof text NOT NULL CHECK(intent_proof ~ '^[a-f0-9]{64}$'),created_at timestamptz NOT NULL DEFAULT clock_timestamp(),PRIMARY KEY(receipt_id,stage)); +CREATE TABLE server_poll_acks ( + receipt_id uuid NOT NULL,stage text NOT NULL,result_proof text NOT NULL CHECK(result_proof ~ '^[a-f0-9]{64}$'),outcome text NOT NULL CHECK(outcome IN ('confirmed','read-failed')), + ack_proof text NOT NULL CHECK(ack_proof ~ '^[a-f0-9]{64}$'),created_at timestamptz NOT NULL DEFAULT clock_timestamp(),PRIMARY KEY(receipt_id,stage),FOREIGN KEY(receipt_id,stage) REFERENCES server_poll_intents(receipt_id,stage),CHECK(stage LIKE 'native.%' OR outcome='confirmed')); +CREATE TABLE server_poll_receipts ( + receipt_id uuid PRIMARY KEY REFERENCES server_poll_executions(receipt_id),receipt_ciphertext text NOT NULL CHECK(octet_length(receipt_ciphertext)<=262144),receipt_proof text NOT NULL CHECK(receipt_proof ~ '^[a-f0-9]{64}$'),created_at timestamptz NOT NULL DEFAULT clock_timestamp()); +CREATE TABLE server_poll_quarantines ( + receipt_id uuid PRIMARY KEY,identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')),execution_id uuid NOT NULL,context_proof text NOT NULL CHECK(context_proof ~ '^[a-f0-9]{64}$'),created_at timestamptz NOT NULL DEFAULT clock_timestamp(),UNIQUE(identity_kind,execution_id)); +CREATE FUNCTION server_poll_journal_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'server poll journal is append only'; END $$; +CREATE TRIGGER server_poll_execution_immutable BEFORE UPDATE OR DELETE ON server_poll_executions FOR EACH ROW EXECUTE FUNCTION server_poll_journal_immutable(); +CREATE TRIGGER server_poll_intent_immutable BEFORE UPDATE OR DELETE ON server_poll_intents FOR EACH ROW EXECUTE FUNCTION server_poll_journal_immutable(); +CREATE TRIGGER server_poll_ack_immutable BEFORE UPDATE OR DELETE ON server_poll_acks FOR EACH ROW EXECUTE FUNCTION server_poll_journal_immutable(); +CREATE TRIGGER server_poll_receipt_immutable BEFORE UPDATE OR DELETE ON server_poll_receipts FOR EACH ROW EXECUTE FUNCTION server_poll_journal_immutable(); +CREATE TRIGGER server_poll_quarantine_immutable BEFORE UPDATE OR DELETE ON server_poll_quarantines FOR EACH ROW EXECUTE FUNCTION server_poll_journal_immutable(); +CREATE FUNCTION public.server_partition_writer_guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock_shared(8723110::bigint); IF TG_OP='DELETE' THEN RETURN OLD; END IF; RETURN NEW; END; $$; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_buckets" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_buckets" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_minute_stats" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_minute_stats" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_minute_dims" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_minute_dims" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_daily_dims" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_daily_dims" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_hour_facts" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_hour_facts" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."flow_daily_facts" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."flow_daily_facts" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."traffic_samples" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."traffic_samples" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."servers_rest_ping_samples" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."servers_rest_ping_samples" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."uptime_probe_samples" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."uptime_probe_samples" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."uptime_resource_samples" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."uptime_resource_samples" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."server_snapshots" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."server_snapshots" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +CREATE TRIGGER server_partition_writer_guard BEFORE INSERT OR UPDATE OR DELETE ON public."internet_path_snapshots" FOR EACH ROW EXECUTE FUNCTION public.server_partition_writer_guard(); +ALTER TABLE public."internet_path_snapshots" ENABLE ALWAYS TRIGGER server_partition_writer_guard; +INSERT INTO schema_migrations(id) VALUES ('0038_server_poll_journal_and_writer_guards') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0039_server_poll_storage.sql b/backend/drizzle/0039_server_poll_storage.sql new file mode 100644 index 00000000..1a9e66bd --- /dev/null +++ b/backend/drizzle/0039_server_poll_storage.sql @@ -0,0 +1,175 @@ +-- Fresh-only enrollment of the reviewed bounded poll journal. No historical backfill. +-- Fixed logical quota; physical heap/index/WAL and vacuum remain independent. +-- Requires the empty 0038 poll journal. No scheduling or public activation. +BEGIN; +LOCK TABLE server_poll_executions,server_poll_intents,server_poll_acks,server_poll_receipts,server_poll_quarantines IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM server_poll_executions) OR EXISTS(SELECT 1 FROM server_poll_intents) + OR EXISTS(SELECT 1 FROM server_poll_acks) OR EXISTS(SELECT 1 FROM server_poll_receipts) + OR EXISTS(SELECT 1 FROM server_poll_quarantines) + THEN RAISE EXCEPTION 'nonempty poll journal needs explicit reconciliation'; END IF; + IF NOT EXISTS(SELECT 1 FROM mcp_settings WHERE id=1 AND requests_retention_days BETWEEN 1 AND 365) + THEN RAISE EXCEPTION 'poll retention unavailable'; END IF; + IF EXISTS(SELECT 1 FROM pg_catalog.pg_class WHERE relname IN('server_poll_storage_usage','server_poll_storage_admissions')) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_proc WHERE proname IN('server_poll_storage_gc_allowed','server_poll_storage_admission_guard','server_poll_storage_row_guard','server_poll_storage_row_meter')) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_trigger WHERE tgname LIKE 'server_poll_storage_%') + THEN RAISE EXCEPTION 'unmarked poll storage requires explicit reconciliation'; END IF; +END $$; +CREATE TABLE server_poll_storage_usage ( + scope text PRIMARY KEY CHECK(scope='global' OR(scope LIKE 'owner:%' AND length(scope) BETWEEN 7 AND 206)), + admissions bigint NOT NULL DEFAULT 0 CHECK(admissions>=0 AND admissions<=128), + reserved_rows bigint NOT NULL DEFAULT 0 CHECK(reserved_rows=admissions*4019), + reserved_bytes bigint NOT NULL DEFAULT 0 CHECK(reserved_bytes=admissions*8421376), + CHECK(scope='global' OR admissions<=8) +); +CREATE TABLE server_poll_storage_admissions ( + receipt_id uuid PRIMARY KEY, identity_kind text NOT NULL CHECK(identity_kind IN('mcp','rest')), + execution_id uuid NOT NULL, connection_id uuid, owner_id text NOT NULL CHECK(length(owner_id) BETWEEN 1 AND 200 AND btrim(owner_id)=owner_id AND owner_id !~ '[[:cntrl:]]'), + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), context_proof text NOT NULL CHECK(context_proof ~ '^[a-f0-9]{64}$'), + authority_issued_at timestamptz NOT NULL CHECK(isfinite(authority_issued_at)), + authority_expires_at timestamptz NOT NULL CHECK(isfinite(authority_expires_at) AND authority_expires_at>authority_issued_at), + max_stages integer NOT NULL CHECK(max_stages BETWEEN 8 AND 2008), + claim_token uuid, claim_attempted_at timestamptz, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + retention_days integer NOT NULL CHECK(retention_days BETWEEN 1 AND 365), + safety_ms integer NOT NULL DEFAULT 60000 CHECK(safety_ms=60000), + gc_not_before timestamptz NOT NULL CHECK(isfinite(gc_not_before)), + used_rows integer NOT NULL DEFAULT 0 CHECK(used_rows BETWEEN 0 AND 4019 AND used_rows<=max_stages*2+3), + used_bytes bigint NOT NULL DEFAULT 0 CHECK(used_bytes BETWEEN 0 AND 8388608), + UNIQUE(identity_kind,execution_id), + CHECK((identity_kind='mcp' AND connection_id IS NOT NULL) OR(identity_kind='rest' AND connection_id IS NULL)), + CHECK((claim_token IS NOT NULL AND claim_attempted_at IS NULL) OR(claim_token IS NULL AND claim_attempted_at IS NOT NULL)), + CHECK(gc_not_before>=authority_expires_at+make_interval(days=>retention_days)+interval '60 seconds') +); +CREATE INDEX server_poll_storage_expiry ON server_poll_storage_admissions(gc_not_before,receipt_id); +INSERT INTO server_poll_storage_usage(scope) VALUES('global'); + +-- Every guard is installed in this same empty-enrollment transaction; +-- no historical admission or journal identity is inferred or imported. +CREATE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' AS $$ +DECLARE a server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz; +BEGIN + IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF; + SELECT requests_retention_days INTO retention FROM mcp_settings WHERE mcp_settings.id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM mcp_operations o + JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections c ON c.id=o.connection_id + WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll' + AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + FOR UPDATE OF o,p,c; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at, + COALESCE((SELECT created_at FROM server_poll_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM server_poll_intents WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM server_poll_acks WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_poll_receipts WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond'=1 AND reserved_rows>=4019 AND reserved_bytes>=8421376; + GET DIAGNOSTICS charged=ROW_COUNT; + IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; + RETURN OLD; + END IF; + IF TG_OP='UPDATE' THEN + IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF; + IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL + AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp() + THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF; + IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at + AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself'; + END IF; + SELECT requests_retention_days INTO retention FROM mcp_settings WHERE id=1; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() + OR NEW.authority_expires_at<=clock_timestamp() OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL + OR NEW.used_rows<>0 OR NEW.used_bytes<>0 + THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF; + IF EXISTS(SELECT 1 FROM server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) + THEN RAISE EXCEPTION 'poll identity already spent'; END IF; + NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000; + NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds'; + INSERT INTO server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING; + IF(SELECT count(*) FROM server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id) + AND admissions2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF; + UPDATE server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 + WHERE scope IN('global','owner:'||NEW.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_storage_admission_guard BEFORE INSERT OR UPDATE OR DELETE ON server_poll_storage_admissions + FOR EACH ROW EXECUTE FUNCTION server_poll_storage_admission_guard(); + +CREATE FUNCTION server_poll_storage_row_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_poll_storage_admissions%ROWTYPE; n integer; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'poll journal needs a committed admission'; END IF; + IF TG_TABLE_NAME='server_poll_quarantines' THEN + IF NEW.identity_kind<>a.identity_kind OR NEW.execution_id<>a.execution_id OR NEW.context_proof<>a.context_proof + THEN RAISE EXCEPTION 'poll quarantine admission mismatch'; END IF; + ELSE + IF a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'poll admission is not consumed and live'; END IF; + END IF; + IF TG_TABLE_NAME='server_poll_executions' THEN + IF NEW.identity_kind<>a.identity_kind OR NEW.execution_id<>a.execution_id OR NEW.context_proof<>a.context_proof OR NEW.binding_proof<>a.binding_proof + THEN RAISE EXCEPTION 'poll anchor admission mismatch'; END IF; + ELSIF TG_TABLE_NAME IN('server_poll_intents','server_poll_acks') THEN + IF NEW.stage LIKE 'publish.%' AND substring(NEW.stage FROM 9)::integer>=a.max_stages-4 THEN RAISE EXCEPTION 'poll stage exceeds private reservation'; END IF; + IF TG_TABLE_NAME='server_poll_intents' THEN SELECT count(*) INTO n FROM server_poll_intents WHERE receipt_id=NEW.receipt_id; + ELSE SELECT count(*) INTO n FROM server_poll_acks WHERE receipt_id=NEW.receipt_id; END IF; + IF n>=a.max_stages THEN RAISE EXCEPTION 'poll stage row budget exhausted'; END IF; + END IF; + IF TG_TABLE_NAME<>'server_poll_quarantines' AND EXISTS(SELECT 1 FROM server_poll_quarantines WHERE receipt_id=NEW.receipt_id) + THEN RAISE EXCEPTION 'poll identity quarantined'; END IF; + NEW.created_at:=clock_timestamp(); + RETURN NEW; +END $$; +-- Meter actual inserts, not BEFORE attempts: ON CONFLICT DO NOTHING must not +-- spend another row/byte allowance for a marker which was never inserted. +CREATE FUNCTION server_poll_storage_row_meter() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE cost bigint; n integer; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + cost:=pg_column_size(NEW); + UPDATE server_poll_storage_admissions SET used_rows=used_rows+1,used_bytes=used_bytes+cost + WHERE receipt_id=NEW.receipt_id AND used_rows1 THEN RAISE EXCEPTION 'poll journal byte or row budget exhausted'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_storage_anchor_guard BEFORE INSERT ON server_poll_executions FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_intent_guard BEFORE INSERT ON server_poll_intents FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_ack_guard BEFORE INSERT ON server_poll_acks FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_receipt_guard BEFORE INSERT ON server_poll_receipts FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_quarantine_guard BEFORE INSERT ON server_poll_quarantines FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_anchor_meter AFTER INSERT ON server_poll_executions FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_intent_meter AFTER INSERT ON server_poll_intents FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_ack_meter AFTER INSERT ON server_poll_acks FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_receipt_meter AFTER INSERT ON server_poll_receipts FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_quarantine_meter AFTER INSERT ON server_poll_quarantines FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE OR REPLACE FUNCTION server_poll_journal_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' AND server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'server poll journal is immutable before its full lifetime elapses'; +END $$; +INSERT INTO schema_migrations(id) VALUES ('0039_server_poll_storage') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0040_mcp_routing_reservations.sql b/backend/drizzle/0040_mcp_routing_reservations.sql new file mode 100644 index 00000000..94cffba5 --- /dev/null +++ b/backend/drizzle/0040_mcp_routing_reservations.sql @@ -0,0 +1,56 @@ +-- Additive, inactive reserved recovery ledger. Existing manual evidence is retained. +-- Legacy reserved rows cannot prove an original two-write debit: never backfill them. +BEGIN; +LOCK TABLE public.mcp_scoped_recovery_intents IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE mode='reserved') + THEN RAISE EXCEPTION 'legacy reserved recovery needs explicit reconciliation'; END IF; + IF EXISTS(SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='public' AND c.relname IN('mcp_routing_reservations','mcp_routing_reservation_slots','mcp_routing_reservation_slots_window')) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_attribute a WHERE a.attrelid='public.mcp_scoped_recovery_intents'::regclass + AND NOT a.attisdropped AND a.attname IN('reservation_id','slot_phase','inverse_intent_sealed','inverse_intent_proof','inverse_configuration_sealed','inverse_configuration_proof')) + OR EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c WHERE c.conrelid='public.mcp_scoped_recovery_intents'::regclass + AND c.conname IN('scoped_reservation_slot_shape','reserved_inverse_evidence_pairs')) + THEN RAISE EXCEPTION 'unmarked routing reservation objects need explicit reconciliation'; END IF; +END $$; +CREATE TABLE public.mcp_routing_reservations ( + id uuid PRIMARY KEY, + source_plan_id uuid NOT NULL REFERENCES public.mcp_plans(id), + source_operation_id uuid NOT NULL REFERENCES public.mcp_operations(id), + source_step_index integer NOT NULL CHECK(source_step_index=0), + profile_id uuid NOT NULL REFERENCES public.mcp_routing_profiles(id), + connection_id uuid NOT NULL REFERENCES public.mcp_connections(id), + binding_sealed text NOT NULL,binding_proof text NOT NULL, + state_sealed text NOT NULL,state_proof text NOT NULL,revision integer NOT NULL CHECK(revision>=1), + deadline timestamptz NOT NULL,created_at timestamptz NOT NULL DEFAULT now(), + UNIQUE(source_operation_id,source_step_index) +); +CREATE TABLE public.mcp_routing_reservation_slots ( + reservation_id uuid NOT NULL REFERENCES public.mcp_routing_reservations(id), + profile_id uuid NOT NULL REFERENCES public.mcp_routing_profiles(id), + lane text NOT NULL CHECK(lane IN ('forward','inverse')), + state text NOT NULL CHECK(state IN ('held','spent','proven-unused','unknown')), + created_at timestamptz NOT NULL DEFAULT now(),PRIMARY KEY(reservation_id,lane) +); +CREATE INDEX mcp_routing_reservation_slots_window ON public.mcp_routing_reservation_slots(profile_id,created_at); +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN reservation_id uuid REFERENCES public.mcp_routing_reservations(id); +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN slot_phase text NOT NULL DEFAULT 'recovery-intent'; +ALTER TABLE public.mcp_scoped_recovery_intents ALTER COLUMN source_configuration_proof DROP NOT NULL; +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN inverse_intent_sealed text; +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN inverse_intent_proof text; +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN inverse_configuration_sealed text; +ALTER TABLE public.mcp_scoped_recovery_intents ADD COLUMN inverse_configuration_proof text; +ALTER TABLE public.mcp_scoped_recovery_intents ADD CONSTRAINT scoped_reservation_slot_shape CHECK ( + (mode='manual' AND reservation_id IS NULL AND slot_phase='recovery-intent' AND source_configuration_proof IS NOT NULL) + OR (mode='reserved' AND reservation_id IS NOT NULL AND + ((slot_phase='reservation-held' AND source_configuration_proof IS NULL) + OR (slot_phase IN ('reservation-bound','reserved-attempt') AND source_configuration_proof IS NOT NULL))) +); +ALTER TABLE public.mcp_scoped_recovery_intents ADD CONSTRAINT reserved_inverse_evidence_pairs CHECK ( + (inverse_intent_sealed IS NULL)=(inverse_intent_proof IS NULL) + AND (inverse_configuration_sealed IS NULL)=(inverse_configuration_proof IS NULL) + AND (inverse_configuration_sealed IS NULL OR inverse_intent_sealed IS NOT NULL) + AND (inverse_intent_sealed IS NULL OR mode='reserved') +); +INSERT INTO public.schema_migrations(id) VALUES ('0040_mcp_routing_reservations') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0041_server_poll_terminal_archives.sql b/backend/drizzle/0041_server_poll_terminal_archives.sql new file mode 100644 index 00000000..7eb3d37b --- /dev/null +++ b/backend/drizzle/0041_server_poll_terminal_archives.sql @@ -0,0 +1,333 @@ +-- Additive terminal poll archives. Existing 0040 full journals are retained. +-- No compaction, retention run, source backfill or worker activation occurs here. +BEGIN; +DO $$ BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + IF EXISTS(SELECT 1 FROM pg_class WHERE relname='server_poll_archives') + OR EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid='public.server_poll_storage_admissions'::regclass AND NOT attisdropped AND attname IN('storage_state','compact_at','archive_bytes','original_activity_at','compaction_claim','compaction_claimed_at','compaction_claim_xid')) + OR EXISTS(SELECT 1 FROM pg_attribute WHERE attrelid='public.server_poll_storage_usage'::regclass AND NOT attisdropped AND attname IN('compacted_admissions','archive_bytes')) + OR EXISTS(SELECT 1 FROM pg_proc WHERE proname IN('server_poll_storage_source_payload','server_poll_storage_compact_allowed','server_poll_storage_accounting_valid','server_poll_archive_insert_guard','server_poll_archive_insert_finish','server_poll_archive_immutable','server_poll_archive_final_guard','server_poll_compacted_row_guard')) + OR EXISTS(SELECT 1 FROM pg_trigger WHERE tgname IN('server_poll_archive_insert_guard','server_poll_archive_insert_finish','server_poll_archive_immutable','server_poll_archive_final_guard','server_poll_admission_final_guard','server_poll_compact_final_guard','server_poll_compacted_anchor_guard','server_poll_compacted_intent_guard','server_poll_compacted_ack_guard','server_poll_compacted_receipt_guard','server_poll_compacted_quarantine_guard')) + THEN RAISE EXCEPTION 'unmarked poll archive objects require explicit reconciliation'; END IF; +END $$; +LOCK TABLE server_poll_storage_usage,server_poll_storage_admissions,server_poll_executions,server_poll_intents,server_poll_acks,server_poll_receipts,server_poll_quarantines IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF (SELECT count(*) FROM server_poll_storage_usage WHERE scope='global')<>1 + OR EXISTS(SELECT 1 FROM server_poll_storage_usage u WHERE + u.admissions<>(SELECT count(*) FROM server_poll_storage_admissions a WHERE u.scope='global' OR u.scope='owner:'||a.owner_id) + OR u.reserved_rows<>u.admissions*4019 OR u.reserved_bytes<>u.admissions*8421376) + OR EXISTS(SELECT 1 FROM server_poll_storage_admissions a WHERE NOT EXISTS(SELECT 1 FROM server_poll_storage_usage u WHERE u.scope='owner:'||a.owner_id)) + THEN RAISE EXCEPTION 'existing poll accounting requires explicit reconciliation'; END IF; + IF NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid='public.server_poll_storage_usage'::regclass AND conname='server_poll_storage_usage_check' + AND pg_get_constraintdef(oid)='CHECK ((reserved_rows = (admissions * 4019)))') + OR NOT EXISTS(SELECT 1 FROM pg_constraint WHERE conrelid='public.server_poll_storage_usage'::regclass AND conname='server_poll_storage_usage_check1' + AND pg_get_constraintdef(oid)='CHECK ((reserved_bytes = (admissions * 8421376)))') + THEN RAISE EXCEPTION 'original poll accounting constraints require explicit reconciliation'; END IF; +END $$; +ALTER TABLE server_poll_storage_admissions + ADD COLUMN storage_state text NOT NULL DEFAULT 'full' CHECK(storage_state IN('full','compacted')), + ADD COLUMN compact_at timestamptz, + ADD COLUMN archive_bytes integer, + ADD COLUMN original_activity_at timestamptz, + ADD COLUMN compaction_claim uuid, + ADD COLUMN compaction_claimed_at timestamptz, + ADD COLUMN compaction_claim_xid xid8, + ADD CONSTRAINT server_poll_compaction_claim_shape CHECK( + (compaction_claim IS NULL AND compaction_claimed_at IS NULL AND compaction_claim_xid IS NULL) + OR(compaction_claim IS NOT NULL AND compaction_claimed_at IS NOT NULL AND compaction_claim_xid IS NOT NULL AND isfinite(compaction_claimed_at))), + ADD CONSTRAINT server_poll_storage_compact_shape CHECK( + (storage_state='full' AND compact_at IS NULL AND archive_bytes IS NULL AND original_activity_at IS NULL) + OR(storage_state='compacted' AND identity_kind='mcp' AND claim_token IS NULL AND claim_attempted_at IS NOT NULL + AND compaction_claim IS NOT NULL AND compaction_claimed_at IS NOT NULL AND compaction_claim_xid IS NOT NULL + AND compact_at IS NOT NULL AND archive_bytes IS NOT NULL AND original_activity_at IS NOT NULL + AND isfinite(compact_at) AND archive_bytes BETWEEN 1 AND 131072 AND isfinite(original_activity_at) + AND original_activity_at>=created_at AND compact_at>authority_expires_at+safety_ms*interval '1 millisecond')); +ALTER TABLE server_poll_storage_usage + DROP CONSTRAINT server_poll_storage_usage_check, + DROP CONSTRAINT server_poll_storage_usage_check1, + ADD COLUMN compacted_admissions bigint NOT NULL DEFAULT 0 CHECK(compacted_admissions>=0), + ADD COLUMN archive_bytes bigint NOT NULL DEFAULT 0 CHECK(archive_bytes>=0), + ADD CONSTRAINT server_poll_storage_rows_charge CHECK(reserved_rows=admissions*4019+compacted_admissions*2 + AND reserved_rows<=CASE WHEN scope='global' THEN 514432 ELSE 32152 END), + ADD CONSTRAINT server_poll_storage_bytes_charge CHECK(reserved_bytes=admissions*8421376+compacted_admissions*32768+archive_bytes + AND reserved_bytes<=CASE WHEN scope='global' THEN 1077936128 ELSE 67371008 END), + ADD CONSTRAINT server_poll_storage_archive_charge CHECK(archive_bytes>=compacted_admissions AND archive_bytes<=compacted_admissions*131072); +CREATE TABLE server_poll_archives ( + receipt_id uuid PRIMARY KEY REFERENCES server_poll_storage_admissions(receipt_id),version smallint NOT NULL CHECK(version=1), + execution_id uuid NOT NULL UNIQUE,connection_id uuid NOT NULL,owner_id text NOT NULL, + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'),context_proof text NOT NULL CHECK(context_proof ~ '^[a-f0-9]{64}$'), + source_digest text NOT NULL CHECK(source_digest ~ '^[a-f0-9]{64}$'),source_sha256 text NOT NULL CHECK(source_sha256 ~ '^[a-f0-9]{64}$'), + terminal_digest text NOT NULL CHECK(terminal_digest ~ '^[a-f0-9]{64}$'), + original_activity_at timestamptz NOT NULL CHECK(isfinite(original_activity_at)),original_expires_at timestamptz NOT NULL CHECK(isfinite(original_expires_at)), + decoded_bytes integer NOT NULL CHECK(decoded_bytes BETWEEN 1 AND 16777216),compressed_bytes integer NOT NULL CHECK(compressed_bytes BETWEEN 1 AND 98048), + ciphertext text NOT NULL,ciphertext_bytes integer NOT NULL CHECK(ciphertext_bytes BETWEEN 1 AND 131072 AND ciphertext_bytes=octet_length(ciphertext)), + archive_proof text NOT NULL CHECK(archive_proof ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL,compaction_xid xid8 NOT NULL +); + +-- SQL checks structure/ownership/current rows. The fixed private producer and +-- historical reader additionally authenticate every existing keyed proof and +-- the archive MAC; SQL never receives an encryption/HMAC key or boolean grant. +CREATE FUNCTION server_poll_storage_compact_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE a public.server_poll_storage_admissions%ROWTYPE; o public.mcp_operations%ROWTYPE; p public.mcp_plans%ROWTYPE; s public.mcp_operation_steps%ROWTYPE; n integer; pubs integer; activity timestamptz; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.storage_state<>'full' OR a.identity_kind<>'mcp' OR a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL + OR a.authority_expires_at+a.safety_ms*interval '1 millisecond'>=clock_timestamp() + OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=id OR(identity_kind=a.identity_kind AND execution_id=a.execution_id)) + THEN RETURN false; END IF; + IF NOT pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'||a.execution_id::text,0)) THEN RETURN false; END IF; + IF NOT pg_try_advisory_xact_lock(hashtextextended('server-poll-execution-v1:mcp:'||a.execution_id::text,0)) THEN RETURN false; END IF; + IF NOT pg_try_advisory_xact_lock(hashtextextended('server-poll-receipt-v1:'||a.receipt_id::text,0)) THEN RETURN false; END IF; + BEGIN + SELECT * INTO o FROM public.mcp_operations WHERE mcp_operations.id=a.execution_id AND connection_id=a.connection_id FOR UPDATE NOWAIT; + IF NOT FOUND OR o.state<>'applied' OR o.verification<>'passed' OR o.error_code IS NOT NULL OR o.finished_at IS NULL THEN RETURN false; END IF; + SELECT * INTO p FROM public.mcp_plans WHERE mcp_plans.id=o.plan_id AND connection_id=a.connection_id FOR UPDATE NOWAIT; + IF NOT FOUND OR p.operation_id<>'servers.poll' OR p.operation_version<>1 OR p.state<>'completed' OR p.approved_by IS NULL OR p.approved_at IS NULL + OR p.created_at>p.approved_at OR p.approved_at>o.created_at OR o.created_at>=LEAST(p.expires_at,a.authority_expires_at) THEN RETURN false; END IF; + SELECT * INTO s FROM public.mcp_operation_steps WHERE operation_id=o.id AND step_index=0 FOR UPDATE NOWAIT; + IF NOT FOUND OR s.state<>'verified' OR s.server_id IS NOT NULL OR s.result IS NULL OR s.finished_at IS NULL + OR(SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)<>1 + OR NOT EXISTS(SELECT 1 FROM public.mcp_connections c WHERE c.id=a.connection_id AND c.owner_id=a.owner_id) + OR s.intended_state->>'kind' IS DISTINCT FROM 'service' OR s.intended_state->>'service' IS DISTINCT FROM 'servers.poll' + OR s.intended_state#>>'{input,receiptId}' IS DISTINCT FROM id::text + OR s.result->>'operationId' IS DISTINCT FROM o.id::text OR s.result->>'receiptId' IS DISTINCT FROM id::text + OR s.result->'nativeReads' IS DISTINCT FROM '4'::jsonb + OR s.result->'complete' IS DISTINCT FROM 'true'::jsonb OR s.result->'maintenanceComplete' IS DISTINCT FROM 'true'::jsonb + OR s.result->'configurationVerified' IS DISTINCT FROM 'true'::jsonb OR s.result->'automaticReplay' IS DISTINCT FROM 'false'::jsonb + THEN RETURN false; END IF; + EXCEPTION WHEN lock_not_available THEN RETURN false; END; + IF NOT EXISTS(SELECT 1 FROM public.server_poll_executions e WHERE e.receipt_id=id AND e.identity_kind=a.identity_kind AND e.execution_id=a.execution_id AND e.binding_proof=a.binding_proof AND e.context_proof=a.context_proof) + OR NOT EXISTS(SELECT 1 FROM public.server_poll_receipts r WHERE r.receipt_id=id AND r.created_atpubs+4 OR pubs NOT IN(a.max_stages-6,a.max_stages-4) OR a.used_rows<>2*n+2 + OR(SELECT count(*) FROM public.server_poll_acks WHERE receipt_id=id)<>n + OR s.result->'confirmedPublications' IS DISTINCT FROM to_jsonb(pubs) + OR s.result->'completedPublications' IS DISTINCT FROM to_jsonb(pubs) + OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=id AND outcome<>'confirmed') + OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=id AND created_at>=a.authority_expires_at) + OR EXISTS(SELECT 1 FROM unnest(ARRAY['native.identity','native.resource','native.interfaces','native.addresses']) wanted(stage) + WHERE NOT EXISTS(SELECT 1 FROM public.server_poll_intents i JOIN public.server_poll_acks k USING(receipt_id,stage) WHERE i.receipt_id=id AND i.stage=wanted.stage AND k.outcome='confirmed')) + OR EXISTS(SELECT 1 FROM generate_series(0,pubs-1) wanted(stage) WHERE NOT EXISTS(SELECT 1 FROM public.server_poll_intents i JOIN public.server_poll_acks k USING(receipt_id,stage) WHERE i.receipt_id=id AND i.stage='publish.'||wanted.stage::text AND k.outcome='confirmed')) + THEN RETURN false; END IF; + SELECT GREATEST(a.authority_expires_at,a.created_at,a.claim_attempted_at,o.finished_at,p.expires_at,s.finished_at, + (SELECT created_at FROM public.server_poll_executions WHERE receipt_id=id),(SELECT max(created_at) FROM public.server_poll_intents WHERE receipt_id=id), + (SELECT max(created_at) FROM public.server_poll_acks WHERE receipt_id=id),(SELECT created_at FROM public.server_poll_receipts WHERE receipt_id=id)) INTO activity; + RETURN isfinite(activity) AND activity+a.safety_ms*interval '1 millisecond'16777216 THEN RETURN NULL; END IF; + RETURN payload; +END $$; +CREATE FUNCTION server_poll_storage_accounting_valid() RETURNS boolean LANGUAGE sql SET search_path=pg_catalog,public AS $$ + WITH charges AS ( + SELECT owner_id,CASE WHEN storage_state='full' THEN 1 ELSE 0 END::bigint AS heavy, + CASE WHEN storage_state='compacted' THEN 1 ELSE 0 END::bigint AS compact, + COALESCE(archive_bytes,0)::bigint AS archives, + CASE WHEN storage_state='full' THEN 4019 ELSE 2 END::bigint AS rows, + CASE WHEN storage_state='full' THEN 8421376 ELSE 32768+archive_bytes END::bigint AS bytes + FROM public.server_poll_storage_admissions + ), expected AS ( + SELECT 'global' AS scope,COALESCE(sum(heavy),0) AS heavy,COALESCE(sum(compact),0) AS compact,COALESCE(sum(archives),0) AS archives,COALESCE(sum(rows),0) AS rows,COALESCE(sum(bytes),0) AS bytes FROM charges + UNION ALL SELECT 'owner:'||owner_id,sum(heavy),sum(compact),sum(archives),sum(rows),sum(bytes) FROM charges GROUP BY owner_id + ) SELECT NOT EXISTS(SELECT 1 FROM public.server_poll_storage_usage u FULL JOIN expected e USING(scope) + WHERE u.scope IS NULL OR u.admissions<>COALESCE(e.heavy,0) OR u.compacted_admissions<>COALESCE(e.compact,0) + OR u.archive_bytes<>COALESCE(e.archives,0) OR u.reserved_rows<>COALESCE(e.rows,0) OR u.reserved_bytes<>COALESCE(e.bytes,0)) + AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions a LEFT JOIN public.server_poll_archives r USING(receipt_id) + WHERE (a.storage_state='full' AND r.receipt_id IS NOT NULL) OR(a.storage_state='compacted' AND(r.receipt_id IS NULL OR a.archive_bytes<>r.ciphertext_bytes))) +$$; +CREATE FUNCTION server_poll_archive_insert_guard() RETURNS trigger LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE a public.server_poll_storage_admissions%ROWTYPE; payload jsonb; activity timestamptz; +BEGIN + payload:=public.server_poll_storage_source_payload(NEW.receipt_id); + IF payload IS NULL OR NOT public.server_poll_storage_accounting_valid() THEN RAISE EXCEPTION 'poll source is not eligible for terminal archive'; END IF; + SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + SELECT GREATEST(a.authority_expires_at,a.created_at,a.claim_attempted_at,(payload#>>'{terminal,operation,finished_at}')::timestamptz, + (payload#>>'{terminal,plan,expires_at}')::timestamptz,(payload#>>'{terminal,step,finished_at}')::timestamptz, + (payload#>>'{journal,execution,created_at}')::timestamptz,(payload#>>'{journal,receipt,created_at}')::timestamptz, + (SELECT max((v->>'created_at')::timestamptz) FROM jsonb_array_elements(payload#>'{journal,intents}') v), + (SELECT max((v->>'created_at')::timestamptz) FROM jsonb_array_elements(payload#>'{journal,acks}') v)) INTO activity; + IF a.compaction_claim IS NULL OR a.compaction_claimed_at IS NULL OR NOT isfinite(a.compaction_claimed_at) + OR a.compaction_claim_xid IS NULL OR a.compaction_claim_xid=pg_current_xact_id() + OR NEW.version<>1 OR NEW.execution_id<>a.execution_id OR NEW.connection_id<>a.connection_id OR NEW.owner_id<>a.owner_id + OR NEW.binding_proof<>a.binding_proof OR NEW.context_proof<>a.context_proof OR NEW.original_expires_at<>a.authority_expires_at + OR NEW.original_activity_at<>activity OR NEW.decoded_bytes<>octet_length(payload::text) + OR NEW.source_sha256<>encode(sha256(convert_to(payload::text,'UTF8')),'hex') + OR NEW.ciphertext_bytes<>octet_length(NEW.ciphertext) OR NEW.ciphertext_bytes>131072 + OR NEW.ciphertext !~ '^[A-Za-z0-9+/]+={0,2}$' + OR replace(encode(decode(NEW.ciphertext,'base64'),'base64'),E'\n','')<>NEW.ciphertext + OR octet_length(decode(NEW.ciphertext,'base64'))<>NEW.compressed_bytes+28 + THEN RAISE EXCEPTION 'poll archive does not match the entire immutable source'; END IF; + NEW.created_at:=clock_timestamp(); NEW.compaction_xid:=pg_current_xact_id(); + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_archive_insert_guard BEFORE INSERT ON server_poll_archives FOR EACH ROW EXECUTE FUNCTION server_poll_archive_insert_guard(); + +CREATE OR REPLACE FUNCTION server_poll_journal_immutable() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ BEGIN + IF TG_OP='DELETE' AND EXISTS(SELECT 1 FROM public.server_poll_archives r JOIN public.server_poll_storage_admissions a USING(receipt_id) + WHERE r.receipt_id=OLD.receipt_id AND r.compaction_xid=pg_current_xact_id() AND a.storage_state='full') THEN RETURN OLD; END IF; + IF TG_OP='DELETE' AND public.server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'server poll journal is immutable outside complete archive or original lifetime GC'; +END $$; +CREATE OR REPLACE FUNCTION server_poll_storage_admission_guard() RETURNS trigger LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE retention integer; charged integer; r public.server_poll_archives%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + IF TG_OP='DELETE' THEN + IF NOT public.server_poll_storage_gc_allowed(OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id) THEN RAISE EXCEPTION 'poll lifetime or children prevent admission deletion'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions-CASE WHEN OLD.storage_state='full' THEN 1 ELSE 0 END, + compacted_admissions=compacted_admissions-CASE WHEN OLD.storage_state='compacted' THEN 1 ELSE 0 END, + archive_bytes=archive_bytes-COALESCE(OLD.archive_bytes,0), + reserved_rows=reserved_rows-CASE WHEN OLD.storage_state='full' THEN 4019 ELSE 2 END, + reserved_bytes=reserved_bytes-CASE WHEN OLD.storage_state='full' THEN 8421376 ELSE 32768+OLD.archive_bytes END + WHERE scope IN('global','owner:'||OLD.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; RETURN OLD; + END IF; + IF TG_OP='UPDATE' THEN + -- This durable one-shot marker is written and ACKed in its OWN transaction + -- before archive construction. A committed/unknown claim never resets and + -- the worker never retries destructive work against an already claimed full + -- source. Claim time is maintenance metadata, not renewed execution activity. + IF OLD.storage_state='full' AND NEW.storage_state='full' AND OLD.compaction_claim IS NULL AND OLD.compaction_claimed_at IS NULL AND OLD.compaction_claim_xid IS NULL + AND NEW.compaction_claim IS NOT NULL AND NEW.compaction_claim_xid IS NULL + AND(to_jsonb(OLD)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid'])=(to_jsonb(NEW)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid']) + AND public.server_poll_storage_compact_allowed(OLD.receipt_id) + THEN NEW.compaction_claimed_at:=clock_timestamp(); NEW.compaction_claim_xid:=pg_current_xact_id(); RETURN NEW; END IF; + IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF; + IF OLD.storage_state<>'full' THEN RAISE EXCEPTION 'compacted poll identity cannot reopen or change'; END IF; + IF NEW.storage_state='compacted' THEN + SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND compaction_xid=pg_current_xact_id(); + IF NOT FOUND OR NEW.used_rows<>OLD.used_rows OR NEW.used_bytes<>OLD.used_bytes OR NEW.claim_token IS DISTINCT FROM OLD.claim_token + OR NEW.claim_attempted_at IS DISTINCT FROM OLD.claim_attempted_at OR NEW.archive_bytes<>r.ciphertext_bytes OR NEW.original_activity_at<>r.original_activity_at + OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + THEN RAISE EXCEPTION 'poll compaction requires the complete current-transaction archive'; END IF; + NEW.compact_at:=r.created_at; + UPDATE public.server_poll_storage_usage SET admissions=admissions-1,compacted_admissions=compacted_admissions+1, + archive_bytes=archive_bytes+r.ciphertext_bytes,reserved_rows=reserved_rows-4019+2,reserved_bytes=reserved_bytes-8421376+32768+r.ciphertext_bytes + WHERE scope IN('global','owner:'||OLD.owner_id) AND admissions>=1; + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll compaction charge mismatch'; END IF; + RETURN NEW; + END IF; + IF NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL THEN RAISE EXCEPTION 'full poll cannot carry archive metadata'; END IF; + IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL + AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp() + THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF; + IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at + AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself'; + END IF; + IF NEW.storage_state<>'full' OR NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL + OR NEW.compaction_claim IS NOT NULL OR NEW.compaction_claimed_at IS NOT NULL OR NEW.compaction_claim_xid IS NOT NULL + OR NOT public.server_poll_storage_accounting_valid() THEN RAISE EXCEPTION 'new poll requires full reservation and exact ledger'; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() OR NEW.authority_expires_at<=clock_timestamp() + OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL OR NEW.used_rows<>0 OR NEW.used_bytes<>0 THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF; + IF EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) THEN RAISE EXCEPTION 'poll identity already spent'; END IF; + NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000; + NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds'; + INSERT INTO public.server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING; + IF(SELECT count(*) FROM public.server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id) + AND admissions2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope IN('global','owner:'||NEW.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF; + RETURN NEW; +END $$; +CREATE FUNCTION server_poll_archive_insert_finish() RETURNS trigger LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ BEGIN + DELETE FROM public.server_poll_acks WHERE receipt_id=NEW.receipt_id; + DELETE FROM public.server_poll_intents WHERE receipt_id=NEW.receipt_id; + DELETE FROM public.server_poll_receipts WHERE receipt_id=NEW.receipt_id; + DELETE FROM public.server_poll_executions WHERE receipt_id=NEW.receipt_id; + UPDATE public.server_poll_storage_admissions SET storage_state='compacted',compact_at=NEW.created_at,archive_bytes=NEW.ciphertext_bytes,original_activity_at=NEW.original_activity_at WHERE receipt_id=NEW.receipt_id AND storage_state='full'; + IF NOT FOUND THEN RAISE EXCEPTION 'poll admission did not transition with archive'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_archive_insert_finish AFTER INSERT ON server_poll_archives FOR EACH ROW EXECUTE FUNCTION server_poll_archive_insert_finish(); + +CREATE OR REPLACE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE a public.server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz; +BEGIN + IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE mcp_settings.id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + IF NOT pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'||a.execution_id::text,0)) THEN RETURN false; END IF; + BEGIN + SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll' + AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL FOR UPDATE OF o,p NOWAIT; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + EXCEPTION WHEN lock_not_available THEN RETURN false; END; + END IF; + IF a.storage_state='compacted' THEN activity:=GREATEST(a.original_activity_at,a.authority_expires_at,terminal_at); + ELSE + SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at, + COALESCE((SELECT created_at FROM public.server_poll_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_intents WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_acks WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_receipts WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity; + END IF; + RETURN isfinite(activity) AND activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond' 'public' OR TG_TABLE_NAME NOT IN + ('traffic_settings', 'uptime_settings', 'internet_path_settings') + OR TG_LEVEL <> 'ROW' OR TG_WHEN <> 'BEFORE' THEN + RAISE EXCEPTION 'unexpected poll settings epoch trigger target'; + END IF; + IF TG_OP = 'INSERT' THEN + -- Never accept an epoch supplied by an importer or application caller. + NEW.poll_configuration_epoch := pg_catalog.gen_random_uuid(); + ELSIF TG_OP = 'UPDATE' THEN + -- Every unknown/future field is configuration by default. Only these four + -- current collector fields are excluded; created_at stays significant. + IF (pg_catalog.to_jsonb(NEW) - ARRAY['poll_configuration_epoch', + 'last_collected_at', 'last_duration_ms', 'last_error', 'updated_at']) + IS DISTINCT FROM + (pg_catalog.to_jsonb(OLD) - ARRAY['poll_configuration_epoch', + 'last_collected_at', 'last_duration_ms', 'last_error', 'updated_at']) THEN + NEW.poll_configuration_epoch := pg_catalog.gen_random_uuid(); + ELSE + NEW.poll_configuration_epoch := OLD.poll_configuration_epoch; + END IF; + ELSE + RAISE EXCEPTION 'unexpected poll settings epoch trigger operation'; + END IF; + RETURN NEW; +END; +$epoch$; +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.traffic_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.uptime_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.internet_path_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +ALTER TABLE public.traffic_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; +ALTER TABLE public.uptime_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; +ALTER TABLE public.internet_path_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; +INSERT INTO schema_migrations(id) VALUES('0042_server_poll_settings_epoch') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0043_mcp_routing_forward_charges.sql b/backend/drizzle/0043_mcp_routing_forward_charges.sql new file mode 100644 index 00000000..a2f68777 --- /dev/null +++ b/backend/drizzle/0043_mcp_routing_forward_charges.sql @@ -0,0 +1,62 @@ +-- Durable persistent-forward charge; no execution authority is issued. +-- Reuses the accepted B2.a immutable-parent charging rule, narrowed to ONE +-- forward charge. Existing 0040 two-slot reservations/0031 remain untouched. +-- This SQL cannot authenticate a binding or grant execution authority. +BEGIN; +DO $$ BEGIN + IF to_regclass('public.mcp_routing_forward_charges') IS NOT NULL + THEN RAISE EXCEPTION 'unmarked forward charge storage requires explicit reconciliation'; END IF; +END $$; +CREATE TABLE public.mcp_routing_forward_charges ( + -- Logical historical identities, intentionally NOT lifetime FKs. INSERT is + -- checked against the actual current source below; later source/profile + -- retention cannot erase this charge or be blocked forever by old charges. + source_operation_id uuid PRIMARY KEY, + source_plan_id uuid NOT NULL, + source_step_index integer NOT NULL CHECK(source_step_index=0), + profile_id uuid NOT NULL, + connection_id uuid NOT NULL, + charge_units integer NOT NULL DEFAULT 1 CHECK(charge_units=1), + binding_sealed text NOT NULL CHECK(octet_length(binding_sealed) BETWEEN 1 AND 262144), + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL +); +CREATE INDEX mcp_routing_forward_charges_window + ON public.mcp_routing_forward_charges(profile_id,created_at); +CREATE INDEX mcp_routing_forward_charges_retention + ON public.mcp_routing_forward_charges(created_at,source_operation_id); +-- The immutable parent IS the charged unit. No mutable status/child can drop it +-- from the hourly projection after reject/unknown, human save or restart. +CREATE FUNCTION public.mcp_routing_forward_charge_source() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_routing_profiles r ON r.id=NEW.profile_id AND r.connection_id=o.connection_id + WHERE o.id=NEW.source_operation_id AND p.id=NEW.source_plan_id + AND o.connection_id=NEW.connection_id AND o.state='pending' AND p.state='executing' + AND p.operation_id='routing.community.switch' AND p.operation_version=7 + ) THEN RAISE EXCEPTION 'forward charge differs from actual pending admission source'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mcp_routing_forward_charge_source BEFORE INSERT ON public.mcp_routing_forward_charges + FOR EACH ROW EXECUTE FUNCTION public.mcp_routing_forward_charge_source(); +CREATE FUNCTION public.mcp_routing_forward_charge_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE retention integer; +BEGIN + IF TG_OP<>'DELETE' THEN RAISE EXCEPTION 'immutable forward charge; rewrite/refund forbidden'; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 + OR OLD.created_at>clock_timestamp()-interval '1 hour' + OR OLD.created_at>=clock_timestamp()-make_interval(days=>retention) + OR EXISTS(SELECT 1 FROM public.mcp_operations WHERE id=OLD.source_operation_id AND state IN('pending','running')) + THEN RAISE EXCEPTION 'forward charge is still retained or unresolved'; END IF; + RETURN OLD; +END $$; +CREATE TRIGGER mcp_routing_forward_charge_immutable BEFORE UPDATE OR DELETE ON public.mcp_routing_forward_charges + FOR EACH ROW EXECUTE FUNCTION public.mcp_routing_forward_charge_immutable(); +CREATE TRIGGER mcp_routing_forward_charge_no_truncate BEFORE TRUNCATE ON public.mcp_routing_forward_charges + FOR EACH STATEMENT EXECUTE FUNCTION public.mcp_routing_forward_charge_immutable(); +-- DDL and exact file marker commit atomically. No SECURITY DEFINER or grants. +INSERT INTO public.schema_migrations (id) VALUES ('0043_mcp_routing_forward_charges') ON CONFLICT (id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0044_mcp_routing_manual_inverse_charges.sql b/backend/drizzle/0044_mcp_routing_manual_inverse_charges.sql new file mode 100644 index 00000000..d175494f --- /dev/null +++ b/backend/drizzle/0044_mcp_routing_manual_inverse_charges.sql @@ -0,0 +1,93 @@ +-- Additive durable scoped manual inverse-one charge. No execution grant. +-- Existing 0043 is immutable. Logical charge identities outlive family FKs. +BEGIN; +DO $$ BEGIN + IF to_regclass('public.mcp_routing_manual_inverse_charges') IS NOT NULL + THEN RAISE EXCEPTION 'unmarked manual inverse storage requires reconciliation'; END IF; +END $$; +CREATE TABLE public.mcp_routing_manual_inverse_charges ( + recovery_operation_id uuid PRIMARY KEY, + recovery_plan_id uuid NOT NULL, + source_operation_id uuid NOT NULL, + source_plan_id uuid NOT NULL, + source_step_index integer NOT NULL CHECK(source_step_index=0), + profile_id uuid NOT NULL, + profile_version integer NOT NULL CHECK(profile_version>0), + connection_id uuid NOT NULL, + owner_id text NOT NULL CHECK(octet_length(owner_id) BETWEEN 1 AND 1000), + charge_units integer NOT NULL CHECK(charge_units=1), + binding_sealed text NOT NULL CHECK(octet_length(binding_sealed) BETWEEN 1 AND 262144), + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + CONSTRAINT mcp_manual_inverse_source_step_unique UNIQUE(source_operation_id,source_step_index) +); +CREATE INDEX mcp_routing_manual_inverse_charges_window + ON public.mcp_routing_manual_inverse_charges(profile_id,created_at); +CREATE INDEX mcp_routing_manual_inverse_charges_retention + ON public.mcp_routing_manual_inverse_charges(created_at,recovery_operation_id); +CREATE INDEX mcp_scoped_recovery_manual_history + ON public.mcp_scoped_recovery_intents(mode,source_operation_id,source_step_index) + WHERE mode='manual'; +CREATE FUNCTION public.mcp_routing_manual_inverse_charge_source() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_routing_profiles r ON r.id=NEW.profile_id AND r.connection_id=o.connection_id + JOIN public.mcp_operations source ON source.id=NEW.source_operation_id AND source.connection_id=o.connection_id + JOIN public.mcp_plans sp ON sp.id=source.plan_id AND sp.connection_id=source.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=NEW.recovery_operation_id AND p.id=NEW.recovery_plan_id + AND o.connection_id=NEW.connection_id AND c.owner_id=NEW.owner_id + AND r.version=NEW.profile_version AND o.state='pending' AND p.state='executing' + AND p.operation_id='operations.recover' AND p.operation_version=7 + AND sp.id=NEW.source_plan_id AND sp.operation_id='routing.community.switch' AND sp.operation_version=7 + AND source.state IN('applied','partial') + ) THEN RAISE EXCEPTION 'manual inverse charge differs from pending admission'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mcp_routing_manual_inverse_charge_source BEFORE INSERT ON public.mcp_routing_manual_inverse_charges + FOR EACH ROW EXECUTE FUNCTION public.mcp_routing_manual_inverse_charge_source(); +CREATE FUNCTION public.mcp_routing_manual_inverse_charge_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE retention integer; +BEGIN + IF TG_OP<>'DELETE' THEN RAISE EXCEPTION 'immutable manual inverse charge; rewrite/refund forbidden'; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 + OR OLD.created_at>clock_timestamp()-interval '1 hour' + OR OLD.created_at>=clock_timestamp()-make_interval(days=>retention) + OR EXISTS(SELECT 1 FROM public.mcp_operations WHERE id=OLD.recovery_operation_id AND state IN('pending','running')) + THEN RAISE EXCEPTION 'manual inverse charge still retained or unresolved'; END IF; + RETURN OLD; +END $$; +CREATE TRIGGER mcp_routing_manual_inverse_charge_immutable BEFORE UPDATE OR DELETE ON public.mcp_routing_manual_inverse_charges + FOR EACH ROW EXECUTE FUNCTION public.mcp_routing_manual_inverse_charge_immutable(); +CREATE TRIGGER mcp_routing_manual_inverse_charge_no_truncate BEFORE TRUNCATE ON public.mcp_routing_manual_inverse_charges + FOR EACH STATEMENT EXECUTE FUNCTION public.mcp_routing_manual_inverse_charge_immutable(); +-- This is a version/identity fence, never authentication of encrypted source. +-- Old binaries cannot create a fresh v6 manual intent after this DDL commits. +-- Already-claimed old work requires the normal drained-writer cutover. +CREATE FUNCTION public.mcp_scoped_recovery_manual_version() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.mode='manual' AND NOT pg_try_advisory_xact_lock(hashtextextended('mcp-runner-admission',0)) + THEN RAISE EXCEPTION 'manual intent admission snapshot busy'; END IF; + IF NEW.mode='manual' AND NOT EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + JOIN public.mcp_operations source ON source.id=NEW.source_operation_id AND source.connection_id=o.connection_id + WHERE o.id=NEW.recovery_operation_id AND o.state='running' + AND p.state='executing' AND p.operation_id='operations.recover' AND p.operation_version=7 + AND NEW.source_step_index=0 AND s.state='intent' + AND s.intended_state->>'kind'='service' + AND s.intended_state->>'service'='routing.community.scoped.recover' + AND s.intended_state->'input'->>'mode'='manual' + AND s.intended_state->'input'->>'sourceOperationId'=NEW.source_operation_id::text + AND s.intended_state->'input'->>'receiptId'=NEW.recovery_receipt_id::text + ) THEN RAISE EXCEPTION 'manual inverse requires current exact running recovery'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mcp_scoped_recovery_manual_version BEFORE INSERT ON public.mcp_scoped_recovery_intents + FOR EACH ROW EXECUTE FUNCTION public.mcp_scoped_recovery_manual_version(); +INSERT INTO public.schema_migrations(id) VALUES('0044_mcp_routing_manual_inverse_charges') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0045_mcp_operation_replay_tombstones.sql b/backend/drizzle/0045_mcp_operation_replay_tombstones.sql new file mode 100644 index 00000000..08b3d253 --- /dev/null +++ b/backend/drizzle/0045_mcp_operation_replay_tombstones.sql @@ -0,0 +1,13 @@ +BEGIN; +CREATE TABLE public.mcp_operation_replay_tombstones ( + operation_id uuid PRIMARY KEY, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL REFERENCES public.mcp_connections(id), + idempotency_digest text NOT NULL CHECK (idempotency_digest ~ '^[0-9a-f]{64}$'), + action_id text NOT NULL CHECK (length(action_id) BETWEEN 1 AND 200), + target_server_ids bigint[] NOT NULL CHECK (array_position(target_server_ids, NULL) IS NULL), + target_global boolean NOT NULL, + UNIQUE (connection_id,idempotency_digest) +); +INSERT INTO public.schema_migrations(id) VALUES ('0045_mcp_operation_replay_tombstones') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0046_server_poll_operation_retirement.sql b/backend/drizzle/0046_server_poll_operation_retirement.sql new file mode 100644 index 00000000..7cc40b5d --- /dev/null +++ b/backend/drizzle/0046_server_poll_operation_retirement.sql @@ -0,0 +1,186 @@ +-- Inactive poll-history V1-to-V2 retirement. Application owner authenticates +-- both ciphertext versions; SQL independently fences identity and lifetime. +BEGIN; +LOCK TABLE public.server_poll_archives,public.server_poll_storage_admissions IN ACCESS EXCLUSIVE MODE; +ALTER TABLE public.server_poll_archives DROP CONSTRAINT server_poll_archives_version_check; +ALTER TABLE public.server_poll_archives ADD CONSTRAINT server_poll_archives_version_check + CHECK(version=1 OR (version=2 AND decoded_bytes<=24576 AND compressed_bytes<=24576 AND ciphertext_bytes<=33000)); + +CREATE OR REPLACE FUNCTION public.server_poll_archive_immutable() RETURNS trigger +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='DELETE' AND public.server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + IF TG_OP='UPDATE' AND OLD.version=1 AND NEW.version=2 + AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text + AND NEW.ciphertext_bytes<=OLD.ciphertext_bytes + AND NEW.ciphertext ~ '^[A-Za-z0-9+/]+={0,2}$' + AND replace(encode(decode(NEW.ciphertext,'base64'),'base64'),E'\n','')=NEW.ciphertext + AND octet_length(decode(NEW.ciphertext,'base64'))=NEW.compressed_bytes+28 + AND (to_jsonb(OLD)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof'])= + (to_jsonb(NEW)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof']) + AND EXISTS( + SELECT 1 FROM public.server_poll_storage_admissions a JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0 + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=o.id AND t.plan_id=p.id AND t.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id + WHERE a.receipt_id=OLD.receipt_id AND a.identity_kind='mcp' AND a.storage_state='compacted' + AND a.execution_id=OLD.execution_id AND a.connection_id=OLD.connection_id AND a.owner_id=OLD.owner_id + AND a.binding_proof=OLD.binding_proof AND a.context_proof=OLD.context_proof AND a.archive_bytes=OLD.ciphertext_bytes + AND a.original_activity_at=OLD.original_activity_at AND a.authority_expires_at=OLD.original_expires_at + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at) + AND o.finished_at0 + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll archive immutable outside exact retirement or original lifetime GC'; +END $$; + +CREATE FUNCTION public.server_poll_retirement_final_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones t + WHERE t.operation_id=NEW.execution_id AND t.connection_id=NEW.connection_id AND t.action_id='servers.poll' + AND t.target_global AND NOT EXISTS(SELECT 1 FROM public.mcp_plans p WHERE p.id=t.plan_id)) + OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=NEW.execution_id) + OR NOT public.server_poll_storage_accounting_valid() + THEN RAISE EXCEPTION 'poll retirement is incomplete'; END IF; + RETURN NULL; +END $$; +CREATE CONSTRAINT TRIGGER server_poll_retirement_final_guard AFTER UPDATE ON public.server_poll_archives + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2) + EXECUTE FUNCTION public.server_poll_retirement_final_guard(); + +CREATE OR REPLACE FUNCTION server_poll_storage_admission_guard() RETURNS trigger LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE retention integer; charged integer; r public.server_poll_archives%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + IF TG_OP='DELETE' THEN + IF NOT public.server_poll_storage_gc_allowed(OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id) THEN RAISE EXCEPTION 'poll lifetime or children prevent admission deletion'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions-CASE WHEN OLD.storage_state='full' THEN 1 ELSE 0 END, + compacted_admissions=compacted_admissions-CASE WHEN OLD.storage_state='compacted' THEN 1 ELSE 0 END, + archive_bytes=archive_bytes-COALESCE(OLD.archive_bytes,0), + reserved_rows=reserved_rows-CASE WHEN OLD.storage_state='full' THEN 4019 ELSE 2 END, + reserved_bytes=reserved_bytes-CASE WHEN OLD.storage_state='full' THEN 8421376 ELSE 32768+OLD.archive_bytes END + WHERE scope IN('global','owner:'||OLD.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; RETURN OLD; + END IF; + IF TG_OP='UPDATE' THEN + IF OLD.storage_state='compacted' AND NEW.storage_state='compacted' + AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text + AND (to_jsonb(OLD)-'archive_bytes')=(to_jsonb(NEW)-'archive_bytes') THEN + SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND version=2; + IF NOT FOUND OR r.ciphertext_bytes<>NEW.archive_bytes OR NEW.archive_bytes>OLD.archive_bytes + OR r.execution_id<>OLD.execution_id OR r.connection_id<>OLD.connection_id OR r.owner_id<>OLD.owner_id + THEN RAISE EXCEPTION 'retired poll charge source mismatch'; END IF; + UPDATE public.server_poll_storage_usage + SET archive_bytes=archive_bytes-OLD.archive_bytes+NEW.archive_bytes, + reserved_bytes=reserved_bytes-OLD.archive_bytes+NEW.archive_bytes + WHERE scope IN ('global','owner:'||OLD.owner_id) AND compacted_admissions>=1 AND archive_bytes>=OLD.archive_bytes; + GET DIAGNOSTICS charged=ROW_COUNT; + IF charged<>2 THEN RAISE EXCEPTION 'retired poll charge adjustment mismatch'; END IF; + RETURN NEW; + END IF; + -- This durable one-shot marker is written and ACKed in its OWN transaction + -- before archive construction. A committed/unknown claim never resets and + -- the worker never retries destructive work against an already claimed full + -- source. Claim time is maintenance metadata, not renewed execution activity. + IF OLD.storage_state='full' AND NEW.storage_state='full' AND OLD.compaction_claim IS NULL AND OLD.compaction_claimed_at IS NULL AND OLD.compaction_claim_xid IS NULL + AND NEW.compaction_claim IS NOT NULL AND NEW.compaction_claim_xid IS NULL + AND(to_jsonb(OLD)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid'])=(to_jsonb(NEW)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid']) + AND public.server_poll_storage_compact_allowed(OLD.receipt_id) + THEN NEW.compaction_claimed_at:=clock_timestamp(); NEW.compaction_claim_xid:=pg_current_xact_id(); RETURN NEW; END IF; + IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF; + IF OLD.storage_state<>'full' THEN RAISE EXCEPTION 'compacted poll identity cannot reopen or change'; END IF; + IF NEW.storage_state='compacted' THEN + SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND compaction_xid=pg_current_xact_id(); + IF NOT FOUND OR NEW.used_rows<>OLD.used_rows OR NEW.used_bytes<>OLD.used_bytes OR NEW.claim_token IS DISTINCT FROM OLD.claim_token + OR NEW.claim_attempted_at IS DISTINCT FROM OLD.claim_attempted_at OR NEW.archive_bytes<>r.ciphertext_bytes OR NEW.original_activity_at<>r.original_activity_at + OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id) + OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id) + THEN RAISE EXCEPTION 'poll compaction requires the complete current-transaction archive'; END IF; + NEW.compact_at:=r.created_at; + UPDATE public.server_poll_storage_usage SET admissions=admissions-1,compacted_admissions=compacted_admissions+1, + archive_bytes=archive_bytes+r.ciphertext_bytes,reserved_rows=reserved_rows-4019+2,reserved_bytes=reserved_bytes-8421376+32768+r.ciphertext_bytes + WHERE scope IN('global','owner:'||OLD.owner_id) AND admissions>=1; + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll compaction charge mismatch'; END IF; + RETURN NEW; + END IF; + IF NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL THEN RAISE EXCEPTION 'full poll cannot carry archive metadata'; END IF; + IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL + AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp() + THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF; + IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at + AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself'; + END IF; + IF NEW.storage_state<>'full' OR NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL + OR NEW.compaction_claim IS NOT NULL OR NEW.compaction_claimed_at IS NOT NULL OR NEW.compaction_claim_xid IS NOT NULL + OR NOT public.server_poll_storage_accounting_valid() THEN RAISE EXCEPTION 'new poll requires full reservation and exact ledger'; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() OR NEW.authority_expires_at<=clock_timestamp() + OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL OR NEW.used_rows<>0 OR NEW.used_bytes<>0 THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF; + IF EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) THEN RAISE EXCEPTION 'poll identity already spent'; END IF; + NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000; + NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds'; + INSERT INTO public.server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING; + IF(SELECT count(*) FROM public.server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id) + AND admissions2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF; + UPDATE public.server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope IN('global','owner:'||NEW.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF; + RETURN NEW; +END $$; +CREATE OR REPLACE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE a public.server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz; +BEGIN + IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF; + SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE mcp_settings.id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + IF NOT pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'||a.execution_id::text,0)) THEN RETURN false; END IF; + BEGIN + SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll' + AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL FOR UPDATE OF o,p NOWAIT; + IF NOT FOUND THEN + IF a.storage_state<>'compacted' OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=a.execution_id) OR NOT EXISTS ( + SELECT 1 FROM public.mcp_operation_replay_tombstones t JOIN public.mcp_connections c ON c.id=t.connection_id + WHERE t.operation_id=a.execution_id AND t.connection_id=a.connection_id AND c.owner_id=a.owner_id + AND t.action_id='servers.poll' AND t.target_global AND cardinality(t.target_server_ids)>0 + AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE mcp_plans.id=t.plan_id) + ) OR EXISTS(SELECT 1 FROM public.server_poll_archives r WHERE r.receipt_id=id AND + (r.version<>2 OR r.execution_id<>a.execution_id OR r.connection_id<>a.connection_id OR r.owner_id<>a.owner_id + OR r.original_activity_at IS DISTINCT FROM a.original_activity_at OR r.original_expires_at<>a.authority_expires_at + OR r.binding_proof<>a.binding_proof OR r.context_proof<>a.context_proof OR r.ciphertext_bytes<>a.archive_bytes)) + THEN RETURN false; END IF; + terminal_at:=a.original_activity_at; + END IF; + IF terminal_at IS NULL THEN RETURN false; END IF; + EXCEPTION WHEN lock_not_available THEN RETURN false; END; + END IF; + IF a.storage_state='compacted' THEN activity:=GREATEST(a.original_activity_at,a.authority_expires_at,terminal_at); + ELSE + SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at, + COALESCE((SELECT created_at FROM public.server_poll_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_intents WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM public.server_poll_acks WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_receipts WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM public.server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity; + END IF; + RETURN isfinite(activity) AND activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond'1 AND NOT EXISTS( + SELECT 1 FROM public.server_poll_storage_admissions a WHERE a.receipt_id=OLD.receipt_id + ) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'poll retirement claim cannot be replaced or reopened'; +END $$; +CREATE TRIGGER server_poll_retirement_claim_immutable + BEFORE INSERT OR UPDATE OR DELETE ON public.server_poll_retirement_claims + FOR EACH ROW EXECUTE FUNCTION public.server_poll_retirement_claim_immutable(); +CREATE TRIGGER server_poll_retirement_claim_no_truncate + BEFORE TRUNCATE ON public.server_poll_retirement_claims + FOR EACH STATEMENT EXECUTE FUNCTION public.server_poll_retirement_claim_immutable(); +CREATE FUNCTION public.server_poll_retirement_requires_claim() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c + WHERE c.receipt_id=OLD.receipt_id AND c.old_archive_proof=OLD.archive_proof + AND c.claim_token::text=current_setting('mm.poll_retire_claim',true) + AND c.claim_xid<>pg_current_xact_id()) + THEN RAISE EXCEPTION 'poll retirement requires prior exact claim'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_retirement_requires_claim BEFORE UPDATE ON public.server_poll_archives + FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2) + EXECUTE FUNCTION public.server_poll_retirement_requires_claim(); +INSERT INTO public.schema_migrations(id) VALUES ('0047_server_poll_retirement_claims') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql b/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql new file mode 100644 index 00000000..c0107ac5 --- /dev/null +++ b/backend/drizzle/0048_mcp_ordinary_retirement_claims.sql @@ -0,0 +1,27 @@ +BEGIN; +CREATE TABLE public.mcp_ordinary_retirement_claims ( + operation_id uuid PRIMARY KEY, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL REFERENCES public.mcp_connections(id), + source_proof text NOT NULL CHECK(source_proof ~ '^[0-9a-f]{64}$'), + claim_token uuid NOT NULL, + claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id() +); +CREATE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at>'version') IS NOT DISTINCT FROM '1' + AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text + AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text) +); +CREATE FUNCTION public.server_probe_retired_source_immutable() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + RAISE EXCEPTION 'retired diagnostic source is immutable'; +END $$; +CREATE TRIGGER server_probe_retired_source_immutable + BEFORE UPDATE OR DELETE ON public.server_probe_retired_sources + FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_immutable(); +CREATE TRIGGER server_probe_retired_source_no_truncate BEFORE TRUNCATE ON public.server_probe_retired_sources + FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_retired_source_immutable(); +INSERT INTO public.schema_migrations(id) VALUES ('0050_server_probe_retired_sources') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0051_server_probe_retirement_claims.sql b/backend/drizzle/0051_server_probe_retirement_claims.sql new file mode 100644 index 00000000..15e52072 --- /dev/null +++ b/backend/drizzle/0051_server_probe_retirement_claims.sql @@ -0,0 +1,52 @@ +BEGIN; +CREATE TABLE public.server_probe_retirement_claims ( + receipt_id uuid PRIMARY KEY, + operation_id uuid NOT NULL UNIQUE, + plan_id uuid NOT NULL UNIQUE, + connection_id uuid NOT NULL, + source_proof text NOT NULL CHECK(source_proof ~ '^[a-f0-9]{64}$'), + claim_token uuid NOT NULL, + claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id() +); +CREATE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.server_probe_storage_admissions a + JOIN public.mcp_operations o ON o.id=a.execution_id + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id + WHERE a.receipt_id=NEW.receipt_id AND a.identity_kind='mcp' AND a.claim_token IS NULL + AND a.claim_attempted_at IS NOT NULL AND a.cancelled_at IS NULL AND a.purged_at IS NULL + AND o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_atpg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true) + AND NEW.source->>'planId'=r.plan_id::text AND NEW.source->>'connectionId'=r.connection_id::text + AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof + AND NEW.source->'serverIds'=to_jsonb(t.target_server_ids) + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'diagnostic archive requires an exact committed retirement claim'; +END $$; +CREATE TRIGGER server_probe_retired_source_insert_guard BEFORE INSERT ON public.server_probe_retired_sources + FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_insert_guard(); +INSERT INTO public.schema_migrations(id) VALUES ('0051_server_probe_retirement_claims') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0052_server_probe_retired_gc.sql b/backend/drizzle/0052_server_probe_retired_gc.sql new file mode 100644 index 00000000..18fc51ea --- /dev/null +++ b/backend/drizzle/0052_server_probe_retired_gc.sql @@ -0,0 +1,96 @@ +BEGIN; +ALTER TABLE public.server_probe_retired_sources + DROP CONSTRAINT server_probe_retired_sources_receipt_id_fkey, + ADD CONSTRAINT server_probe_retired_sources_receipt_id_fkey + FOREIGN KEY(receipt_id) REFERENCES public.server_probe_storage_admissions(receipt_id) ON DELETE CASCADE; + +CREATE FUNCTION public.server_probe_retired_gc_allowed(target_receipt uuid,current_retention integer) RETURNS boolean +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +DECLARE activity timestamptz; retained_days integer; safety integer; +BEGIN + IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on' + OR current_setting('mm.probe_retired_gc_receipt',true) IS DISTINCT FROM target_receipt::text THEN RETURN false; END IF; + SELECT (r.source->>'activityAt')::timestamptz,a.retention_days,a.safety_ms + INTO activity,retained_days,safety + FROM public.server_probe_retired_sources r + JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id + JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id + WHERE r.receipt_id=target_receipt AND r.proof=current_setting('mm.probe_retired_gc_proof',true) + AND a.identity_kind='mcp' AND a.cancelled_at IS NULL AND a.claim_token IS NULL AND isfinite(a.claim_attempted_at) + AND t.action_id='servers.probes.run' AND NOT t.target_global + AND r.source->>'planId'=t.plan_id::text AND r.source->>'connectionId'=c.id::text + AND r.source->'serverIds'=to_jsonb(t.target_server_ids) AND r.source->>'ownerId'=a.owner_id + AND r.source->>'bindingProof'=a.binding_proof + AND (r.source->>'authorityExpiresAt')::timestamptz=a.authority_expires_at + AND (r.source->>'gcNotBefore')::timestamptz=a.gc_not_before + AND (r.source->>'retentionDays')::int=a.retention_days AND (r.source->>'safetyMs')::int=a.safety_ms + AND isfinite(a.authority_expires_at) AND isfinite(a.gc_not_before) + AND a.authority_expires_atactivity)) + OR EXISTS(SELECT 1 FROM public.server_probe_execution_stages s WHERE s.receipt_id=target_receipt + AND GREATEST(s.created_at,COALESCE(s.received_at,s.created_at))>activity) + THEN RETURN false; END IF; + RETURN activity+make_interval(days=>GREATEST(retained_days,current_retention)) + +GREATEST(safety,60000)*interval '1 millisecond'=clock_timestamp() OR a.authority_expires_at>=clock_timestamp() THEN RETURN false; END IF; + SELECT settings.requests_retention_days INTO current_retention FROM mcp_settings settings WHERE settings.id=1; + IF current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + IF EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=id) THEN + RETURN public.server_probe_retired_gc_allowed(id,current_retention); + END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at,COALESCE(cancellation.created_at,o.finished_at)) INTO terminal_at + FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections owner_connection ON owner_connection.id=o.connection_id + LEFT JOIN mcp_diagnostic_cancellations cancellation ON cancellation.operation_id=o.id + WHERE o.id=a.execution_id AND owner_connection.owner_id=a.owner_id AND p.operation_id='servers.probes.run' + AND o.state IN ('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + AND (cancellation.operation_id IS NULL OR(cancellation.connection_id=o.connection_id AND cancellation.actor_id=a.owner_id)) + FOR UPDATE OF o,p,owner_connection; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.created_at,a.authority_expires_at,terminal_at,COALESCE(a.cancelled_at,a.created_at), + COALESCE((SELECT finished_at FROM server_probe_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_probe_execution_quarantines WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(GREATEST(created_at,COALESCE(received_at,created_at))) FROM server_probe_execution_stages WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,current_retention))+a.safety_ms*interval '1 millisecond'>'version') IS NOT DISTINCT FROM '1' + AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text + AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text) +); +CREATE FUNCTION public.server_probe_terminal_proof_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='INSERT' THEN + IF EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + JOIN public.server_probe_storage_admissions a ON a.execution_id=o.id AND a.identity_kind='mcp' AND a.owner_id=c.owner_id + JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id + JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id AND b.receipt_proof=j.receipt_proof + WHERE o.id=NEW.operation_id AND a.receipt_id=NEW.receipt_id AND a.purged_at IS NULL AND a.cancelled_at IS NULL + AND a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND p.state='completed' AND p.operation_id='servers.probes.run' AND p.operation_version=1 + AND j.state='completed' AND j.execution_id=o.id AND j.mcp_connection_id=c.id AND j.actor_id=c.owner_id + AND NEW.source->>'connectionId'=c.id::text AND NEW.source->>'planId'=p.id::text + AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof + AND NEW.source->>'receiptProof'=b.receipt_proof + AND NEW.source->>'ciphertextSha256'=encode(sha256(convert_to(b.receipt_ciphertext,'UTF8')),'hex') + ) THEN RETURN NEW; END IF; + ELSIF TG_OP='DELETE' THEN + IF EXISTS( + SELECT 1 FROM public.server_probe_retirement_claims r + JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.plan_id=r.plan_id AND t.connection_id=r.connection_id + WHERE r.operation_id=OLD.operation_id AND r.receipt_id=OLD.receipt_id AND r.source_proof=OLD.source_proof + AND r.claim_xid<>pg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true) + AND t.action_id='servers.probes.run' AND NOT t.target_global AND t.idempotency_digest=OLD.idempotency_digest + AND OLD.source->'serverIds'=to_jsonb(t.target_server_ids) + ) THEN RETURN OLD; END IF; + END IF; + RAISE EXCEPTION 'diagnostic terminal proof must match its original source or exact retirement claim'; +END $$; +CREATE TRIGGER server_probe_terminal_proof_guard BEFORE INSERT OR UPDATE OR DELETE + ON public.server_probe_terminal_proofs FOR EACH ROW EXECUTE FUNCTION public.server_probe_terminal_proof_guard(); +CREATE TRIGGER server_probe_terminal_proof_no_truncate BEFORE TRUNCATE + ON public.server_probe_terminal_proofs FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_terminal_proof_guard(); + +CREATE OR REPLACE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.mcp_operations o + JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN public.mcp_connections c ON c.id=o.connection_id + WHERE o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id + AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at>'ownerId'=c.owner_id AND f.source->>'connectionId'=c.id::text AND f.source->>'planId'=p.id::text) + AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions a + WHERE a.receipt_id=NEW.receipt_id OR(a.identity_kind='mcp' AND a.execution_id=o.id)))) + ) THEN RETURN NEW; END IF; + RAISE EXCEPTION 'diagnostic retirement claim cannot be replaced or reopened'; +END $$; +INSERT INTO public.schema_migrations(id) VALUES ('0053_server_probe_terminal_proofs') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0054_mcp_geoip_retirement.sql b/backend/drizzle/0054_mcp_geoip_retirement.sql new file mode 100644 index 00000000..6b59b470 --- /dev/null +++ b/backend/drizzle/0054_mcp_geoip_retirement.sql @@ -0,0 +1,18 @@ +BEGIN; +CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger +LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$ +BEGIN + IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS( + SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id + AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL + AND isfinite(o.finished_at) AND o.finished_at>'kind'='service' + AND s.intended_state->>'service'='routing.community.scoped.recover' + AND s.intended_state->'input'->>'mode'='manual' + AND s.intended_state->'input'->>'sourceOperationId'=NEW.source_operation_id::text + AND s.intended_state->'input'->>'receiptId'=NEW.recovery_receipt_id::text + ) THEN RAISE EXCEPTION 'manual inverse requires current exact running recovery'; END IF; + RETURN NEW; +END $$; +INSERT INTO public.schema_migrations(id) VALUES('0056_mcp_manual_inverse_recovery_v8') ON CONFLICT(id) DO NOTHING; +COMMIT; diff --git a/backend/drizzle/0057_mcp_routing_charge_audit_index.sql b/backend/drizzle/0057_mcp_routing_charge_audit_index.sql new file mode 100644 index 00000000..c5e17796 --- /dev/null +++ b/backend/drizzle/0057_mcp_routing_charge_audit_index.sql @@ -0,0 +1,8 @@ +BEGIN; + +CREATE INDEX mcp_audit_routing_charge_profile_window + ON public.mcp_audit (connection_id, (payload -> 'anchor' ->> 'profileId'), created_at, id) + WHERE action = 'routing.charge.admitted'; + +INSERT INTO public.schema_migrations (id) VALUES ('0057_mcp_routing_charge_audit_index') ON CONFLICT (id) DO NOTHING; +COMMIT; diff --git a/backend/src/db/application-operator-transactions.ts b/backend/src/db/application-operator-transactions.ts new file mode 100644 index 00000000..7d07f9d3 --- /dev/null +++ b/backend/src/db/application-operator-transactions.ts @@ -0,0 +1,194 @@ +import {randomBytes} from 'node:crypto' +import pg,{type PoolClient,type QueryResult,type QueryResultRow} from 'pg' + +export interface ApplicationOperatorSql { + query(text:string,values?:unknown[]):Promise> +} +export type PairDecision={disposition:'commit'|'rollback';value:T} +export type PairResult={disposition:'committed'|'rolled-back';value:T} +type Outcome='not-started'|'rolled-back'|'committed'|'unknown' +export class ApplicationOperatorTransactionError extends Error { + readonly automaticReplay=false + constructor(readonly outcome:Outcome,readonly phase:string,readonly committedValue:T|undefined,readonly hasCommittedValue:boolean,readonly cleanupUnconfirmed:boolean,options?:ErrorOptions){ + super('Application/operator transaction pairing refused; do not replay automatically',options) + } +} +const CONTEXT=`SELECT CURRENT_USER::text AS actor,SESSION_USER::text AS login, + d.oid::text AS database_oid,pg_catalog.pg_backend_pid() AS pid, + pg_catalog.current_setting('server_version_num')::int AS version, + pg_catalog.current_setting('transaction_read_only') AS read_only, + pg_catalog.pg_current_xact_id_if_assigned()::text AS xid, + (r.rolsuper OR r.rolcreatedb OR r.rolcreaterole OR r.rolreplication OR r.rolbypassrls) AS unsafe, + EXISTS(SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname='public' AND n.nspowner=r.oid) AS owns_public, + EXISTS(SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relowner=r.oid) AS owns_application_object, + pg_catalog.pg_has_role(r.oid,($1::pg_catalog.regrole)::pg_catalog.oid,'MEMBER') AS member_operator + FROM pg_catalog.pg_roles r CROSS JOIN pg_catalog.pg_database d WHERE r.rolname=CURRENT_USER AND d.datname=pg_catalog.current_database()` +const WITNESS=`SELECT pid,database::text AS database_oid,mode,granted FROM pg_catalog.pg_locks + WHERE locktype='advisory' AND database=(SELECT oid FROM pg_catalog.pg_database WHERE datname=pg_catalog.current_database()) + AND classid=$1::oid AND objid=$2::oid AND objsubid=2 ORDER BY pid,mode LIMIT 3` +type Context={actor:string;login:string;database_oid:string;pid:number;version:number;read_only:'on'|'off';xid:string|null;unsafe:boolean;owns_public:boolean;owns_application_object:boolean;member_operator:boolean} +function exactContext(rows:QueryResultRow[],actor:string,readOnly:boolean):Context { + const row=rows[0] as Context|undefined + if(rows.length!==1||!row||row.actor!==actor||row.login!==actor||!Number.isSafeInteger(row.pid)||row.pid<=0||!/^\d+$/.test(row.database_oid)||row.version<180000||row.version>=190000||row.read_only!==(readOnly?'on':'off')||row.unsafe!==false||typeof row.member_operator!=='boolean'||typeof row.owns_public!=='boolean'||typeof row.owns_application_object!=='boolean'||row.xid!==null&&!/^[1-9][0-9]*$/.test(row.xid))throw Error('Actual PG18 expected role/transaction context unavailable') + if(readOnly&&(row.owns_public||row.owns_application_object||row.member_operator))throw Error('Runtime role is an application owner/operator member') + if(!readOnly&&!row.owns_public)throw Error('Operator is not the explicitly configured application schema owner') + return row +} +// Defensive direct-boundary refusal, NOT an untrusted SQL parser/sandbox. +// Only fixed trusted application statements may reach this internal facade. +function fixedApplicationStatement(text:string){ + if(typeof text!=='string'||!text.trim()||Buffer.byteLength(text)>1048576)throw Error('Bounded trusted application SQL required') + const head=text.replace(/^(?:\s|--[^\r\n]*(?:\r?\n|$)|\/\*[^]*?\*\/)+/,'') + if(/^(?:BEGIN|START\s+TRANSACTION|COMMIT|END|ROLLBACK|SAVEPOINT|RELEASE|PREPARE\s+TRANSACTION|SET\s+(?:(?:LOCAL|SESSION)\s+)?ROLE|RESET\s+ROLE)\b/i.test(head))throw Error('Transaction and role ownership belongs to pairing component') +} +/** Private trusted host assembly only. No source/profile issuer, SQL sandbox, + * configured URL, Pool creation, role provisioning, fallback or READY grant. */ +export function createApplicationOperatorTransactions(config:{runtimePool:pg.Pool;operatorPool:pg.Pool;runtimeRole:string;operatorRole:string;maxConcurrentPairs:1|2}){ + const {runtimePool,operatorPool,runtimeRole,operatorRole,maxConcurrentPairs}=config + if(!(runtimePool instanceof pg.Pool)||!(operatorPool instanceof pg.Pool)||runtimePool===operatorPool||runtimeRole===operatorRole||![runtimeRole,operatorRole].every(role=>/^[a-z_][a-z0-9_]{0,62}$/.test(role))||![1,2].includes(maxConcurrentPairs))throw Error('Fixed actual pools, separated roles and bounded capacity required') + const runtimeConnect=runtimePool.connect.bind(runtimePool),operatorConnect=operatorPool.connect.bind(operatorPool) + let activePairs=0,poisoned=false + return Object.freeze({ + async run(work:(tx:ApplicationOperatorSql)=>Promise>,input:{deadlineAt:number;signal?:AbortSignal;authorize?:()=>Promise}):Promise>{ + const options=Object.freeze({deadlineAt:input.deadlineAt,signal:input.signal,authorize:input.authorize}) + if(poisoned||activePairs>=maxConcurrentPairs||typeof work!=='function'||!Number.isSafeInteger(options.deadlineAt)||options.deadlineAt<=Date.now()||options.deadlineAt-Date.now()>120000||options.signal?.aborted)throw new ApplicationOperatorTransactionError('not-started','admission',undefined,false,poisoned) + activePairs++ + let runtime:PoolClient|undefined,operator:PoolClient|undefined,runtimeReleased=false,operatorReleased=false + let runtimeBegun=false,operatorBegun=false,commitSent=false,commitConfirmed=false,operatorTerminal=false,witnessTerminal=false + let phase='checkout-runtime',accepting=false,retired=false,primary:unknown,hasPrimary=false,workFailed=false,queuedFailure:unknown,queuedFailed=false,tail:Promise=Promise.resolve() + const retain=(cause:unknown)=>{if(!hasPrimary){primary=cause;hasPrimary=true}} + let committedValue:T|undefined,hasCommittedValue=false,decision:PairDecision|undefined + const random=randomBytes(8),first=random.readInt32BE(0),second=random.readInt32BE(4),key=[first,second],unsigned=[first>>>0,second>>>0] + const errors=new Mapvoid>() + let stopReject!:(error:Error)=>void + const stopped=new Promise((_,reject)=>{stopReject=reject});void stopped.catch(()=>{}) + const dispose=(client:PoolClient|undefined,unsafe:boolean)=>{ + if(!client)return + if(client===runtime){if(runtimeReleased)return;runtimeReleased=true} + else if(client===operator){if(operatorReleased)return;operatorReleased=true} + const listener=errors.get(client) + if(listener){client.off('error',listener);client.off('end',listener)} + try{client.release(unsafe)}catch(error){poisoned=true;retain(error)} + } + const stop=(cause:unknown)=>{ + if(retired)return + retired=true;accepting=false;poisoned=true;retain(cause) + dispose(operator,true);dispose(runtime,true) + stopReject(new ApplicationOperatorTransactionError(commitConfirmed?'committed':operatorBegun?'unknown':'not-started',phase,committedValue,hasCommittedValue,true,{cause})) + } + const fence=()=>{if(retired||options.signal?.aborted||Date.now()>=options.deadlineAt)throw hasPrimary?primary:Error('Pairing lifetime expired')} + const bounded=(operation:Promise)=>Promise.race([operation,stopped]) + const observe=(client:PoolClient)=>{const listener=()=>stop(Error('Owned physical connection ended unexpectedly'));errors.set(client,listener);client.on('error',listener);client.on('end',listener)} + const borrow=async(connect:()=>Promise,kind:'runtime'|'operator')=>{ + const pending=connect().then(client=>{ + if(kind==='runtime')runtime=client;else operator=client + if(retired){dispose(client,true);return client} + observe(client);return client + }) + // Handler remains attached if cancellation wins; a late checkout is retired. + void pending.catch(()=>{}) + const client=await bounded(pending);fence();return client + } + const query=async(client:PoolClient,text:string,values?:unknown[])=>{fence();const result=await bounded(client.query(text,values));fence();return result} + const authorize=async()=>{fence();await bounded(Promise.resolve().then(()=>options.authorize?.()));fence()} + const onAbort=()=>stop(Error('Pairing cancelled')) + const timer=setTimeout(()=>stop(Error('Pairing deadline exceeded')),Math.max(1,options.deadlineAt-Date.now())) + options.signal?.addEventListener('abort',onAbort,{once:true}) + let operatorContext:Context|undefined,runtimeContext:Context|undefined + const assertPair=async()=>{ + if(!operator||!operatorContext||!runtimeContext)throw Error('Live pair is not established') + try{ + const current=exactContext((await query(operator,CONTEXT,[operatorRole])).rows,operatorRole,false) + if(current.pid!==operatorContext.pid||current.xid!==operatorContext.xid||current.database_oid!==operatorContext.database_oid||current.xid===null)throw Error('Operator backend/owned writable transaction changed') + const locks=(await query(operator,WITNESS,unsigned)).rows + if(locks.length!==1||locks[0].pid!==runtimeContext.pid||locks[0].database_oid!==runtimeContext.database_oid||locks[0].mode!=='ShareLock'||locks[0].granted!==true)throw Error('Original runtime witness is not live on this application database') + }catch(error){stop(error);throw error} + } + try{ + await authorize();runtime=await borrow(runtimeConnect,'runtime') + await query(runtime,'BEGIN ISOLATION LEVEL READ COMMITTED READ ONLY');runtimeBegun=true + await query(runtime,"SET LOCAL statement_timeout='10s'");await query(runtime,"SET LOCAL lock_timeout='5s'") + runtimeContext=exactContext((await query(runtime,CONTEXT,[operatorRole])).rows,runtimeRole,true) + await query(runtime,'SELECT pg_catalog.pg_advisory_xact_lock_shared($1::int,$2::int)',key) + phase='checkout-operator';operator=await borrow(operatorConnect,'operator') + await query(operator,'BEGIN ISOLATION LEVEL READ COMMITTED READ WRITE');operatorBegun=true + await query(operator,"SET LOCAL statement_timeout='120s'");await query(operator,"SET LOCAL lock_timeout='10s'");await query(operator,'SET LOCAL synchronous_commit=on') + await query(operator,'SELECT pg_catalog.pg_current_xact_id()') + operatorContext=exactContext((await query(operator,CONTEXT,[operatorRole])).rows,operatorRole,false) + if(operatorContext.xid===null)throw Error('Owned writable transaction identity unavailable') + phase='live-source-pair' + const attempt=(await query(operator,'SELECT pg_catalog.pg_try_advisory_xact_lock($1::int,$2::int) AS acquired',key)).rows + if(attempt.length!==1||attempt[0].acquired!==false)throw Error('Operator is not paired to original live application database') + await assertPair();await authorize();phase='work';accepting=true + const facade:ApplicationOperatorSql=Object.freeze({query:(text:string,values?:unknown[]):Promise>=>{ + const rejected=(error:unknown):Promise>=>{const failure=Promise.reject(error);void failure.catch(()=>{});return failure} + if(!accepting||retired)return rejected(new ApplicationOperatorTransactionError(commitConfirmed?'committed':'unknown','closed-facade',committedValue,hasCommittedValue,false)) + try{fixedApplicationStatement(text)}catch(error){if(!queuedFailed){queuedFailure=error;queuedFailed=true}return rejected(error)} + const parameters=values?.slice() + const pending=tail.then(async()=>{ + if(queuedFailed)throw queuedFailure + await authorize();await assertPair() + const result=await query(operator!,text,parameters) + await assertPair();await authorize();return result as QueryResult + }) + // Keep EVERY accepted request in a serial drain, including detached work. + tail=pending.then(()=>{},error=>{if(!queuedFailed){queuedFailure=error;queuedFailed=true}}) + void pending.catch(()=>{}) + return pending + }}) + try{decision=await bounded(Promise.resolve().then(()=>work(facade)))}catch(error){workFailed=true;retain(error)} + finally{accepting=false} + phase='drain-work';await bounded(tail) + if(workFailed||queuedFailed)throw workFailed?primary:queuedFailure + if(!decision||!['commit','rollback'].includes(decision.disposition)||!Object.hasOwn(decision,'value'))throw Error('Trusted work must return explicit terminal disposition/value') + decision=Object.freeze({disposition:decision.disposition,value:decision.value}) + await authorize();await assertPair() + phase='operator-terminal' + if(decision.disposition==='commit'){ + commitSent=true + const terminal=await bounded(operator.query('COMMIT').then(result=>{ + // Preserve ACK/value BEFORE a cancellation/deadline fence can throw. + if(result.command==='COMMIT'){commitConfirmed=true;operatorTerminal=true;operatorBegun=false;committedValue=decision!.value;hasCommittedValue=true} + return result + })) + if(terminal.command!=='COMMIT')throw Error('Operator COMMIT acknowledgement unavailable') + fence() + }else{ + const terminal=await query(operator,'ROLLBACK') + if(terminal.command!=='ROLLBACK')throw Error('Operator ROLLBACK acknowledgement unavailable') + operatorTerminal=true;operatorBegun=false + } + phase='witness-terminal' + const terminal=await query(runtime,'ROLLBACK') + if(terminal.command!=='ROLLBACK')throw Error('Runtime witness cleanup acknowledgement unavailable') + witnessTerminal=true;runtimeBegun=false + dispose(operator,false);dispose(runtime,false) + if(hasPrimary)throw primary + return {disposition:decision.disposition==='commit'?'committed':'rolled-back',value:decision.value} + }catch(error){ + accepting=false;retain(error) + if(!retired){ + // Drain accepted SQL BEFORE cleanup; deadline/physical faults destroy both + // instead of sending ROLLBACK concurrently over an in-flight query. + try{await bounded(tail)}catch(cause){stop(cause)} + if(!retired){ + try{if(operator&&operatorBegun&&!commitSent){const terminal=await query(operator,'ROLLBACK');if(terminal.command!=='ROLLBACK')throw Error('Operator rollback unconfirmed');operatorBegun=false;operatorTerminal=true}} + catch(cause){stop(cause)} + try{if(!retired&&runtime&&runtimeBegun){const terminal=await query(runtime,'ROLLBACK');if(terminal.command!=='ROLLBACK')throw Error('Witness rollback unconfirmed');runtimeBegun=false;witnessTerminal=true}} + catch(cause){stop(cause)} + } + } + const cleanupUnconfirmed=retired||runtimeBegun||operatorBegun||(commitSent&&!commitConfirmed) + if(cleanupUnconfirmed)poisoned=true + dispose(operator,cleanupUnconfirmed);dispose(runtime,cleanupUnconfirmed) + const outcome:Outcome=commitConfirmed?'committed':commitSent?'unknown':operatorTerminal?'rolled-back':runtimeBegun||operatorBegun?'unknown':'not-started' + throw new ApplicationOperatorTransactionError(outcome,phase,committedValue,hasCommittedValue,cleanupUnconfirmed,{cause:primary}) + }finally{ + accepting=false;clearTimeout(timer);options.signal?.removeEventListener('abort',onAbort) + if(!runtimeReleased&&runtime)dispose(runtime,!witnessTerminal) + if(!operatorReleased&&operator)dispose(operator,!operatorTerminal) + activePairs-- + } + }, + }) +} diff --git a/backend/src/db/application-writer-profile-canonical.ts b/backend/src/db/application-writer-profile-canonical.ts new file mode 100644 index 00000000..7f9716b0 --- /dev/null +++ b/backend/src/db/application-writer-profile-canonical.ts @@ -0,0 +1,9 @@ +import {createHash} from 'node:crypto' + +/** Exact writer-profile v1 canonical bytes. Pure value codec; no trust/authority. */ +export function canonical(value:unknown):string { + return JSON.stringify(value,(_key,item:unknown)=>item&&typeof item==='object'&&!Array.isArray(item) + ?Object.fromEntries(Object.entries(item).sort(([a],[b])=>ab?1:0)):item) +} +export const sha=(value:Uint8Array|string)=>createHash('sha256').update(value).digest('hex') +export const bytes=(value:unknown)=>Buffer.from(canonical(value)+'\n','utf8') diff --git a/backend/src/db/application-writer-profile-format.test.ts b/backend/src/db/application-writer-profile-format.test.ts new file mode 100644 index 00000000..6d52ac55 --- /dev/null +++ b/backend/src/db/application-writer-profile-format.test.ts @@ -0,0 +1,16 @@ +// Self-contained production-target unit: no historical tmp oracle/key files/DB. +import assert from 'node:assert/strict' +import {generateKeyPairSync,sign} from 'node:crypto' +import {bytes,canonical,sha,epoch,nextEpoch,POLICY_DOMAIN,HEAD_DOMAIN,signedEnvelope,liveValidity,type ApplicationWriterPolicy} from './application-writer-profile-format.js' +const key=generateKeyPairSync('ed25519'),now=Date.now(),installationId='11111111-1111-4111-8111-111111111111' +const subject={installationId,publicKeyPem:key.publicKey.export({format:'pem',type:'spki'}).toString(),publicKeySpkiSha256:sha(key.publicKey.export({format:'der',type:'spki'}))} +const policy:ApplicationWriterPolicy={format:'application-writer-policy-v1',installationId,epoch:'9007199254740993',issuedAt:now-1000,notBefore:now-1000,notAfter:now+60000,source:{systemIdentifier:'18446744073709551615',databaseOid:'4294967295',databaseName:'synthetic',serverVersionNum:180004,inRecovery:false},sourceObservationSha256:'a'.repeat(64),roles:{runtime:'11',operator:'12',capture:'13',publisher:'14',control:'15'},manifestSha256:'b'.repeat(64),definitionSha256:'c'.repeat(64),catalogSha256:'d'.repeat(64),futureLeaves:'fixed12-native-policy-v1'} +function signed(body:unknown,domain=POLICY_DOMAIN){return bytes({body,signature:sign(null,Buffer.from(domain+canonical(body)),key.privateKey).toString('base64url')})} +assert.equal(canonical({z:[2,1],a:{я:'ё',b:1}}),'{"a":{"b":1,"я":"ё"},"z":[2,1]}') +assert.equal(epoch('9007199254740993'),'9007199254740993');assert.equal(nextEpoch('9007199254740993'),'9007199254740994');assert.throws(()=>nextEpoch('9223372036854775807'));assert.throws(()=>epoch(9007199254740993));assert.throws(()=>epoch('01')) +const raw=signed(policy),parsed=signedEnvelope(raw,'policy',subject);assert.deepEqual(parsed.body,policy);assert.equal(parsed.sha256,sha(raw));assert.deepEqual(Object.keys(parsed).sort(),['body','sha256']);liveValidity(parsed.body,now) +assert.throws(()=>signedEnvelope(signed({...policy,unknown:true}),'policy',subject));assert.throws(()=>signedEnvelope(signed({...policy,roles:{...policy.roles,unknown:'16'}}),'policy',subject));assert.throws(()=>signedEnvelope(signed({...policy,source:{...policy.source,inRecovery:true}}),'policy',subject)) +const tampered=JSON.parse(raw.toString());tampered.body.catalogSha256='0'.repeat(64);assert.throws(()=>signedEnvelope(bytes(tampered),'policy',subject));assert.throws(()=>signedEnvelope(raw,'policy',{...subject,publicKeySpkiSha256:'0'.repeat(64)}));assert.throws(()=>signedEnvelope(Buffer.concat([raw,Buffer.from(' ')]),'policy',subject));assert.throws(()=>signedEnvelope(Buffer.from([255]),'policy',subject)) +const head={format:'application-writer-head-v1',installationId,epoch:'1',policySha256:sha(raw),previousHeadSha256:null};assert.deepEqual(signedEnvelope(signed(head,HEAD_DOMAIN),'head',subject).body,head);assert.throws(()=>signedEnvelope(signed(head,POLICY_DOMAIN),'head',subject));assert.throws(()=>signedEnvelope(signed({...head,epoch:'2'},HEAD_DOMAIN),'head',subject)) +const expired={...policy,issuedAt:now-20000,notBefore:now-20000,notAfter:now-10000};assert.deepEqual(signedEnvelope(signed(expired),'policy',subject).body,expired);assert.throws(()=>liveValidity(expired,now));assert.throws(()=>liveValidity(policy,now+6000)) +console.log(JSON.stringify({syntheticOnly:true,unitGroups:7,privateKeysWritten:false,postgres:false,writerAuthorityMinted:false,readyAuthorized:false})) diff --git a/backend/src/db/application-writer-profile-format.ts b/backend/src/db/application-writer-profile-format.ts new file mode 100644 index 00000000..c0c35832 --- /dev/null +++ b/backend/src/db/application-writer-profile-format.ts @@ -0,0 +1,56 @@ +import {createPublicKey,verify,type KeyObject} from 'node:crypto' +import {z} from 'zod' +import {bytes,canonical,sha} from './application-writer-profile-canonical.js' +export {bytes,canonical,sha} + +// Internal schema/crypto only: no key loader, DB/CLI/runtime, installation issuer, +// caller-JSON pin parser, provider, source/current-head proof or writer capability. +// A verified signature under a supplied public key is DATA, never authorization. +export const KEY=8723110,PUBLISHER='barrier_profile_publisher',OPERATOR='barrier_operator' +export const POLICY_DOMAIN='mmapp:application-writer-policy:ed25519:v1\n',HEAD_DOMAIN='mmapp:application-writer-head:ed25519:v1\n' +export const MAX_EPOCH=9223372036854775807n +export function epoch(value:unknown):string { + if(typeof value!=='string'||!(/^[1-9][0-9]{0,18}$/).test(value)||BigInt(value)>MAX_EPOCH)throw Error('Canonical PG bigint epoch required') + return value +} +export function nextEpoch(value:unknown):string { + const next=BigInt(epoch(value))+1n + if(next>MAX_EPOCH)throw Error('PG bigint epoch exhausted') + return next.toString() +} +const hex=z.string().regex(/^[a-f0-9]{64}$/),uuid=z.string().uuid(),decimal=z.string().refine(x=>{try{return epoch(x)===x}catch{return false}}) +const oid=z.string().regex(/^[1-9][0-9]{0,9}$/).refine(value=>BigInt(value)<=4294967295n) +const roles=z.object({runtime:oid,operator:oid,capture:oid,publisher:oid,control:oid}).strict() +const source=z.object({systemIdentifier:z.string().regex(/^[1-9][0-9]{0,19}$/).refine(value=>BigInt(value)<=18446744073709551615n),databaseOid:oid,databaseName:z.string().min(1).max(63),serverVersionNum:z.number().int().min(180000).max(189999),inRecovery:z.literal(false)}).strict() +export const policySchema=z.object({format:z.literal('application-writer-policy-v1'),installationId:uuid,epoch:decimal,issuedAt:z.number().int().nonnegative(),notBefore:z.number().int().nonnegative(),notAfter:z.number().int().nonnegative(),source,sourceObservationSha256:hex,roles,manifestSha256:hex,definitionSha256:hex,catalogSha256:hex,futureLeaves:z.literal('fixed12-native-policy-v1')}).strict().refine(p=>p.notBefore===p.issuedAt&&p.notAfter>p.notBefore&&p.notAfter-p.notBefore<=86400000) +export const headSchema=z.object({format:z.literal('application-writer-head-v1'),installationId:uuid,epoch:decimal,policySha256:hex,previousHeadSha256:hex.nullable()}).strict().refine(p=>(p.epoch==='1')===(p.previousHeadSha256===null)) +const envelope=z.object({body:z.unknown(),signature:z.string().regex(/^[A-Za-z0-9_-]{86}$/)}).strict() +export type ApplicationWriterPolicy=z.infer +export type ApplicationWriterHead=z.infer +type ArtifactKind='policy'|'head' +// Not exported as a provisioning/config API. Future private host must obtain +// this subject from protected installation state, never a request or provider. +interface PublicVerificationSubject {readonly installationId:string;readonly publicKeyPem:string;readonly publicKeySpkiSha256:string} +type ParsedData={body:T;sha256:string} +export function publicKey(installation:PublicVerificationSubject):KeyObject { + const key=createPublicKey(installation.publicKeyPem) + if(key.asymmetricKeyType!=='ed25519'||sha(key.export({format:'der',type:'spki'}))!==installation.publicKeySpkiSha256)throw Error('Fixed installation SPKI required') + return key +} +export function signedEnvelope(raw:Uint8Array|string,kind:'policy',installation:PublicVerificationSubject):ParsedData +export function signedEnvelope(raw:Uint8Array|string,kind:'head',installation:PublicVerificationSubject):ParsedData +export function signedEnvelope(raw:Uint8Array|string,kind:ArtifactKind,installation:PublicVerificationSubject):ParsedData +export function signedEnvelope(raw:Uint8Array|string,kind:ArtifactKind,installation:PublicVerificationSubject):ParsedData { + const data=Buffer.from(raw) + if(data.length<1||data.length>(kind==='policy'?1048576:8192))throw Error('Signed artifact bound') + const value=envelope.parse(JSON.parse(new TextDecoder('utf8',{fatal:true}).decode(data))) + if(!bytes(value).equals(data))throw Error('Canonical signed artifact required') + if(Buffer.from(value.signature,'base64url').toString('base64url')!==value.signature)throw Error('Canonical signature encoding required') + const body=(kind==='policy'?policySchema:headSchema).parse(value.body) + if(body.installationId!==installation.installationId||!verify(null,Buffer.from((kind==='policy'?POLICY_DOMAIN:HEAD_DOMAIN)+canonical(body)),publicKey(installation),Buffer.from(value.signature,'base64url')))throw Error('Installation signature refused') + return {body,sha256:sha(data)} +} +export function liveValidity(policy:ApplicationWriterPolicy,serverNow:number):void { + const now=Date.now() + if(!Number.isSafeInteger(serverNow)||Math.abs(serverNow-now)>5000||now=policy.notAfter||serverNow=policy.notAfter)throw Error('Current signed policy interval/clock refused') +} diff --git a/backend/src/db/application-writer-profile-registry-five-query-oracle.test.ts b/backend/src/db/application-writer-profile-registry-five-query-oracle.test.ts new file mode 100644 index 00000000..a9bc225f --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-five-query-oracle.test.ts @@ -0,0 +1,52 @@ +// Frozen five-query regression oracle; TEST ONLY, excluded from backend build. +import {epoch,signedEnvelope,liveValidity,type ApplicationWriterPolicy} from './application-writer-profile-format.js' + +// Internal read-side value component ONLY. No configured pool, provider/pin JSON +// parser, publish/DDL/CAS write, key loader, role provisioning or authority mint. +// Caller retains authenticated backend/source/lock/transaction/lifetime ownership. +// Structure/index/ACL checks and protected subject provenance are separate gates. +export const HISTORY='application_writer_profile_history',CURRENT='application_writer_profile_current' +export interface ApplicationWriterRegistryReadSql { + query=Record>(text:string,values?:unknown[]):Promise<{rows:R[]}> +} +type Subject=Parameters[2] +interface CurrentRow extends Record {installation_id:string;epoch:string;head_sha256:string} +interface HeadRow extends CurrentRow {policy_sha256:string;previous_installation_id:string|null;previous_epoch:string|null;previous_head_sha256:string|null;head_bytes:Uint8Array|string|null} +interface PolicyRow extends Record {policy_sha256:string;policy_bytes:Uint8Array|string|null} +export interface ApplicationWriterRegistryReadOptions {allowEmpty?:boolean;validNow?:boolean} +/** Ordinary observed data, NOT a CAS reservation/current-state capability. + * Subject must be fixed by a future protected private host, never a request. + * validNow=false is reserved for trusted expired-policy inspection/renewal; + * it is not a runtime authorization option. No trusted state is created here. */ +export interface ApplicationWriterRegistryCurrentData {epoch:string;headSha256:string;policy:ApplicationWriterPolicy;policySha256:string;serverNow:number} +function payload(value:Uint8Array|string|null):Uint8Array|string { + // SQL CASE turns oversize/empty bytes into NULL; reject before parsing. Same + // refusal as MJS Buffer.from(null), with an explicit internal diagnostic. + if(value===null)throw Error('Bounded signed registry payload absent') + return value +} +export async function readCurrentApplicationWriterProfile(client:ApplicationWriterRegistryReadSql,installation:Subject,{allowEmpty=false,validNow=true}:ApplicationWriterRegistryReadOptions={}):Promise{ + const foreign=(await client.query(`(SELECT installation_id FROM mmapp_control.${HISTORY} WHERE installation_id<$1::uuid ORDER BY installation_id DESC LIMIT 1) UNION ALL (SELECT installation_id FROM mmapp_control.${HISTORY} WHERE installation_id>$1::uuid ORDER BY installation_id LIMIT 1)`,[installation.installationId])).rows + if(foreign.length)throw Error('Foreign installation history refused') + const current=(await client.query(`SELECT installation_id::text,epoch::text,head_sha256 FROM mmapp_control.${CURRENT} WHERE id=1`)).rows + const latest=(await client.query(`SELECT installation_id::text,epoch::text,head_sha256,policy_sha256,previous_installation_id::text,previous_epoch::text,previous_head_sha256,CASE WHEN pg_catalog.octet_length(head_bytes) BETWEEN 1 AND 8192 THEN head_bytes ELSE NULL END AS head_bytes FROM mmapp_control.${HISTORY} WHERE installation_id=$1::uuid ORDER BY epoch DESC LIMIT 2`,[installation.installationId])).rows + if(current.length===0&&latest.length===0&&allowEmpty)return null + const c=current[0],top=latest[0] + if(current.length!==1||!top||c.installation_id!==installation.installationId||c.epoch!==epoch(top.epoch)||c.head_sha256!==top.head_sha256)throw Error('Current does not match indexed append-only latest') + const row=(await client.query(`SELECT policy_sha256,CASE WHEN pg_catalog.octet_length(policy_bytes) BETWEEN 1 AND 1048576 THEN policy_bytes ELSE NULL END AS policy_bytes FROM mmapp_control.${HISTORY} WHERE installation_id=$1::uuid AND epoch=$2::bigint AND head_sha256=$3`,[c.installation_id,c.epoch,c.head_sha256])).rows + if(row.length!==1)throw Error('Exact current policy row absent') + const policy=signedEnvelope(payload(row[0].policy_bytes),'policy',installation),head=signedEnvelope(payload(top.head_bytes),'head',installation) + if(head.sha256!==c.head_sha256||head.body.epoch!==c.epoch||policy.body.epoch!==c.epoch||head.body.policySha256!==policy.sha256||row[0].policy_sha256!==policy.sha256||top.policy_sha256!==policy.sha256||head.body.previousHeadSha256!==top.previous_head_sha256)throw Error('Exact current signature/hash/link refused') + if(c.epoch==='1'){ + if(latest.length!==1||top.previous_epoch!==null||top.previous_installation_id!==null||top.previous_head_sha256!==null)throw Error('Unique genesis required') + }else{ + const previous=latest[1] + if(!previous||BigInt(epoch(previous.epoch))+1n!==BigInt(c.epoch)||top.previous_epoch!==previous.epoch||top.previous_installation_id!==installation.installationId||top.previous_head_sha256!==previous.head_sha256)throw Error('Exact consecutive predecessor required') + const before=signedEnvelope(payload(previous.head_bytes),'head',installation) + if(before.sha256!==previous.head_sha256||before.body.epoch!==previous.epoch||before.body.policySha256!==previous.policy_sha256||before.body.previousHeadSha256!==previous.previous_head_sha256)throw Error('Historical signed predecessor refused') + } + const now=(await client.query<{now:string}>("SELECT (EXTRACT(EPOCH FROM pg_catalog.clock_timestamp())*1000)::bigint::text AS now")).rows + if(now.length!==1)throw Error('Actual server clock required') + if(validNow)liveValidity(policy.body,Number(now[0].now)) + return {epoch:c.epoch,headSha256:c.head_sha256,policy:policy.body,policySha256:policy.sha256,serverNow:Number(now[0].now)} +} diff --git a/backend/src/db/application-writer-profile-registry-read-model.test.ts b/backend/src/db/application-writer-profile-registry-read-model.test.ts new file mode 100644 index 00000000..fab8979f --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-read-model.test.ts @@ -0,0 +1,55 @@ +// Self-contained MODEL unit. No genuine SQL/ownership/authority claim. +import {generateKeyPairSync,sign} from 'node:crypto' +import {bytes,canonical,sha,POLICY_DOMAIN,HEAD_DOMAIN,type ApplicationWriterPolicy} from './application-writer-profile-format.js' +import type {ApplicationWriterRegistryReadSql} from './application-writer-profile-registry-read.js' +export function syntheticFixture(value='2'){ + const key=generateKeyPairSync('ed25519'),installationId='11111111-1111-4111-8111-111111111111',now=Date.now() + const subject={installationId,publicKeyPem:key.publicKey.export({format:'pem',type:'spki'}).toString(),publicKeySpkiSha256:sha(key.publicKey.export({format:'der',type:'spki'}))} + const policy:ApplicationWriterPolicy={format:'application-writer-policy-v1',installationId,epoch:value,issuedAt:now-1000,notBefore:now-1000,notAfter:now+60000,source:{systemIdentifier:'123',databaseOid:'456',databaseName:'synthetic',serverVersionNum:180004,inRecovery:false},sourceObservationSha256:'a'.repeat(64),roles:{runtime:'11',operator:'12',capture:'13',publisher:'14',control:'15'},manifestSha256:'b'.repeat(64),definitionSha256:'c'.repeat(64),catalogSha256:'d'.repeat(64),futureLeaves:'fixed12-native-policy-v1'} + const signBody=(body:unknown,domain:string)=>bytes({body,signature:sign(null,Buffer.from(domain+canonical(body)),key.privateKey).toString('base64url')}) + function head(epoch:string,policyHash:string,previous:string|null){const head_bytes=signBody({format:'application-writer-head-v1',installationId,epoch,policySha256:policyHash,previousHeadSha256:previous},HEAD_DOMAIN);return {installation_id:installationId,epoch,head_sha256:sha(head_bytes),policy_sha256:policyHash,previous_installation_id:epoch==='1'?null:installationId,previous_epoch:epoch==='1'?null:(BigInt(epoch)-1n).toString(),previous_head_sha256:previous,head_bytes}} + const policy_bytes=signBody(policy,POLICY_DOMAIN),policyHash=sha(policy_bytes),previous=value==='1'?undefined:head((BigInt(value)-1n).toString(),'e'.repeat(64),value==='2'?null:'f'.repeat(64)),top=head(value,policyHash,previous?.head_sha256??null) + return {subject,policy,signBody,privateKey:key.privateKey,state:{foreign:[] as Record[],current:[{installation_id:installationId,epoch:value,head_sha256:top.head_sha256}],latest:previous?[top,previous]:[top],policy:[{policy_sha256:policyHash,policy_bytes:policy_bytes as Uint8Array|null}],clock:[{now:String(now)}]}} +} +export type Fixture=ReturnType +export function modelClient(state:Fixture['state']){ + const trace:{text:string;values?:unknown[]}[]=[] + const client:ApplicationWriterRegistryReadSql={async query=Record>(text:string,values?:unknown[]){trace.push({text,values:structuredClone(values)});let rows:unknown[] + if(text.startsWith('(SELECT installation_id'))rows=state.foreign + else if(text.includes('WHERE id=1'))rows=state.current + else if(text.includes('ORDER BY epoch DESC LIMIT 2'))rows=state.latest + else if(text.startsWith('SELECT policy_sha256,CASE'))rows=state.policy + else if(text.includes('clock_timestamp()'))rows=state.clock + else throw Error('MODEL unknown SQL; no fake success') + return {rows:structuredClone(rows) as R[]} + }} + return {client,trace} +} + +type Row=Record +type State={foreign:Row[];current:Row[];latest:Row[];policy:Row[];clock:Row[]} +const keys=['installation_id','epoch','head_sha256','policy_sha256','previous_installation_id','previous_epoch','previous_head_sha256','head_bytes','policy_bytes'] +function frames(state:State,id:string):Row[]{ + const output:Row[]=[] + const add=(tag:number,rows:Row[])=>rows.forEach((row,index)=>output.push({tag,ordinal:index+1,...Object.fromEntries(keys.map(key=>[key,row[key]??null]))})) + add(0,state.foreign.slice(0,2));if(state.foreign.length)return output + add(1,state.current.slice(0,2)) + const c=state.current[0],top=state.latest[0] + const valid=state.current.length===1&&c.installation_id===id&&top&&c.epoch===top.epoch&&c.head_sha256===top.head_sha256&&c.installation_id===top.installation_id&&typeof top.epoch==='string'&&/^[1-9][0-9]*$/.test(top.epoch) + add(2,state.latest.slice(0,2).map(row=>({...row,head_bytes:valid?row.head_bytes:null}))) + if(valid)add(3,state.policy.slice(0,2)) + return output +} +export function stagedModelClient(state:Fixture['state']){ + const trace:{text:string;values?:unknown[]}[]=[] + const client:ApplicationWriterRegistryReadSql={async query=Record>(text:string,values?:unknown[]){ + trace.push({text,values:structuredClone(values)});let rows:unknown[] + if(text.startsWith('WITH')){ + if(typeof values?.[0]!=='string')throw Error('MODEL installation parameter required') + rows=frames(state,values[0]) + }else if(text.includes('clock_timestamp()'))rows=state.clock + else throw Error('MODEL unknown grouped SQL; no fake success') + return {rows:structuredClone(rows) as R[]} + }} + return {client,trace} +} diff --git a/backend/src/db/application-writer-profile-registry-read-parity.test.ts b/backend/src/db/application-writer-profile-registry-read-parity.test.ts new file mode 100644 index 00000000..7773a610 --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-read-parity.test.ts @@ -0,0 +1,117 @@ +// MODEL ONLY. Models deliberately cannot establish PostgreSQL execution/order. +import assert from 'node:assert/strict' +import {readFileSync} from 'node:fs' +import {createHash} from 'node:crypto' +import {syntheticFixture,modelClient,type Fixture} from './application-writer-profile-registry-read-model.test.js' +import {readCurrentApplicationWriterProfile as original} from './application-writer-profile-registry-five-query-oracle.test.js' +import {readCurrentApplicationWriterProfile as staged,type ApplicationWriterRegistryReadSql} from './application-writer-profile-registry-read.js' +import {POLICY_DOMAIN} from './application-writer-profile-format.js' +type Row=Record +type State={foreign:Row[];current:Row[];latest:Row[];policy:Row[];clock:Row[]} +const keys=['installation_id','epoch','head_sha256','policy_sha256','previous_installation_id','previous_epoch','previous_head_sha256','head_bytes','policy_bytes'] +function frames(state:State,id:string):Row[]{ + const output:Row[]=[] + const add=(tag:number,rows:Row[])=>rows.forEach((row,index)=>output.push({tag,ordinal:index+1,...Object.fromEntries(keys.map(key=>[key,row[key]??null]))})) + add(0,state.foreign.slice(0,2));if(state.foreign.length)return output + add(1,state.current.slice(0,2)) + const c=state.current[0],top=state.latest[0] + const valid=state.current.length===1&&c.installation_id===id&&top&&c.epoch===top.epoch&&c.head_sha256===top.head_sha256&&c.installation_id===top.installation_id&&typeof top.epoch==='string'&&/^[1-9][0-9]*$/.test(top.epoch) + add(2,state.latest.slice(0,2).map(row=>({...row,head_bytes:valid?row.head_bytes:null}))) + if(valid)add(3,state.policy.slice(0,2)) + return output +} +function groupedModel(state:State,id:string,mutate?:(rows:Row[])=>Row[],clockError?:string){ + const trace:string[]=[],returned:Row[][]=[] + const client:ApplicationWriterRegistryReadSql={async query(text:string,values?:unknown[]){ + trace.push(text) + if(text.startsWith('WITH')){ + assert.deepEqual(values,[id]);assert.equal(trace.length,1) + const rows=mutate?mutate(frames(state,id)):frames(state,id);returned.push(rows) + return {rows:rows as R[]} + } + assert(text.includes('clock_timestamp()'));assert.equal(trace.length,2) + if(clockError)throw Error(clockError) + return {rows:structuredClone(state.clock) as R[]} + }} + return {client,trace,returned} +} +async function outcome(run:()=>Promise){try{return {value:await run()}}catch(error){assert(error instanceof Error);return {error:error.message}}} +let cases=0 +async function parity(name:string,modify:(state:State,f:Fixture)=>void=()=>{},options={}){ + const f=syntheticFixture(),state=structuredClone(f.state) as State;modify(state,f) + const oracle=modelClient(state as Fixture['state']),candidate=groupedModel(state,f.subject.installationId) + const a=await outcome(()=>original(oracle.client,f.subject,options)),b=await outcome(()=>staged(candidate.client,f.subject,options)) + assert.deepEqual(b,a,name);assert.equal(candidate.trace.length,'value' in a&&a.value!==null?2:1,name) + cases++;return {state,f,a,candidate} +} +for(const epoch of ['1','2','9007199254740993','9223372036854775807']){ + const f=syntheticFixture(epoch),a=modelClient(f.state),b=groupedModel(f.state,f.subject.installationId) + assert.deepEqual(await staged(b.client,f.subject),await original(a.client,f.subject));assert.equal(a.trace.length,5);assert.equal(b.trace.length,2);cases++ +} +await parity('empty allowed',s=>{s.current=[];s.latest=[]},{allowEmpty:true}) +await parity('empty refused',s=>{s.current=[];s.latest=[]}) +await parity('orphan latest',s=>{s.current=[]}) +await parity('orphan current',s=>{s.latest=[]}) +await parity('foreign sentinel',s=>{s.foreign=[{installation_id:'22222222-2222-4222-8222-222222222222'}]}) +await parity('two foreign sentinels',s=>{s.foreign=[{installation_id:'0'},{installation_id:'2'}]}) +await parity('duplicate current',s=>{s.current.push({...s.current[0]})}) +await parity('current epoch mismatch',s=>{s.current[0].epoch='1'}) +await parity('current hash mismatch',s=>{s.current[0].head_sha256='f'.repeat(64)}) +await parity('current installation mismatch',s=>{s.current[0].installation_id='22222222-2222-4222-8222-222222222222'}) +await parity('current epoch NULL',s=>{s.current[0].epoch=null}) +await parity('latest epoch NULL',s=>{s.latest[0].epoch=null}) +await parity('missing exact policy',s=>{s.policy=[]}) +await parity('duplicate exact policy',s=>{s.policy.push({...s.policy[0]})}) +await parity('oversize SQL-masked policy',s=>{s.policy[0].policy_bytes=null}) +await parity('oversize SQL-masked current head',s=>{s.latest[0].head_bytes=null}) +await parity('oversize SQL-masked previous head',s=>{s.latest[1].head_bytes=null}) +await parity('forged policy',s=>{s.policy[0].policy_bytes=Buffer.from('tampered')}) +await parity('forged top head',s=>{s.latest[0].head_bytes=Buffer.from('tampered')}) +await parity('forged previous head',s=>{s.latest[1].head_bytes=Buffer.from('tampered')}) +await parity('missing predecessor',s=>{s.latest.pop()}) +await parity('predecessor gap',s=>{s.latest[1].epoch='3'}) +await parity('predecessor installation mismatch',s=>{s.latest[0].previous_installation_id='22222222-2222-4222-8222-222222222222'}) +await parity('predecessor link mismatch',s=>{s.latest[0].previous_head_sha256='f'.repeat(64)}) +await parity('policy hash mismatch',s=>{s.policy[0].policy_sha256='f'.repeat(64)}) +await parity('raw UTF8 string payloads',s=>{for(const row of s.latest)row.head_bytes=Buffer.from(row.head_bytes as Uint8Array).toString('utf8');s.policy[0].policy_bytes=Buffer.from(s.policy[0].policy_bytes as Uint8Array).toString('utf8')}) +for(const clock of [[],[{now:'1'},{now:'2'}],[{now:'invalid'}]]){ + const f=syntheticFixture(),s=structuredClone(f.state) as State;s.clock=clock + const a=modelClient(s as Fixture['state']),b=groupedModel(s,f.subject.installationId) + assert.deepEqual(await outcome(()=>staged(b.client,f.subject)),await outcome(()=>original(a.client,f.subject)));assert.equal(b.trace.length,2);cases++ +} +{const f=syntheticFixture('1');f.state.latest.push({...f.state.latest[0]});const a=modelClient(f.state),b=groupedModel(f.state,f.subject.installationId);assert.deepEqual(await outcome(()=>staged(b.client,f.subject)),await outcome(()=>original(a.client,f.subject)));assert.equal(b.trace.length,1);cases++} +// The clock is later than cryptographic validation, including an expiry that +// happens after a snapshot/model response; an expired inspection still returns. +for(const validNow of [true,false]){ + const f=syntheticFixture(),s=structuredClone(f.state) as State;s.clock=[{now:String(f.policy.notAfter+1)}] + const a=modelClient(s as Fixture['state']),b=groupedModel(s,f.subject.installationId) + assert.deepEqual(await outcome(()=>staged(b.client,f.subject,{validNow})),await outcome(()=>original(a.client,f.subject,{validNow})));assert.equal(b.trace.length,2);cases++ +} +// A supplied subject does not authenticate installation ownership. Reject a +// genuine signature for another supplied public key just as the frozen reader. +{const f=syntheticFixture(),other=syntheticFixture(),b=groupedModel(f.state,f.subject.installationId),a=modelClient(f.state);assert.deepEqual(await outcome(()=>staged(b.client,{...f.subject,publicKeyPem:other.subject.publicKeyPem,publicKeySpkiSha256:other.subject.publicKeySpkiSha256})),await outcome(()=>original(a.client,{...f.subject,publicKeyPem:other.subject.publicKeyPem,publicKeySpkiSha256:other.subject.publicKeySpkiSha256})));assert.equal(b.trace.length,1);cases++} +// Re-signing the policy alone cannot bypass the exact signed head hash binding. +await parity('new signed policy without linked head',(s,f)=>{s.policy[0].policy_bytes=f.signBody({...f.policy,catalogSha256:'f'.repeat(64)},POLICY_DOMAIN)}) +// Direct frame attacks test defensive transport shape validation, not SQL. +const attacks:((rows:Row[])=>Row[])[]=[r=>r.reverse(),r=>[...r,r[0]],r=>r.map((v,i)=>i? v:{...v,extra:true}),r=>r.map((v,i)=>i?v:{...v,ordinal:2}),r=>r.map((v,i)=>i?v:{...v,tag:4}),r=>r.map((v,i)=>i?v:{...v,epoch:2}),r=>r.map(v=>v.tag===3?{...v,policy_bytes:{data:[]}}:v),r=>r.map((v,i)=>i?v:{...v,policy_bytes:Buffer.from('leak')})] +for(const attack of attacks){const f=syntheticFixture(),b=groupedModel(f.state,f.subject.installationId,attack);await assert.rejects(()=>staged(b.client,f.subject));assert.equal(b.trace.length,1);cases++} +for(const section of [1,2,3]){const f=syntheticFixture(),b=groupedModel(f.state,f.subject.installationId,r=>{const rows=r.filter(v=>v.tag===section);return r.filter(v=>Number(v.tag)({...v,ordinal:rows.length+i+1})),r.filter(v=>Number(v.tag)>section))});await assert.rejects(()=>staged(b.client,f.subject));assert.equal(b.trace.length,1);cases++} +{let calls=0;const f=syntheticFixture(),client:ApplicationWriterRegistryReadSql={async query(){calls++;throw Error('snapshot-query-failed')}};await assert.rejects(()=>staged(client,f.subject),/snapshot-query-failed/);assert.equal(calls,1);cases++} +{const f=syntheticFixture(),b=groupedModel(f.state,f.subject.installationId,undefined,'clock-query-failed');await assert.rejects(()=>staged(b.client,f.subject),/clock-query-failed/);assert.equal(b.trace.length,2);cases++} +for(const foreign of [false,true]){ + const f=syntheticFixture(),s=structuredClone(f.state) as State;if(foreign)s.foreign=[{installation_id:'other'}];else s.current[0].epoch='1' + const b=groupedModel(s,f.subject.installationId);await assert.rejects(()=>staged(b.client,f.subject));assert.equal(b.trace.length,1) + assert(b.returned[0].every(r=>r.head_bytes===null&&r.policy_bytes===null));if(foreign)assert(b.returned[0].every(r=>r.tag===0));cases++ +} +// Pin the frozen TEST oracle independently of the changed production reader. +// Normalize CRLF so a checkout's line-ending policy does not change semantics. +const oracle=readFileSync(new URL('./application-writer-profile-registry-five-query-oracle.test.ts',import.meta.url),'utf8').replace(/\r\n/g,'\n') +assert.equal(createHash('sha256').update(oracle).digest('hex'),'b820596939ce295188564307c41126277752ed65303208f17f9b35eb5bf2322f') +// Structural source checks are not a SQL parser/execution proof. The next +// owned PostgreSQL acceptance must prove actual suppression and same snapshot. +const fixture=syntheticFixture(),sql=groupedModel(fixture.state,fixture.subject.installationId);await staged(sql.client,fixture.subject) +assert(sql.trace[0].includes('NOT EXISTS(SELECT 1 FROM foreign_installations)')) +assert(sql.trace[0].includes('JOIN current_gate c')) +assert(sql.trace[0].includes('CASE WHEN EXISTS(SELECT 1 FROM current_gate)')) +assert(sql.trace[0].endsWith('FROM frames ORDER BY tag,ordinal')) +console.log(JSON.stringify({modelOnly:true,cases,postgres:false,runtimeAuthority:false,readFacadeCalls:{before:5,after:2},staticBeforeAfterSqlReduction:30,performanceProved:false,writerAuthority:false,ready:false})) diff --git a/backend/src/db/application-writer-profile-registry-read.test.ts b/backend/src/db/application-writer-profile-registry-read.test.ts new file mode 100644 index 00000000..e217fb8e --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-read.test.ts @@ -0,0 +1,14 @@ +// Self-contained MODEL unit. No genuine SQL/ownership/authority claim. +import assert from 'node:assert/strict' +import {readCurrentApplicationWriterProfile} from './application-writer-profile-registry-read.js' +import {syntheticFixture,stagedModelClient as modelClient} from './application-writer-profile-registry-read-model.test.js' +export {syntheticFixture,stagedModelClient as modelClient,type Fixture} from './application-writer-profile-registry-read-model.test.js' +// Tests execute on direct invocation AND normal test-file import. Exported fixture +// helpers are test-only, used by staged parity; never production authority. +for(const epoch of ['1','2','9007199254740993','9223372036854775807']){const f=syntheticFixture(epoch),m=modelClient(f.state),value=await readCurrentApplicationWriterProfile(m.client,f.subject);assert(value);assert.equal(value.epoch,epoch);assert.equal(value.headSha256,f.state.current[0].head_sha256);assert.equal(m.trace.length,2);assert.ok(m.trace.every(q=>/^(?:WITH|\(?SELECT)\b/.test(q.text)));assert.deepEqual(Object.keys(value).sort(),['epoch','headSha256','policy','policySha256','serverNow'])} +{const f=syntheticFixture('1');f.state.current=[];f.state.latest=[];const m=modelClient(f.state);assert.equal(await readCurrentApplicationWriterProfile(m.client,f.subject,{allowEmpty:true}),null);assert.equal(m.trace.length,1);await assert.rejects(()=>readCurrentApplicationWriterProfile(modelClient(f.state).client,f.subject))} +{const f=syntheticFixture();f.state.current[0].epoch='1';await assert.rejects(()=>readCurrentApplicationWriterProfile(modelClient(f.state).client,f.subject),/Current does not match/)} +{const f=syntheticFixture();f.state.latest[1].head_bytes=Buffer.from('tampered');await assert.rejects(()=>readCurrentApplicationWriterProfile(modelClient(f.state).client,f.subject))} +{const f=syntheticFixture();f.state.policy[0].policy_bytes=null;await assert.rejects(()=>readCurrentApplicationWriterProfile(modelClient(f.state).client,f.subject),/Bounded signed registry payload absent/)} +{const f=syntheticFixture();f.state.foreign=[{installation_id:'22222222-2222-4222-8222-222222222222'}];const m=modelClient(f.state);await assert.rejects(()=>readCurrentApplicationWriterProfile(m.client,f.subject),/Foreign installation/);assert.equal(m.trace.length,1)} +console.log(JSON.stringify({modelUnit:true,unitGroups:6,postgres:false,currentCasWrite:false,writerAuthorityMinted:false,readyAuthorized:false})) diff --git a/backend/src/db/application-writer-profile-registry-read.ts b/backend/src/db/application-writer-profile-registry-read.ts new file mode 100644 index 00000000..a647c8c8 --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-read.ts @@ -0,0 +1,128 @@ +import {epoch,signedEnvelope,liveValidity,type ApplicationWriterPolicy} from './application-writer-profile-format.js' + +// Internal read-side value component ONLY. No configured pool, provider/pin JSON +// parser, publish/DDL/CAS write, key loader, role provisioning or authority mint. +// Caller retains authenticated backend/source/lock/transaction/lifetime ownership. +// Structure/index/ACL checks and protected subject provenance are separate gates. +export const HISTORY='application_writer_profile_history',CURRENT='application_writer_profile_current' +export interface ApplicationWriterRegistryReadSql { + query=Record>(text:string,values?:unknown[]):Promise<{rows:R[]}> +} +type Subject=Parameters[2] +interface CurrentRow extends Record {installation_id:string;epoch:string;head_sha256:string} +interface HeadRow extends CurrentRow {policy_sha256:string;previous_installation_id:string|null;previous_epoch:string|null;previous_head_sha256:string|null;head_bytes:Uint8Array|string|null} +interface PolicyRow extends Record {policy_sha256:string;policy_bytes:Uint8Array|string|null} +export interface ApplicationWriterRegistryReadOptions {allowEmpty?:boolean;validNow?:boolean} +/** Ordinary observed data, NOT a CAS reservation/current-state capability. + * Subject must be fixed by a future protected private host, never a request. + * validNow=false is reserved for trusted expired-policy inspection/renewal; + * it is not a runtime authorization option. No trusted state is created here. */ +export interface ApplicationWriterRegistryCurrentData {epoch:string;headSha256:string;policy:ApplicationWriterPolicy;policySha256:string;serverNow:number} +function payload(value:Uint8Array|string|null):Uint8Array|string { + // SQL CASE turns oversize/empty bytes into NULL; reject before parsing. Same + // refusal as MJS Buffer.from(null), with an explicit internal diagnostic. + if(value===null)throw Error('Bounded signed registry payload absent') + return value +} +// Staged fixed DATA query. Every relation-dependent read shares this statement +// snapshot. Foreign/current-invalid metadata never releases policy/head bytes. +const FIXED_REGISTRY_SNAPSHOT_SQL=`WITH + foreign_installations AS MATERIALIZED ( + (SELECT installation_id FROM mmapp_control.application_writer_profile_history WHERE installation_id<$1::uuid ORDER BY installation_id DESC LIMIT 1) + UNION ALL + (SELECT installation_id FROM mmapp_control.application_writer_profile_history WHERE installation_id>$1::uuid ORDER BY installation_id LIMIT 1)), + current_rows AS MATERIALIZED ( + SELECT installation_id,epoch,head_sha256 FROM mmapp_control.application_writer_profile_current + WHERE id=1 AND NOT EXISTS(SELECT 1 FROM foreign_installations) + ORDER BY installation_id,epoch,head_sha256 LIMIT 2), + latest_metadata AS MATERIALIZED ( + SELECT installation_id,epoch,head_sha256,policy_sha256,previous_installation_id,previous_epoch,previous_head_sha256 + FROM mmapp_control.application_writer_profile_history + WHERE installation_id=$1::uuid AND NOT EXISTS(SELECT 1 FROM foreign_installations) + ORDER BY epoch DESC LIMIT 2), + current_gate AS MATERIALIZED ( + SELECT c.* FROM current_rows c JOIN (SELECT * FROM latest_metadata ORDER BY epoch DESC LIMIT 1)t + ON c.installation_id=t.installation_id AND c.epoch=t.epoch AND c.head_sha256=t.head_sha256 + WHERE (SELECT count(*) FROM current_rows)=1 AND c.installation_id=$1::uuid AND t.epoch>0), + policy_rows AS MATERIALIZED ( + SELECT h.policy_sha256,CASE WHEN pg_catalog.octet_length(h.policy_bytes) BETWEEN 1 AND 1048576 THEN h.policy_bytes ELSE NULL END AS policy_bytes + FROM mmapp_control.application_writer_profile_history h JOIN current_gate c + ON h.installation_id=c.installation_id AND h.epoch=c.epoch AND h.head_sha256=c.head_sha256 + ORDER BY h.policy_sha256 LIMIT 2), + frames AS ( + SELECT 0::int AS tag,(row_number() OVER(ORDER BY installation_id))::int AS ordinal,installation_id::text, + NULL::text AS epoch,NULL::text AS head_sha256,NULL::text AS policy_sha256,NULL::text AS previous_installation_id, + NULL::text AS previous_epoch,NULL::text AS previous_head_sha256,NULL::bytea AS head_bytes,NULL::bytea AS policy_bytes + FROM foreign_installations + UNION ALL + SELECT 1,(row_number() OVER(ORDER BY installation_id,epoch,head_sha256))::int,installation_id::text,epoch::text,head_sha256, + NULL,NULL,NULL,NULL,NULL::bytea,NULL::bytea FROM current_rows + UNION ALL + SELECT 2,(row_number() OVER(ORDER BY m.epoch DESC))::int,m.installation_id::text,m.epoch::text,m.head_sha256,m.policy_sha256, + m.previous_installation_id::text,m.previous_epoch::text,m.previous_head_sha256, + CASE WHEN EXISTS(SELECT 1 FROM current_gate) THEN + (SELECT CASE WHEN pg_catalog.octet_length(h.head_bytes) BETWEEN 1 AND 8192 THEN h.head_bytes ELSE NULL END + FROM mmapp_control.application_writer_profile_history h + WHERE h.installation_id=m.installation_id AND h.epoch=m.epoch AND h.head_sha256=m.head_sha256 LIMIT 1) + ELSE NULL::bytea END,NULL::bytea FROM latest_metadata m + UNION ALL + SELECT 3,(row_number() OVER(ORDER BY policy_sha256))::int,NULL,NULL,NULL,policy_sha256,NULL,NULL,NULL,NULL::bytea,policy_bytes FROM policy_rows) + SELECT tag,ordinal,installation_id,epoch,head_sha256,policy_sha256,previous_installation_id,previous_epoch,previous_head_sha256,head_bytes,policy_bytes + FROM frames ORDER BY tag,ordinal` + +function decodeSnapshot(rows:Record[],installationId:string) { + const result={foreign:[] as Record[],current:[] as CurrentRow[],latest:[] as HeadRow[],policy:[] as PolicyRow[]} + const fields=['tag','ordinal','installation_id','epoch','head_sha256','policy_sha256','previous_installation_id','previous_epoch','previous_head_sha256','head_bytes','policy_bytes'] + const used=[['installation_id'],['installation_id','epoch','head_sha256'],['installation_id','epoch','head_sha256','policy_sha256','previous_installation_id','previous_epoch','previous_head_sha256','head_bytes'],['policy_sha256','policy_bytes']] + let previousTag=-1,ordinal=0 + if(!Array.isArray(rows)||rows.length>8)throw Error('Bounded registry snapshot required') + for(const row of rows){ + if(!row||typeof row!=='object'||Reflect.ownKeys(row).length!==fields.length||fields.some(k=>!Object.hasOwn(row,k)))throw Error('Exact registry snapshot frame required') + const tag=row.tag + if(typeof tag!=='number'||!Number.isInteger(tag)||tag<0||tag>3||tag2)throw Error('Bounded ordered registry snapshot section required') + for(const name of fields.slice(2)){ + const value=row[name] + if(!used[tag].includes(name)){if(value!==null)throw Error('Unused registry snapshot field refused');continue} + if(name.endsWith('_bytes')){if(value!==null&&typeof value!=='string'&&!(value instanceof Uint8Array))throw Error('Raw registry snapshot bytes required')} + else if(value!==null&&typeof value!=='string')throw Error('Text registry snapshot field required') + } + if(tag===0)result.foreign.push({installation_id:row.installation_id}) + if(tag===1)result.current.push({installation_id:row.installation_id,epoch:row.epoch,head_sha256:row.head_sha256} as CurrentRow) + if(tag===2)result.latest.push(Object.fromEntries(used[2].map(k=>[k,row[k]])) as HeadRow) + if(tag===3)result.policy.push({policy_sha256:row.policy_sha256,policy_bytes:row.policy_bytes} as PolicyRow) + } + // SQL must suppress all dependent sections behind a foreign sentinel, and + // payload-bearing sections behind invalid current/top metadata. Fail closed + // even if an unexpected transport/model supplies contradictory frames. + if(result.foreign.length&&(result.current.length||result.latest.length||result.policy.length))throw Error('Foreign registry snapshot released dependent rows') + const c=result.current[0],t=result.latest[0] + const matches=result.current.length===1&&t&&c.installation_id===installationId&&c.installation_id===t.installation_id&&c.epoch===t.epoch&&c.head_sha256===t.head_sha256&&typeof t.epoch==='string'&&/^[1-9][0-9]*$/.test(t.epoch) + if(!matches&&(result.policy.length||result.latest.some(r=>r.head_bytes!==null)))throw Error('Invalid current registry snapshot released payload') + return result +} +export async function readCurrentApplicationWriterProfile(client:ApplicationWriterRegistryReadSql,installation:Subject,{allowEmpty=false,validNow=true}:ApplicationWriterRegistryReadOptions={}):Promise{ + const sections=decodeSnapshot((await client.query(FIXED_REGISTRY_SNAPSHOT_SQL,[installation.installationId])).rows,installation.installationId) + const {foreign,current,latest}=sections + if(foreign.length)throw Error('Foreign installation history refused') + if(current.length===0&&latest.length===0&&allowEmpty)return null + const c=current[0],top=latest[0] + if(current.length!==1||!top||c.installation_id!==installation.installationId||c.epoch!==epoch(top.epoch)||c.head_sha256!==top.head_sha256)throw Error('Current does not match indexed append-only latest') + const row=sections.policy + if(row.length!==1)throw Error('Exact current policy row absent') + const policy=signedEnvelope(payload(row[0].policy_bytes),'policy',installation),head=signedEnvelope(payload(top.head_bytes),'head',installation) + if(head.sha256!==c.head_sha256||head.body.epoch!==c.epoch||policy.body.epoch!==c.epoch||head.body.policySha256!==policy.sha256||row[0].policy_sha256!==policy.sha256||top.policy_sha256!==policy.sha256||head.body.previousHeadSha256!==top.previous_head_sha256)throw Error('Exact current signature/hash/link refused') + if(c.epoch==='1'){ + if(latest.length!==1||top.previous_epoch!==null||top.previous_installation_id!==null||top.previous_head_sha256!==null)throw Error('Unique genesis required') + }else{ + const previous=latest[1] + if(!previous||BigInt(epoch(previous.epoch))+1n!==BigInt(c.epoch)||top.previous_epoch!==previous.epoch||top.previous_installation_id!==installation.installationId||top.previous_head_sha256!==previous.head_sha256)throw Error('Exact consecutive predecessor required') + const before=signedEnvelope(payload(previous.head_bytes),'head',installation) + if(before.sha256!==previous.head_sha256||before.body.epoch!==previous.epoch||before.body.policySha256!==previous.policy_sha256||before.body.previousHeadSha256!==previous.previous_head_sha256)throw Error('Historical signed predecessor refused') + } + const now=(await client.query<{now:string}>("SELECT (EXTRACT(EPOCH FROM pg_catalog.clock_timestamp())*1000)::bigint::text AS now")).rows + if(now.length!==1)throw Error('Actual server clock required') + if(validNow)liveValidity(policy.body,Number(now[0].now)) + return {epoch:c.epoch,headSha256:c.head_sha256,policy:policy.body,policySha256:policy.sha256,serverNow:Number(now[0].now)} +} diff --git a/backend/src/db/application-writer-profile-registry-structure.test.ts b/backend/src/db/application-writer-profile-registry-structure.test.ts new file mode 100644 index 00000000..db852e0c --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-structure.test.ts @@ -0,0 +1,55 @@ +// MODEL metadata only. Never a PG/role/ownership/authority attestor. +import assert from 'node:assert/strict' +import {readProtectedRegistryStructureData,type RegistryLogicalRole,type RegistryStructureReadSql} from './application-writer-profile-registry-structure.js' +type ModelRow=Record +const current='application_writer_profile_current',history='application_writer_profile_history' +const logical=['runtime','operator','capture','publisher','control'] as const +export function structureFixture(custom=false){ + const names=custom?['run \"data','operator ; --','capture unicode-ё','pub []','control \"owner']:['barrier_runtime','barrier_operator','barrier_capture','barrier_profile_publisher','barrier_admin'] + const bindings=Object.fromEntries(logical.map((scope,i)=>[scope,{name:names[i],signedOid:String(1001+i)}])) as Record + const rows:Record={} + rows.roles=logical.filter(scope=>scope!=='control').map(scope=>({rolname:bindings[scope].name,oid:bindings[scope].signedOid,rolinherit:true,rolcanlogin:true,unsafe:false,db_owner:false,membership:false,predefined_member:false,db_create:false,db_temp:false,set_replication:false,set_event:false,source_read:scope==='operator'||scope==='publisher'})).sort((a,b)=>a.rolname.localeCompare(b.rolname)) + const fileFunctions=["pg_catalog.lo_import(text)", "pg_catalog.lo_import(text,oid)", "pg_catalog.lo_export(oid,text)", "pg_catalog.pg_read_file(text)", "pg_catalog.pg_read_file(text,boolean)", "pg_catalog.pg_read_file(text,bigint,bigint)", "pg_catalog.pg_read_file(text,bigint,bigint,boolean)", "pg_catalog.pg_read_binary_file(text)", "pg_catalog.pg_read_binary_file(text,boolean)", "pg_catalog.pg_read_binary_file(text,bigint,bigint)", "pg_catalog.pg_read_binary_file(text,bigint,bigint,boolean)", "pg_catalog.pg_ls_dir(text)", "pg_catalog.pg_ls_dir(text,boolean,boolean)", "pg_catalog.pg_stat_file(text)", "pg_catalog.pg_stat_file(text,boolean)"] + rows.fileAcl=rows.roles.flatMap(role=>fileFunctions.map((signature,i)=>({rolname:role.rolname,signature,function_oid:String(2001+i),execute:false}))).sort((a,b)=>(String(a.rolname)+a.signature).localeCompare(String(b.rolname)+b.signature)) + rows.privileged=[{rolname:bindings.operator.name,nspname:'mmapp_control',proname:'candidate_truncate_preadmitted',oid:'3001',arguments:0}] + rows.tables=[current,history].map((relname,i)=>({oid:String(5001+i),relname,relkind:'r',relpersistence:'p',relrowsecurity:false,relforcerowsecurity:false,relhasrules:false,owner:bindings.control.name,inherited:false,user_trigger:false})) + rows.owner=[{oid:bindings.control.signedOid,name:bindings.control.name}] + const fields:Record={"application_writer_profile_history": [["installation_id", "uuid", true], ["epoch", "int8", true], ["head_sha256", "text", true], ["previous_installation_id", "uuid", false], ["previous_epoch", "int8", false], ["previous_head_sha256", "text", false], ["policy_sha256", "text", true], ["policy_bytes", "bytea", true], ["head_bytes", "bytea", true]], "application_writer_profile_current": [["id", "int2", true], ["installation_id", "uuid", true], ["epoch", "int8", true], ["head_sha256", "text", true]]} + rows.columns=[current,history].flatMap(relname=>fields[relname].map(([attname,typname,attnotnull])=>({relname,attname,typname,type_namespace:'pg_catalog',attnotnull,attidentity:'',attgenerated:'',column_acl:false,has_default:false}))) + rows.acl=[] + for(const relname of [current,history])for(const [grantee,permissions]of [[bindings.control.name,['DELETE','INSERT','MAINTAIN','REFERENCES','SELECT','TRIGGER','TRUNCATE','UPDATE']],[bindings.operator.name,['SELECT']],[bindings.publisher.name,relname===current?['INSERT','SELECT','UPDATE']:['INSERT','SELECT']]] as [string,string[]][])for(const privilege_type of permissions)rows.acl.push({relname,grantee,grantor:bindings.control.name,privilege_type,is_grantable:false}) + rows.constraints=[] + function constraint(relname:string,contype:string,definition:string,keys:number[],referenced:string|null=null,referenced_keys:number[]|null=null,confmatchtype=' '){return {relname,contype,convalidated:true,condeferrable:false,condeferred:false,confmatchtype,keys,referenced,referenced_keys,definition}} + rows.constraints.push(constraint(current,'c','CHECK ((id = 1))',[1]),constraint(current,'f','FOREIGN KEY (installation_id, epoch, head_sha256) REFERENCES mmapp_control.application_writer_profile_history(installation_id, epoch, head_sha256)',[2,3,4],history,[1,2,3],'s'),constraint(current,'p','PRIMARY KEY (id)',[1])) + for(const definition of ["CHECK ((epoch > 0))", "CHECK ((head_sha256 ~ '^[a-f0-9]{64}$'::text))", "CHECK ((policy_sha256 ~ '^[a-f0-9]{64}$'::text))", "CHECK (((octet_length(policy_bytes) >= 1) AND (octet_length(policy_bytes) <= 1048576)))", "CHECK (((octet_length(head_bytes) >= 1) AND (octet_length(head_bytes) <= 8192)))", "CHECK ((((epoch = 1) AND (previous_installation_id IS NULL) AND (previous_epoch IS NULL) AND (previous_head_sha256 IS NULL)) OR ((epoch > 1) AND (previous_installation_id IS NOT NULL) AND (previous_installation_id = installation_id) AND (previous_epoch IS NOT NULL) AND (previous_epoch = (epoch - 1)) AND (previous_head_sha256 IS NOT NULL))))"])rows.constraints.push(constraint(history,'c',definition,[1])) + rows.constraints.push(constraint(history,'f','FOREIGN KEY (previous_installation_id, previous_epoch, previous_head_sha256) REFERENCES mmapp_control.application_writer_profile_history(installation_id, epoch, head_sha256) MATCH FULL',[4,5,6],history,[1,2,3],'f'),constraint(history,'p','PRIMARY KEY (installation_id, epoch)',[1,2]),constraint(history,'u','UNIQUE (installation_id, epoch, head_sha256)',[1,2,3])) + rows.triggers=[...['RI_FKey_check_ins','RI_FKey_check_upd'].map(proname=>({relname:current,tgisinternal:true,tgenabled:'O',nspname:'pg_catalog',proname})),...['RI_FKey_check_ins','RI_FKey_check_upd','RI_FKey_noaction_del','RI_FKey_noaction_upd','RI_FKey_noaction_del','RI_FKey_noaction_upd'].map(proname=>({relname:history,tgisinternal:true,tgenabled:'O',nspname:'pg_catalog',proname}))] + rows.indexes=[{relname:current,indisprimary:true,keys:[1]},{relname:history,indisprimary:false,keys:[1,2,3]},{relname:history,indisprimary:true,keys:[1,2]}].map(row=>({...row,indisunique:true,indisvalid:true,indisready:true,indislive:true,unconditional:true,simple:true,amname:'btree'})) + return {bindings,rows} +} +export function structureModel(fixture:ReturnType,options:{firstQuery?:()=>void;constraintError?:Error;restoreError?:Error}={}){ + const trace:{sql:string;values?:unknown[]}[]=[] + const client:RegistryStructureReadSql={async query=Record>(sql:string,values?:unknown[]){trace.push({sql,values:structuredClone(values)});if(trace.length===1)options.firstQuery?.();let key:string + if(sql.startsWith('SELECT r.rolname,r.oid::text AS oid,r.rolinherit'))key='roles' + else if(sql.startsWith('SELECT r.rolname,f.signature'))key='fileAcl' + else if(sql.startsWith('SELECT r.rolname,n.nspname,p.proname'))key='privileged' + else if(sql.startsWith('SELECT c.oid::text AS oid,c.relname'))key='tables' + else if(sql.startsWith('SELECT n.nspowner::text AS oid')){const owner=fixture.rows.owner;return {rows:structuredClone(sql.includes(' AS name')?owner:owner.map(row=>({oid:row.oid}))) as R[]}} + else if(sql.startsWith('SELECT c.relname,a.attname'))key='columns' + else if(sql.startsWith('SELECT c.relname,grantee.rolname'))key='acl' + else if(sql.startsWith("SELECT pg_catalog.current_setting('search_path')"))return {rows:[{path:'model_outer_local'}] as unknown as R[]} + else if(sql.startsWith('SELECT c.relname,k.contype')){if(options.constraintError)throw options.constraintError;key='constraints'} + else if(sql.startsWith("SELECT pg_catalog.set_config('search_path',$1,true)")){if(options.restoreError)throw options.restoreError;return {rows:[]}} + else if(sql.startsWith('SELECT c.relname,t.tgisinternal'))key='triggers' + else if(sql.startsWith('SELECT c.relname,i.indisunique'))key='indexes' + else throw Error('MODEL unrecognized SQL; no fake success') + return {rows:structuredClone(fixture.rows[key]) as R[]} + }} + return {client,trace} +} +const fixture=structureFixture(),model=structureModel(fixture),result=await readProtectedRegistryStructureData(model.client,fixture.bindings) +assert.match(result.sha256,/^[a-f0-9]{64}$/);assert.equal(model.trace.length,12);assert.deepEqual(result.roles,Object.fromEntries(logical.map(scope=>[scope,fixture.bindings[scope].signedOid])));assert.deepEqual(Object.keys(result).sort(),['roles','sha256']) +{const f=structureFixture(true),m=structureModel(f);await readProtectedRegistryStructureData(m.client,f.bindings);for(const row of m.trace)for(const pair of Object.values(f.bindings))assert(!row.sql.includes(pair.name),'configured names must be bound DATA only')} +{const f=structureFixture(),m=structureModel(f,{firstQuery(){f.bindings.operator.name='mutated_after_await';f.bindings.control.signedOid='9999'}});await readProtectedRegistryStructureData(m.client,f.bindings)} +for(const mutation of ['owner-name','grant-option','fk','index'] as const){const f=structureFixture();if(mutation==='owner-name')f.rows.owner[0].name='renamed_control';if(mutation==='grant-option')f.rows.acl[0].is_grantable=true;if(mutation==='fk')f.rows.constraints.find(r=>r.contype==='f'&&r.relname===history)!.confmatchtype='s';if(mutation==='index')f.rows.indexes[0].indisvalid=false;await assert.rejects(()=>readProtectedRegistryStructureData(structureModel(f).client,f.bindings))} +console.log(JSON.stringify({modelUnit:true,groups:4,alternateNamesAreDataOnly:true,immutableBeforeAwait:true,postgres:false,authority:false})) diff --git a/backend/src/db/application-writer-profile-registry-structure.ts b/backend/src/db/application-writer-profile-registry-structure.ts new file mode 100644 index 00000000..93f052cd --- /dev/null +++ b/backend/src/db/application-writer-profile-registry-structure.ts @@ -0,0 +1,137 @@ +import {canonical,sha} from './application-writer-profile-canonical.js' + +// Private read-side DATA only. No role provisioning, pool/provider, public JSON +// pin parser, keyloader/publish/CAS/DDL, context ownership or writer authority. +export type RegistryLogicalRole='runtime'|'operator'|'capture'|'publisher'|'control' +export interface RegistryRoleBindingData {readonly name:string;readonly signedOid:string} +export type RegistryRoleBindingsData=Readonly>> +export interface RegistryStructureReadSql {query=Record>(text:string,values?:unknown[]):Promise<{rows:R[]}>} +export interface RegistryStructureReadData {readonly sha256:string;readonly roles:Readonly>} +const logicalRoles=['runtime','operator','capture','publisher','control'] as const +const actorRoles=['runtime','operator','capture','publisher'] as const +export const HISTORY='application_writer_profile_history',CURRENT='application_writer_profile_current' +type RegistryTable=typeof HISTORY|typeof CURRENT +type Row=Record +interface RoleRow extends Row {rolname:string;oid:string;rolinherit:boolean;rolcanlogin:boolean;unsafe:boolean;db_owner:boolean;membership:boolean;predefined_member:boolean;db_create:boolean;db_temp:boolean;set_replication:boolean;set_event:boolean;source_read:boolean} +interface FileAclRow extends Row {rolname:string;signature:string;function_oid:string|null;execute:boolean} +interface PrivilegedFunctionRow extends Row {rolname:string;nspname:string;proname:string;oid:string;arguments:number} +interface TableRow extends Row {oid:string;relname:RegistryTable;relkind:string;relpersistence:string;relrowsecurity:boolean;relforcerowsecurity:boolean;relhasrules:boolean;owner:string;inherited:boolean;user_trigger:boolean} +interface OwnerRow extends Row {oid:string;name:string} +interface ColumnRow extends Row {relname:RegistryTable;attname:string;typname:string;type_namespace:string;attnotnull:boolean;attidentity:string;attgenerated:string;column_acl:boolean;has_default:boolean} +interface AclRow extends Row {relname:RegistryTable;grantee:string|null;grantor:string;privilege_type:string;is_grantable:boolean} +interface ConstraintRow extends Row {relname:RegistryTable;contype:string;convalidated:boolean;condeferrable:boolean;condeferred:boolean;confmatchtype:string;keys:number[];referenced:string|null;referenced_keys:number[]|null;definition:string} +interface TriggerRow extends Row {relname:RegistryTable;tgisinternal:boolean;tgenabled:string;nspname:string;proname:string} +interface IndexRow extends Row {relname:RegistryTable;indisunique:boolean;indisprimary:boolean;indisvalid:boolean;indisready:boolean;indislive:boolean;keys:number[];unconditional:boolean;simple:boolean;amname:string} +const same=(a:unknown,b:unknown,phase:string)=>{if(canonical(a)!==canonical(b))throw Error('Protected registry refused: '+phase)} +function object(value:unknown):value is Record{return value!==null&&typeof value==='object'&&!Array.isArray(value)&&Object.getPrototypeOf(value)===Object.prototype} +function exactKeys(value:Record,keys:readonly string[]){const actual=Reflect.ownKeys(value);return actual.every(k=>typeof k==='string')&&actual.length===keys.length&&actual.slice().sort().join('\0')===keys.slice().sort().join('\0')} +function captureBindings(input:unknown):RegistryRoleBindingsData { + if(!object(input)||!exactKeys(input,logicalRoles))throw Error('Exact five private role bindings required') + const names=new Set(),oids=new Set(),captured={} as Record> + for(const scope of logicalRoles){ + const pair=input[scope] + if(!object(pair)||!exactKeys(pair,['name','signedOid']))throw Error('Exact private role name/OID pair required') + // Read each property ONCE, validate primitives, and never touch caller data + // after the first await. Freeze our own copies, not the caller's objects. + const name=pair.name,signedOid=pair.signedOid + if(typeof name!=='string'||!name||name.includes('\0')||Buffer.byteLength(name,'utf8')>63||Buffer.from(name,'utf8').toString('utf8')!==name||typeof signedOid!=='string'||!(/^[1-9][0-9]{0,9}$/).test(signedOid)||BigInt(signedOid)>4294967295n||names.has(name)||oids.has(signedOid))throw Error('Distinct canonical private role identities required') + names.add(name);oids.add(signedOid);captured[scope]=Object.freeze({name,signedOid}) + } + return Object.freeze(captured) +} +function constraintSql(request:string):string { + // Exactly ONE fixed query-owned deparser call, not a caller SQL assembly API. + const expression='pg_catalog.pg_get_constraintdef(k.oid)' + if(request.split(expression).length!==2)throw Error('Fixed registry deparser template changed') + return request.replace(expression,"pg_catalog.pg_get_constraintdef(CASE WHEN pg_catalog.set_config('search_path','pg_catalog, public',true) IS NOT NULL THEN k.oid ELSE NULL END)") +} +const fields:Record={ + [HISTORY]:[['installation_id','uuid',true],['epoch','int8',true],['head_sha256','text',true],['previous_installation_id','uuid',false],['previous_epoch','int8',false],['previous_head_sha256','text',false],['policy_sha256','text',true],['policy_bytes','bytea',true],['head_bytes','bytea',true]], + [CURRENT]:[['id','int2',true],['installation_id','uuid',true],['epoch','int8',true],['head_sha256','text',true]], +} +export async function readProtectedRegistryStructureData(client:RegistryStructureReadSql,input:RegistryRoleBindingsData,runtimeReads=false):Promise{ + const captured=captureBindings(input),scopes=Object.fromEntries(logicalRoles.map(scope=>[scope,captured[scope].name])) as Record + const signedOids=Object.fromEntries(logicalRoles.map(scope=>[scope,captured[scope].signedOid])) as Record + const actors=actorRoles.map(scope=>scopes[scope]),OPERATOR=scopes.operator,PUBLISHER=scopes.publisher + const roleRows=(await client.query(`SELECT r.rolname,r.oid::text AS oid,r.rolinherit,r.rolcanlogin,(r.rolsuper OR r.rolcreatedb OR r.rolcreaterole OR r.rolreplication OR r.rolbypassrls) AS unsafe, + EXISTS(SELECT 1 FROM pg_catalog.pg_database d WHERE d.datdba=r.oid) AS db_owner, + EXISTS(SELECT 1 FROM pg_catalog.pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid) AS membership, + EXISTS(SELECT 1 FROM pg_catalog.pg_roles p WHERE p.rolname~'^pg_' AND pg_catalog.pg_has_role(r.oid,p.oid,'MEMBER')) AS predefined_member, + pg_catalog.has_database_privilege(r.oid,(SELECT oid FROM pg_catalog.pg_database WHERE datname=pg_catalog.current_database()),'CREATE') AS db_create, + pg_catalog.has_database_privilege(r.oid,(SELECT oid FROM pg_catalog.pg_database WHERE datname=pg_catalog.current_database()),'TEMP') AS db_temp, + pg_catalog.has_parameter_privilege(r.oid,'session_replication_role','SET') AS set_replication, + pg_catalog.has_parameter_privilege(r.oid,'event_triggers','SET') AS set_event, + pg_catalog.has_function_privilege(r.oid,'pg_catalog.pg_control_system()','EXECUTE') AS source_read + FROM pg_catalog.pg_roles r WHERE r.rolname=ANY($1::text[]) ORDER BY r.rolname`,[actors])).rows + if(roleRows.length!==4)throw Error('Exact subject/publisher roles required') + for(const row of roleRows){const scope=Object.entries(scopes).find(([,name])=>name===row.rolname)?.[0] as RegistryLogicalRole|undefined;if(!scope||row.oid!==signedOids[scope]||row.rolinherit!==true||row.rolcanlogin!==true||['unsafe','db_owner','membership','predefined_member','db_create','db_temp','set_replication','set_event'].some(k=>row[k]!==false)||[PUBLISHER,OPERATOR].includes(row.rolname)&&row.source_read!==true)throw Error('Role ownership/membership/file-program/parameter escape refused')} + const fileFunctions=['pg_catalog.lo_import(text)','pg_catalog.lo_import(text,oid)','pg_catalog.lo_export(oid,text)','pg_catalog.pg_read_file(text)','pg_catalog.pg_read_file(text,boolean)','pg_catalog.pg_read_file(text,bigint,bigint)','pg_catalog.pg_read_file(text,bigint,bigint,boolean)','pg_catalog.pg_read_binary_file(text)','pg_catalog.pg_read_binary_file(text,boolean)','pg_catalog.pg_read_binary_file(text,bigint,bigint)','pg_catalog.pg_read_binary_file(text,bigint,bigint,boolean)','pg_catalog.pg_ls_dir(text)','pg_catalog.pg_ls_dir(text,boolean,boolean)','pg_catalog.pg_stat_file(text)','pg_catalog.pg_stat_file(text,boolean)'] + const fileAcl=(await client.query(`SELECT r.rolname,f.signature,pg_catalog.to_regprocedure(f.signature)::oid::text AS function_oid, + pg_catalog.has_function_privilege(r.oid,pg_catalog.to_regprocedure(f.signature)::oid,'EXECUTE') AS execute + FROM pg_catalog.pg_roles r CROSS JOIN pg_catalog.unnest($2::text[])f(signature) WHERE r.rolname=ANY($1::text[]) ORDER BY r.rolname,f.signature`,[actors,fileFunctions])).rows + if(fileAcl.length!==actors.length*fileFunctions.length||fileAcl.some(row=>row.function_oid===null||row.execute!==false))throw Error('Direct large-object/server-file function grant refused') + const privilegedFunctions=(await client.query(`SELECT r.rolname,n.nspname,p.proname,p.oid::text AS oid,p.pronargs::int AS arguments + FROM pg_catalog.pg_roles r CROSS JOIN pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace + WHERE r.rolname=ANY($1::text[]) AND p.prosecdef AND pg_catalog.has_schema_privilege(r.oid,n.oid,'USAGE') + AND pg_catalog.has_function_privilege(r.oid,p.oid,'EXECUTE') ORDER BY r.rolname,n.nspname,p.proname,p.oid LIMIT 9`,[actors])).rows + if(privilegedFunctions.length!==1||privilegedFunctions.some(row=>row.rolname!==OPERATOR||row.nspname!=='mmapp_control'||row.proname!=='candidate_truncate_preadmitted'||row.arguments!==0))throw Error('Unknown reachable SECURITY DEFINER escape refused') + const tables=(await client.query(`SELECT c.oid::text AS oid,c.relname,c.relkind,c.relpersistence,c.relrowsecurity,c.relforcerowsecurity,c.relhasrules,r.rolname AS owner, + EXISTS(SELECT 1 FROM pg_catalog.pg_inherits i WHERE i.inhrelid=c.oid OR i.inhparent=c.oid) AS inherited, + EXISTS(SELECT 1 FROM pg_catalog.pg_trigger t WHERE t.tgrelid=c.oid AND NOT t.tgisinternal) AS user_trigger + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace JOIN pg_catalog.pg_roles r ON r.oid=c.relowner + WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) ORDER BY c.relname`,[[CURRENT,HISTORY]])).rows + if(tables.length!==2||tables.some(t=>t.relkind!=='r'||t.relpersistence!=='p'||t.relrowsecurity||t.relforcerowsecurity||t.user_trigger||t.relhasrules||t.inherited||t.owner!==scopes.control))throw Error('Protected fixed relation ownership/structure refused') + const owner=(await client.query("SELECT n.nspowner::text AS oid,pg_catalog.pg_get_userbyid(n.nspowner) AS name FROM pg_catalog.pg_namespace n WHERE n.nspname='mmapp_control'")).rows + if(owner.length!==1||owner[0].oid!==signedOids.control||owner[0].name!==scopes.control)throw Error('Protected control namespace owner refused') + const columns=(await client.query(`SELECT c.relname,a.attname,t.typname,tn.nspname AS type_namespace,a.attnotnull,a.attidentity,a.attgenerated,a.attacl IS NOT NULL AS column_acl,d.oid IS NOT NULL AS has_default + FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid=a.attrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + JOIN pg_catalog.pg_type t ON t.oid=a.atttypid JOIN pg_catalog.pg_namespace tn ON tn.oid=t.typnamespace LEFT JOIN pg_catalog.pg_attrdef d ON d.adrelid=c.oid AND d.adnum=a.attnum + WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) AND a.attnum>0 AND NOT a.attisdropped ORDER BY c.relname,a.attnum`,[[CURRENT,HISTORY]])).rows + for(const table of tables){const c=columns.filter(c=>c.relname===table.relname);same(c.map(r=>[r.attname,r.typname,r.attnotnull]),fields[table.relname],'fixed columns');if(c.some(c=>c.has_default||c.attidentity||c.attgenerated||c.column_acl||c.type_namespace!=='pg_catalog'))throw Error('Registry expression/default/column ACL/type refused')} + const acl=(await client.query(`SELECT c.relname,grantee.rolname AS grantee,grantor.rolname AS grantor,a.privilege_type,a.is_grantable + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + CROSS JOIN LATERAL pg_catalog.aclexplode(c.relacl) a LEFT JOIN pg_catalog.pg_roles grantee ON grantee.oid=a.grantee + JOIN pg_catalog.pg_roles grantor ON grantor.oid=a.grantor WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) + ORDER BY c.relname,grantee.rolname,a.privilege_type`,[[CURRENT,HISTORY]])).rows + const expected:AclRow[]=[] + for(const table of [CURRENT,HISTORY] as const)for(const [grantee,permissions]of [[scopes.control,['DELETE','INSERT','MAINTAIN','REFERENCES','SELECT','TRIGGER','TRUNCATE','UPDATE']],[OPERATOR,['SELECT']],[PUBLISHER,table===CURRENT?['INSERT','SELECT','UPDATE']:['INSERT','SELECT']]] as [string,string[]][])for(const privilege of permissions)expected.push({relname:table,grantee,grantor:scopes.control,privilege_type:privilege,is_grantable:false}) + if(runtimeReads)for(const table of [CURRENT,HISTORY] as const)expected.push({relname:table,grantee:scopes.runtime,grantor:scopes.control,privilege_type:'SELECT',is_grantable:false}) + const order=(x:readonly T[])=>x.slice().sort((a,b)=>canonical(a).localeCompare(canonical(b)));same(order(acl),order(expected),'append-only effective grant boundary') + const request=`SELECT c.relname,k.contype,k.convalidated,k.condeferrable,k.condeferred,k.confmatchtype, + k.conkey::int[] AS keys,ref.relname AS referenced,k.confkey::int[] AS referenced_keys,pg_catalog.pg_get_constraintdef(k.oid) AS definition + FROM pg_catalog.pg_constraint k JOIN pg_catalog.pg_class c ON c.oid=k.conrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_catalog.pg_class ref ON ref.oid=k.confrelid WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) AND k.contype<>'n' + ORDER BY c.relname,k.contype,k.conname` + const prior=(await client.query<{path:string}>("SELECT pg_catalog.current_setting('search_path') AS path")).rows[0].path + let constraints:ConstraintRow[],primary:unknown,failed=false + try{constraints=(await client.query(constraintSql(request),[[CURRENT,HISTORY]])).rows}catch(error){primary=error;failed=true;throw error}finally{try{await client.query("SELECT pg_catalog.set_config('search_path',$1,true)",[prior])}catch(error){if(failed)throw new AggregateError([primary,error],'Registry query and LOCAL context restoration failed');throw error}} + if(constraints.some(c=>!c.convalidated||c.condeferrable||c.condeferred))throw Error('Unvalidated/deferred registry constraint refused') + for(const table of [CURRENT,HISTORY] as const){ + const c=constraints.filter(c=>c.relname===table),kinds=c.map(c=>c.contype).sort().join('') + if(kinds!==(table===CURRENT?'cfp':'ccccccfpu'))throw Error('Exact registry constraint set required') + const fk=c.find(c=>c.contype==='f')!;same(fk.keys,table===CURRENT?[2,3,4]:[4,5,6],'predecessor FK columns');same(fk.referenced_keys,[1,2,3],'predecessor FK target columns');if(fk.referenced!==HISTORY||fk.confmatchtype!==(table===CURRENT?'s':'f'))throw Error('Exact predecessor/current FK required') + same(fk.definition,table===CURRENT?'FOREIGN KEY (installation_id, epoch, head_sha256) REFERENCES mmapp_control.application_writer_profile_history(installation_id, epoch, head_sha256)':'FOREIGN KEY (previous_installation_id, previous_epoch, previous_head_sha256) REFERENCES mmapp_control.application_writer_profile_history(installation_id, epoch, head_sha256) MATCH FULL','fixed FK actions/definition') + const pk=c.find(c=>c.contype==='p')!;same(pk.keys,table===CURRENT?[1]:[1,2],'registry primary key') + same(pk.definition,table===CURRENT?'PRIMARY KEY (id)':'PRIMARY KEY (installation_id, epoch)','fixed PK definition') + if(table===HISTORY){same(c.find(c=>c.contype==='u')!.keys,[1,2,3],'registry head unique key');same(c.find(c=>c.contype==='u')!.definition,'UNIQUE (installation_id, epoch, head_sha256)','fixed unique definition')} + } + const checks:Record={ + [CURRENT]:['CHECK ((id = 1))'], + [HISTORY]:[ + 'CHECK ((epoch > 0))', + "CHECK ((head_sha256 ~ '^[a-f0-9]{64}$'::text))", + "CHECK ((policy_sha256 ~ '^[a-f0-9]{64}$'::text))", + 'CHECK (((octet_length(policy_bytes) >= 1) AND (octet_length(policy_bytes) <= 1048576)))', + 'CHECK (((octet_length(head_bytes) >= 1) AND (octet_length(head_bytes) <= 8192)))', + 'CHECK ((((epoch = 1) AND (previous_installation_id IS NULL) AND (previous_epoch IS NULL) AND (previous_head_sha256 IS NULL)) OR ((epoch > 1) AND (previous_installation_id IS NOT NULL) AND (previous_installation_id = installation_id) AND (previous_epoch IS NOT NULL) AND (previous_epoch = (epoch - 1)) AND (previous_head_sha256 IS NOT NULL))))', + ], + } + for(const table of [CURRENT,HISTORY] as const)same(constraints.filter(c=>c.relname===table&&c.contype==='c').map(c=>c.definition).sort(),checks[table].sort(),'fixed PG18 CHECK definitions') + const triggers=(await client.query(`SELECT c.relname,t.tgisinternal,t.tgenabled,nf.nspname,p.proname FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace JOIN pg_catalog.pg_proc p ON p.oid=t.tgfoid JOIN pg_catalog.pg_namespace nf ON nf.oid=p.pronamespace WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) ORDER BY c.relname,p.proname`,[[CURRENT,HISTORY]])).rows + if(triggers.filter(t=>t.relname===CURRENT).length!==2||triggers.filter(t=>t.relname===HISTORY).length!==6||triggers.some(t=>!t.tgisinternal||t.tgenabled!=='O'||t.nspname!=='pg_catalog'||!['RI_FKey_check_ins','RI_FKey_check_upd','RI_FKey_noaction_del','RI_FKey_noaction_upd'].includes(t.proname)))throw Error('Protected registry FK triggers refused') + const indexes=(await client.query(`SELECT c.relname,i.indisunique,i.indisprimary,i.indisvalid,i.indisready,i.indislive,i.indkey::int[] AS keys,i.indpred IS NULL AS unconditional,i.indexprs IS NULL AS simple,a.amname FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class c ON c.oid=i.indrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace JOIN pg_catalog.pg_class x ON x.oid=i.indexrelid JOIN pg_catalog.pg_am a ON a.oid=x.relam WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) ORDER BY c.relname,i.indisprimary`,[[CURRENT,HISTORY]])).rows + if(indexes.length!==3||indexes.some(i=>!i.indisunique||!i.indisvalid||!i.indisready||!i.indislive||!i.unconditional||!i.simple||i.amname!=='btree'))throw Error('Protected registry indexed access refused') + same(order(indexes.map(i=>({table:i.relname,primary:i.indisprimary,keys:i.keys}))),order([{table:CURRENT,primary:true,keys:[1]},{table:HISTORY,primary:true,keys:[1,2]},{table:HISTORY,primary:false,keys:[1,2,3]}]),'exact registry indexes') + const structural={roles:roleRows,fileAcl,privilegedFunctions,tables:tables.map(({oid:_oid,...row})=>row),columns,acl:order(acl),constraints:order(constraints),triggers,indexes} + return {sha256:sha(canonical(structural)),roles:Object.freeze({...signedOids})} +} diff --git a/backend/src/db/bootstrap.ts b/backend/src/db/bootstrap.ts index 2ab8feb8..108348dc 100644 --- a/backend/src/db/bootstrap.ts +++ b/backend/src/db/bootstrap.ts @@ -1,42 +1,95 @@ -import { pool } from "./index.js" -import { applySqlMigrations } from "./migrate.js" +import { assertRuntimeDatabaseActivationCurrent, pool, readRuntimeDatabaseWriterHead } from "./index.js" +import { applySqlMigrations, assertSqlMigrationsApplied } from "./migrate.js" +import { withSystemDatabaseOperationLock } from "./operation-lock.js" import { dropExpiredPartitions, ensurePartitionsAround } from "./partitions.js" import { importSqliteToPostgres, shouldImportSqlite, sqliteFileLooksPresent } from "./sqlite-import.js" import { env } from "../config.js" +import { mcpStateDirectory } from "../modules/mcp/security.js" +import { assertDatabaseMaintenanceStartup, assertDatabaseRestoreJournalCurrent } from "../services/database-maintenance-startup.js" +import { reconcileDatabaseRestoreAudits } from "../services/database-restore-audit.js" +import { assertDatabaseRestoreQuarantineOpen, databaseRestoreQuarantineEnabled } from "./database-restore-quarantine.js" +import { DatabaseRestoreCheckpointRefused, readDatabaseRestoreCheckpoint, restoreJournalHeadsEqual, restoreWriterHeadIsCurrent } from "../services/database-restore-checkpoint.js" const ETL_LOCK = 8723101 -export async function initDatabase(): Promise { - await pool.query("SELECT 1") - await applySqlMigrations(pool) - await ensurePartitionsAround(pool) - const client = await pool.connect() - try { - await client.query("SELECT pg_advisory_lock($1)", [ETL_LOCK]) - if (await shouldImportSqlite(pool, env.DATABASE_PATH)) { - console.log(`SQLite → PostgreSQL: импорт ${env.DATABASE_PATH}`) - const report = await importSqliteToPostgres(pool, env.DATABASE_PATH, { strict: true }) - console.log( - `SQLite → PostgreSQL: готово за ${report.durationMs}ms, таблиц ${Object.keys(report.tables).length}`, - ) - } else { - const marker = await pool.query<{ sqlite_imported_at: string | null }>( - `SELECT sqlite_imported_at FROM data_migration WHERE id = 1`, - ) - if (!marker.rows[0]?.sqlite_imported_at && !sqliteFileLooksPresent(env.DATABASE_PATH)) { - console.warn( - `SQLite → PostgreSQL: файл ${env.DATABASE_PATH} не найден, база после wipe остаётся пустой (defaults settings)`, - ) +export async function initDatabase(mode: "startup" | "operator-partitions" = "startup"): Promise { + const schemaSha256 = process.env.DATABASE_RUNTIME_SCHEMA_SHA256 + const sourceConfigured = process.env.DATABASE_RUNTIME_REQUEST_FILE !== undefined + if ((process.env.DATABASE_RUNTIME_WRITER_APPROVAL_FILE !== undefined || process.env.DATABASE_RUNTIME_WRITER_APPROVAL_SHA256 !== undefined) && + (!schemaSha256 || !databaseRestoreQuarantineEnabled() || !process.env.DATABASE_RESTORE_CHECKPOINT_FILE)) { + throw new Error("Защищённый профиль записи требует утверждённую схему и контрольную точку восстановления PostgreSQL") + } + if (mode === "operator-partitions" && sourceConfigured) { + throw new Error("Обслуживание партиций требует отдельного процесса оператора PostgreSQL") + } + if (schemaSha256 !== undefined && !sourceConfigured) throw new Error("Проверка схемы требует утверждённого источника PostgreSQL") + const stateDirectory = mcpStateDirectory() + await assertDatabaseMaintenanceStartup(stateDirectory, env.DATABASE_URL) + await withSystemDatabaseOperationLock(() => pool.connect(), async (client, discard) => { + // A restore may have recorded an uncertain outcome while startup waited for the lock. + await assertDatabaseMaintenanceStartup(stateDirectory, env.DATABASE_URL) + const checkpoint = await readDatabaseRestoreCheckpoint(stateDirectory, env.DATABASE_URL) + if (checkpoint && (!databaseRestoreQuarantineEnabled() || checkpoint.status !== "sealed")) throw new DatabaseRestoreCheckpointRefused() + if (databaseRestoreQuarantineEnabled()) { + await assertDatabaseRestoreQuarantineOpen(client) + const head = await assertDatabaseRestoreJournalCurrent(client, stateDirectory) + if (checkpoint && (!restoreJournalHeadsEqual(checkpoint.head, head) || + !restoreWriterHeadIsCurrent(checkpoint.writerHead, await readRuntimeDatabaseWriterHead(client)))) { + throw new DatabaseRestoreCheckpointRefused() } } - } finally { - try { - await client.query("SELECT pg_advisory_unlock($1)", [ETL_LOCK]) - } catch { - /* ignore */ + if (mode === "operator-partitions") { + await assertSqlMigrationsApplied(client) + const around = new Date() + await dropExpiredPartitions(pool, around) + await ensurePartitionsAround(pool, around) + return } - client.release() - } - await dropExpiredPartitions(pool) - await ensurePartitionsAround(pool) + if (sourceConfigured) { + assertRuntimeDatabaseActivationCurrent() + if (schemaSha256 !== undefined) { + const { captureRuntimeDatabaseSchema } = await import("./database-runtime-schema.js") + await captureRuntimeDatabaseSchema(client, discard, schemaSha256) + assertRuntimeDatabaseActivationCurrent() + } else await assertSqlMigrationsApplied(client) + if (await shouldImportSqlite(pool, env.DATABASE_PATH)) { + throw new Error("Перед запуском приложения оператор должен завершить импорт SQLite") + } + await ensurePartitionsAround(pool) + await reconcileDatabaseRestoreAudits(pool, stateDirectory) + assertRuntimeDatabaseActivationCurrent() + return + } + await applySqlMigrations(pool) + await ensurePartitionsAround(pool) + try { + await client.query("SELECT pg_advisory_lock($1)", [ETL_LOCK]) + if (await shouldImportSqlite(pool, env.DATABASE_PATH)) { + console.log(`SQLite → PostgreSQL: импорт ${env.DATABASE_PATH}`) + const report = await importSqliteToPostgres(pool, env.DATABASE_PATH, { strict: true }) + console.log( + `SQLite → PostgreSQL: готово за ${report.durationMs}ms, таблиц ${Object.keys(report.tables).length}`, + ) + } else { + const marker = await pool.query<{ sqlite_imported_at: string | null }>( + `SELECT sqlite_imported_at FROM data_migration WHERE id = 1`, + ) + if (!marker.rows[0]?.sqlite_imported_at && !sqliteFileLooksPresent(env.DATABASE_PATH)) { + console.warn( + `SQLite → PostgreSQL: файл ${env.DATABASE_PATH} не найден, база после wipe остаётся пустой (defaults settings)`, + ) + } + } + } finally { + try { + const result = await client.query("SELECT pg_advisory_unlock($1) AS released", [ETL_LOCK]) + if (result.rows[0]?.released !== true) discard() + } catch { + discard() + } + } + await dropExpiredPartitions(pool) + await ensurePartitionsAround(pool) + await reconcileDatabaseRestoreAudits(pool, stateDirectory) + }) } diff --git a/backend/src/db/controlled-migration-history.ts b/backend/src/db/controlled-migration-history.ts new file mode 100644 index 00000000..0efd5a5e --- /dev/null +++ b/backend/src/db/controlled-migration-history.ts @@ -0,0 +1,75 @@ +import { createHash } from "node:crypto" +import { checkDatabaseSchemaControlState, type DatabaseSchemaIntent, type DatabaseSchemaTerminal } from "./database-schema-control-model.js" +import { MIGRATION_MANIFEST, type MigrationReview } from "./migration-manifest.js" +import { fingerprint, privateFingerprinterExisting, privateSecretFingerprint } from "../modules/mcp/security.js" + +const refuse = (reason: string): never => { throw new Error(reason) } +const digest = (value: string) => createHash("sha256").update(value).digest("hex") +export const REVIEWED_PREDECESSORS = [ + { length: 44, manifestSha256: "fffae3381732e6f276ba276aca7185f6a87587edf8272ba42763b7f49bc59ef5" }, + { length: 56, manifestSha256: "b371d8d261f7dd69cf0e2ebc48d4316e3ad804360046bbaaf20e715da8812521" }, +] as const +export const REVIEWED_SUCCESSOR_MANIFEST = "7dbd678e59740c14a29594180c86b85b3b2dd53fd5aa7a72da6d14a513760dbc" +export const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item) +export async function observation(intent: DatabaseSchemaIntent, pin: MigrationReview, existingKey = false) { + // Fixed, purpose-separated server-keyed ACK issuer. Caller-supplied hashes + // cannot mint a prefix receipt. This still does not attest that current + // schema/data is unchanged, and never issues a READY capability. + const value = fingerprint({ format: "controlled-migration-file-ack-v1", intent, file: pin.file, transaction: pin.transaction, selfMarker: pin.selfMarker, fileCommitAck: true, markerAck: true }) + const domain = "database-controlled-migration-file-ack-v1" + return existingKey ? (await privateFingerprinterExisting(domain))(value) : privateSecretFingerprint(value, domain) +} +export async function legacy0037AdoptionObservation(intent: DatabaseSchemaIntent) { + const migration = MIGRATION_MANIFEST[36] + if (migration.file !== "0037_server_probe_storage_v2.sql") return refuse("Reviewed legacy 0037 migration required") + const proof = await privateFingerprinterExisting("database-controlled-legacy-0037-adoption-v1") + return proof(fingerprint({ intent, migration: migration.file, fileSha256: migration.sha256, quarantineMatched: true, committedHandoffObserved: true })) +} +export async function legacy0037ReleaseObservation(terminal: DatabaseSchemaTerminal, approvalSha256: string) { + const proof = await privateFingerprinterExisting("database-controlled-legacy-0037-release-v1") + return proof(fingerprint({ intent: terminal.intent, adoptionSha256: terminal.observationSha256, approvalSha256 })) +} +export async function exactHistory(rows: Record[], current: ReturnType, manifest: string, existingKey = false) { + if (rows.length > MIGRATION_MANIFEST.length || current.intent !== null) refuse("Unresolved or unexpected migration history") + const completed = new Map(), ids = new Set() + const known = manifest === REVIEWED_SUCCESSOR_MANIFEST + ? REVIEWED_PREDECESSORS.filter(stage => digest(JSON.stringify(MIGRATION_MANIFEST.slice(0, stage.length))) === stage.manifestSha256) + : [] + const inherited = new Set() + for (const row of rows) { + if (Object.keys(row).sort().join(" ") !== "id intent terminal" || typeof row.id !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(row.id) || ids.has(row.id)) refuse("Invalid migration history envelope") + ids.add(row.id as string) + // Reuse the strict persisted terminal parser; no boolean/hash alone is a + // readiness grant. A fabricated newer epoch is rejected here as well. + const parsed = checkDatabaseSchemaControlState({ format: current.format, epoch: current.epoch, owner: current.owner, phase: "TERMINAL", intent: null, terminal: row.terminal }) + const terminal = parsed.terminal! + if (canonical(row.intent) !== canonical(terminal.intent) || terminal.intent.id !== row.id || terminal.intent.kind !== "migration" || terminal.outcome !== "observed-success") refuse("Unknown, failed, foreign or mismatched migration receipt") + const receiptManifest = terminal.intent.manifestSha256 + if (receiptManifest !== manifest) { + if (!known.some(stage => stage.manifestSha256 === receiptManifest)) refuse("Unknown migration manifest or predecessor file") + inherited.add(receiptManifest) + } + let pin = MIGRATION_MANIFEST.find(entry => entry.sha256 === terminal.intent.artifactSha256) + const release = terminal.legacyRelease + if (release && (pin || release.observationSha256 !== await legacy0037ReleaseObservation(terminal, release.approvalSha256))) { + return refuse("Migration release is not the exact internally recorded legacy release") + } + if (pin ? terminal.observationSha256 !== await observation(terminal.intent, pin, existingKey) : + terminal.intent.manifestSha256 !== manifest || terminal.observationSha256 !== await legacy0037AdoptionObservation(terminal.intent)) { + return refuse("Migration receipt is not the exact internally recorded file ACK") + } + pin ??= MIGRATION_MANIFEST[36] + if (completed.has(pin.file)) return refuse("Duplicate migration receipt") + completed.set(pin.file, terminal) + } + const prefix = MIGRATION_MANIFEST.slice(0, completed.size) + if (prefix.some(pin => !completed.has(pin.file))) refuse("Migration history is not an exact completed prefix") + const stages = known.filter(stage => inherited.has(stage.manifestSha256)) + if (stages.some(stage => stage.length > prefix.length) || prefix.some((pin, index) => + completed.get(pin.file)!.intent.manifestSha256 !== (stages.find(stage => index < stage.length)?.manifestSha256 ?? manifest))) { + refuse("Reviewed predecessor migration history must be complete and ordered") + } + const last = prefix.length ? completed.get(prefix[prefix.length - 1].file) : null + if (canonical(last) !== canonical(current.terminal)) refuse("Current control terminal disagrees with completed prefix") + return prefix.length +} diff --git a/backend/src/db/controlled-migrations.sql.test.ts b/backend/src/db/controlled-migrations.sql.test.ts new file mode 100644 index 00000000..842766b7 --- /dev/null +++ b/backend/src/db/controlled-migrations.sql.test.ts @@ -0,0 +1,107 @@ +import assert from "node:assert/strict" +import { createHash, randomUUID } from "node:crypto" +import { basename, dirname, resolve } from "node:path" +assert.equal(process.env.MMAPP_OWN_CONTROLLED_MIGRATIONS, "true") +const url = new URL(process.env.DATABASE_URL ?? "") +assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const { Pool } = await import("pg") +const { pool, closePool } = await import("./index.js") +const { applySqlMigrations } = await import("./migrate.js") +const { executeControlledSqlMigrations } = await import("./controlled-migrations.js") +const { initialDatabaseSchemaControlState } = await import("./database-schema-control-model.js") +const { MIGRATION_MANIFEST } = await import("./migration-manifest.js") +const { SYSTEM_DATABASE_OPERATION_LOCK, withOperationLock } = await import("./operation-lock.js") +const originalConnect = pool.connect.bind(pool), databases: InstanceType[] = [], names: string[] = [], cases: string[] = [] +const hashes = new Set(MIGRATION_MANIFEST.map(pin => pin.sha256)) +async function database(seed = true) { + const name = "mm_dispatch_" + randomUUID().replaceAll("-", "") + assert(/^mm_dispatch_[a-f0-9]{32}$/.test(name)) + await pool.query('CREATE DATABASE "' + name + '" TEMPLATE template0') + const target = new URL(url); target.pathname = "/" + name + const connection = new Pool({ connectionString: target.toString(), max: 6 }); databases.push(connection); names.push(name) + if (seed) { + await connection.query("CREATE SCHEMA mmapp_control; CREATE TABLE mmapp_control.database_schema_state(id smallint PRIMARY KEY CHECK(id=1),revision bigint NOT NULL CHECK(revision>=0),state jsonb NOT NULL); CREATE TABLE mmapp_control.database_schema_intents(id uuid PRIMARY KEY,intent jsonb NOT NULL,terminal jsonb)") + await connection.query("INSERT INTO mmapp_control.database_schema_state VALUES(1,0,$1::jsonb)", [JSON.stringify(initialDatabaseSchemaControlState())]) + } + return connection +} +type Fault = "none" | "application-commit" | "terminal-commit" +const requestAdmissionSql = "SELECT pg_catalog.pg_try_advisory_xact_lock_shared($1::bigint) AS acquired" +async function run(connection: InstanceType, fault: Fault = "none") { + const files: string[] = []; let ordinal = 0, controlCommits = 0 + // Owned fixture instrumentation at the fixed module's actual Pool boundary. + // Every SQL statement still executes on a real checked-out private backend. + pool.connect = (async () => { + const client = await connection.connect(), index = ++ordinal, originalQuery = client.query, release = client.release.bind(client), query = originalQuery.bind(client) + client.release = (destroy?: boolean) => { client.query = originalQuery; release(destroy) } + client.query = (async (text: string, values?: unknown[]) => { + const digest = createHash("sha256").update(text).digest("hex") + if (hashes.has(digest)) { + files.push(digest) + if (files.length === 1 || files.length === MIGRATION_MANIFEST.length) { + const { rows } = await connection.query(requestAdmissionSql, [SYSTEM_DATABASE_OPERATION_LOCK]) + assert.equal(rows[0]?.acquired, false, "Migration SQL must exclude MCP request writes") + } + } + const result = await query(text, values) + if (index === 2 && text === "COMMIT" && fault === "application-commit") throw Error("Owned application COMMIT ACK lost") + if (index === 1 && text === "COMMIT" && ++controlCommits === 4 && fault === "terminal-commit") throw Error("Owned terminal publication COMMIT ACK lost") + return result + }) as typeof client.query + return client + }) as typeof pool.connect + try { return { result: await executeControlledSqlMigrations(), files, failed: false } } + catch { return { files, failed: true } } + finally { pool.connect = originalConnect as typeof pool.connect } +} +try { + await applySqlMigrations(pool) + const missing = await database(false), noControl = await run(missing) + assert(noControl.failed); assert.equal(noControl.files.length, 0) + cases.push("missing external control refuses before application migration SQL") + const clean = await database() + await withOperationLock(() => clean.connect(), SYSTEM_DATABASE_OPERATION_LOCK, async () => { + const blocked = await run(clean) + assert(blocked.failed) + assert.deepEqual(blocked.files, []) + assert.equal((await clean.query("SELECT count(*)::int AS n FROM mmapp_control.database_schema_intents")).rows[0].n, 0) + }) + cases.push("live backup/restore lease refuses controlled migration before DDL or intent") + const initial = await run(clean) + assert(!initial.failed); assert.equal(initial.result?.completedFiles, MIGRATION_MANIFEST.length); assert.equal(initial.result?.ready, false) + assert.deepEqual(initial.files, MIGRATION_MANIFEST.map(pin => pin.sha256)) + assert.equal((await clean.query(requestAdmissionSql, [SYSTEM_DATABASE_OPERATION_LOCK])).rows[0].acquired, true) + assert.equal((await clean.query("SELECT count(*)::int AS n FROM mmapp_control.database_schema_intents WHERE terminal->>'outcome'='observed-success'")).rows[0].n, MIGRATION_MANIFEST.length) + cases.push("all current exact pinned files execute once with genuine keyed ACK receipts; READY remains closed") + const repeated = await run(clean); assert(!repeated.failed); assert.equal(repeated.files.length, 0) + cases.push("fresh owner verifies complete known prefix and executes no file twice") + const legacy = await database(); await legacy.query("CREATE TABLE public.unmarked(id integer)") + const refused = await run(legacy); assert(refused.failed); assert.equal(refused.files.length, 0) + cases.push("unmarked legacy application objects are not treated as a fresh database") + const unknown = await database(), lostFile = await run(unknown, "application-commit") + assert(lostFile.failed); assert.equal(lostFile.files.length, 1) + assert.equal((await unknown.query("SELECT terminal->>'outcome' AS outcome FROM mmapp_control.database_schema_intents")).rows[0].outcome, "interrupted-unknown") + const unknownRepeat = await run(unknown); assert(unknownRepeat.failed); assert.equal(unknownRepeat.files.length, 0) + cases.push("actual application COMMIT with lost ACK becomes unknown and is never replayed by fresh owner") + const prefix = await database(), lostTerminal = await run(prefix, "terminal-commit") + assert(lostTerminal.failed); assert.equal(lostTerminal.files.length, 1) + assert.equal((await prefix.query("SELECT terminal->>'outcome' AS outcome FROM mmapp_control.database_schema_intents")).rows[0].outcome, "observed-success") + const resumed = await run(prefix); assert(!resumed.failed); assert.equal(resumed.files.length, MIGRATION_MANIFEST.length - 1) + assert.deepEqual(resumed.files, MIGRATION_MANIFEST.slice(1).map(pin => pin.sha256)) + cases.push("lost terminal publication ACK preserves genuine known file ACK; fresh invocation resumes only remaining manifest files without replay") + const badMarker = await database(), markerPrefix = await run(badMarker, "terminal-commit"); assert(markerPrefix.failed) + await badMarker.query("DELETE FROM public.schema_migrations") + const markerDrift = await run(badMarker); assert(markerDrift.failed); assert.equal(markerDrift.files.length, 0) + cases.push("marker and protected history mismatch refuses before any next file") + const tampered = await database(), tamperPrefix = await run(tampered, "terminal-commit"); assert(tamperPrefix.failed) + await tampered.query("UPDATE mmapp_control.database_schema_intents SET terminal=jsonb_set(terminal,'{observationSha256}',to_jsonb($1::text))", ["f".repeat(64)]) + await tampered.query("UPDATE mmapp_control.database_schema_state SET state=jsonb_set(state,'{terminal,observationSha256}',to_jsonb($1::text))", ["f".repeat(64)]) + const forged = await run(tampered); assert(forged.failed); assert.equal(forged.files.length, 0) + cases.push("matching singleton/history plain-hash forgery cannot replace fixed server-keyed file receipt") + console.log("Owned controlled migration SQL:", JSON.stringify({ cases: cases.length, names: cases, databases: names.length, actualSql: true, fixedSourceFiles: MIGRATION_MANIFEST.length, fullRuns: 2, actualApplicationAckLosses: 1, actualTerminalPublicationAckLosses: 3, readyAuthorized: false, startupWired: false, productionDatabaseTouched: false })) +} finally { + pool.connect = originalConnect as typeof pool.connect + for (const connection of databases) await connection.end() + await closePool() +} diff --git a/backend/src/db/controlled-migrations.test.ts b/backend/src/db/controlled-migrations.test.ts new file mode 100644 index 00000000..4bf09492 --- /dev/null +++ b/backend/src/db/controlled-migrations.test.ts @@ -0,0 +1,322 @@ +import assert from "node:assert/strict" +import { createHash, createHmac, randomUUID } from "node:crypto" +import * as fs from "node:fs/promises" +import { test } from "node:test" +import { runInNewContext } from "node:vm" +import ts from "typescript" +import * as manifest from "./migration-manifest.js" +import * as model from "./database-schema-control-model.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js" +import * as operationLock from "./operation-lock.js" +const { SYSTEM_DATABASE_OPERATION_LOCK, SYSTEM_DATABASE_REQUEST_LOCK } = operationLock + +const pins = manifest.MIGRATION_MANIFEST +const sources = await Promise.all(pins.map(async pin => ({ pin, source: await fs.readFile(new URL("../../drizzle/" + pin.file, import.meta.url), "utf8") }))) +const originalSource = await fs.readFile(new URL("./controlled-migrations.ts", import.meta.url), "utf8") +const compiled = ts.transpileModule(originalSource.replaceAll("import.meta.url", JSON.stringify(new URL("./controlled-migrations.ts", import.meta.url).href)), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, esModuleInterop: true } }).outputText +const compiledHistory = ts.transpileModule(await fs.readFile(new URL("./controlled-migration-history.ts", import.meta.url), "utf8"), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item) +const acknowledgementDomain = "database-controlled-migration-file-ack-v1" +const fixtureFingerprint = (value: unknown) => createHash("sha256").update(canonical(value)).digest("hex") +const fixtureMac = (value: string, domain: string) => createHmac("sha256", "inert-only-issuer:" + domain).update(value).digest("hex") + +/** Execute actual production dispatcher source with inert module I/O. These + * protocol tests do not claim PostgreSQL DDL execution or cryptographic key + * provenance; root's private PG suite independently covers actual SQL. */ +function fixture() { + let state = model.initialDatabaseSchemaControlState(), revision = 0, publicPresent = false, markerIds = new Set() + const history = new Map() + const events: string[] = [], appReleases: boolean[] = [], reads = new Map() + const faults = { beginFile: "", file: "", marker: "", terminal: false, lostCommit: false, rollbackTag: false, driftFile: "", extraFile: false, forgeMarker: false, + guarded: false, guardDrift: false, removeGuardAfterFile: false } + let activeLease = false + const control = { acquireDatabaseSchemaControl: async () => { + if (activeLease) throw Error("live control lease") + activeLease = true; state = model.claimDatabaseSchemaOwnership(state, randomUUID()); revision++ + let closed = false + return { + read: async () => { if (closed) throw Error("closed"); return { revision: String(revision), state: structuredClone(state) } }, + transition: async (command: model.DatabaseSchemaControlCommand) => { + if (closed) throw Error("closed") + if (command.type === "begin") { + const pin = pins.find(pin => pin.sha256 === command.artifactSha256)! + if (faults.beginFile === pin.file) { closed = true; activeLease = false; throw Error("intent unavailable BEFORE publication") } + events.push("intent:" + pin.file) + state = model.transitionDatabaseSchemaControl(state, state.owner!, structuredClone(command)) + history.set(command.id, { id: command.id, intent: structuredClone(state.intent!), terminal: null }) + } else { + const row = history.get(command.id)! + events.push("terminal:" + row.intent.artifactSha256 + ":" + command.outcome) + state = model.transitionDatabaseSchemaControl(state, state.owner!, structuredClone(command)) + row.terminal = structuredClone(state.terminal) + if (faults.terminal) { closed = true; activeLease = false; throw Error("terminal ACK lost") } + } + revision++; return { revision: String(revision), state: structuredClone(state) } + }, + close: async () => { closed = true; activeLease = false }, + } + } } + // These exact strings are read from the actual module exports after load. + const pool = { connect: async () => { + let outerMarkers: Set | undefined, outerPublic: boolean | undefined + return { + query: async (query: string, values: unknown[] = []) => { + if (query === sql.history) return { rows: [...history.values()].map(row => structuredClone(row)) } + if (query === sql.namespaces) return { rows: [{ name: "mmapp_control" }, { name: "public" }] } + if (query === sql.publicObjects) return { rows: [{ present: publicPresent }] } + if (query === sql.markerTable) return { rows: publicPresent ? [{ kind: "r", persistence: "p", rls: false, force_rls: false, user_triggers: faults.guarded }] : [] } + if (query === sql.markers) return { rows: [...markerIds].sort().map(id => ({ id })) } + if (query === sql.marker) { + const id = String(values[0]); events.push("marker:" + id) + if (faults.marker === id) throw Error("marker ACK unavailable") + if (markerIds.has(id)) throw Error("duplicate marker") + markerIds.add(id); return { rows: [] } + } + if (query === "BEGIN") { assert(!outerMarkers); outerMarkers = new Set(markerIds); outerPublic = publicPresent; events.push("BEGIN"); return { rows: [] } } + if (query === "COMMIT") { assert(outerMarkers); outerMarkers = undefined; outerPublic = undefined; events.push("COMMIT"); if (faults.lostCommit) throw Error("application COMMIT ACK lost"); return { rows: [], command: faults.rollbackTag ? "ROLLBACK" : "COMMIT" } } + if (query.startsWith("SET ") || query.startsWith("RESET ")) { events.push(query); return { rows: [] } } + if (query.includes("pg_catalog.pg_try_advisory_lock(")) { assert.deepEqual(Array.from(values), [SYSTEM_DATABASE_OPERATION_LOCK]); events.push("maintenance lease"); return { rows: [{ acquired: true }] } } + if (query.includes("pg_catalog.pg_advisory_lock(")) { + assert([PARTITION_WRITER_ADVISORY_KEY, SYSTEM_DATABASE_REQUEST_LOCK].includes(Number(values[0]))) + events.push(values[0] === SYSTEM_DATABASE_REQUEST_LOCK ? "request drain" : "exclusive lease") + return { rows: [] } + } + if (query.includes("pg_catalog.pg_advisory_unlock(")) { events.push("unlock"); return { rows: [{ unlocked: true, released: true }] } } + const exact = sources.find(entry => entry.source === query) + assert(exact, "Only original pinned file bytes and fixed internal SQL may execute") + events.push("file:" + exact.pin.file) + if (faults.file === exact.pin.file) throw Error("file ACK unknown") + assert.equal(Boolean(outerMarkers), exact.pin.transaction === "owner-transaction") + publicPresent = true + if (exact.pin.selfMarker) markerIds.add(exact.pin.file.slice(0, -4)) + if (faults.removeGuardAfterFile) faults.guarded = false + if (faults.forgeMarker) markerIds.add("9999_unknown") + return exact.pin.transaction === "self-managed-single-transaction" ? [{ command: "BEGIN", rows: [] }, { command: "COMMIT", rows: [] }] : { rows: [] } + }, + release: (destroy: boolean) => { appReleases.push(destroy); if (destroy && outerMarkers) { markerIds = outerMarkers; publicPresent = outerPublic! } }, + } + } } + const modules: Record = { + "node:crypto": { createHash, randomUUID }, + "node:fs/promises": { + readdir: async (...args: Parameters) => { + const rows = await Reflect.apply(fs.readdir, fs, args) as unknown as { name: string; isFile(): boolean }[] + return faults.extraFile ? [...rows, { name: "9999_unreviewed.sql", isFile: () => true }] : rows + }, + readFile: async (file: URL) => { const name = file.pathname.split("/").at(-1)!; reads.set(name, (reads.get(name) ?? 0) + 1); const bytes = await fs.readFile(file); return faults.driftFile === name ? Buffer.concat([bytes, Buffer.from("\n")]) : bytes }, + }, + "./database-schema-control.js": control, + "./database-restore-quarantine.js": { assertDatabaseRestoreQuarantineOpen: async () => {} }, + "./database-schema-control-model.js": { ...model, checkDatabaseSchemaControlState: (value: unknown) => model.checkDatabaseSchemaControlState(structuredClone(value)) }, + "./migration-manifest.js": manifest, + "./partition-guard-inheritance.js": { assertReviewedMigrationMarkerGuard: async () => { + events.push("reviewed marker guards") + if (faults.guardDrift) throw Error("Marker guard definition changed") + } }, + "../services/partition-write-transaction.js": { PARTITION_WRITER_ADVISORY_KEY }, + "./operation-lock.js": operationLock, + "../modules/mcp/security.js": { + fingerprint: fixtureFingerprint, + privateFingerprinterExisting: async (domain: string) => (value: string) => fixtureMac(value, domain), + privateSecretFingerprint: async (value: string, domain: string) => fixtureMac(value, domain), + }, + "./index.js": { pool }, + } + const historyExports = {} + const dependencies = { ...modules, "./controlled-migration-history.js": historyExports } as Record + const globals = { require: (name: string) => { assert(name in dependencies, "Unapproved import: " + name); return dependencies[name] }, Buffer, URL, TextDecoder, Promise, Error, Map, Set, Object, structuredClone } + runInNewContext(compiledHistory, { ...globals, exports: historyExports }, { filename: "controlled-migration-history.ts" }) + const exports = {} as { executeControlledSqlMigrations(): Promise<{ completedFiles: number; ready: false }>; CONTROLLED_MIGRATIONS_SQL: Record } + runInNewContext(compiled, { ...globals, exports }, { filename: "controlled-migrations.ts" }) + const sql = exports.CONTROLLED_MIGRATIONS_SQL + function seedCompletedHistory(count: number, manifestForIndex: (index: number) => string) { + assert.equal(history.size, 0) + state = model.claimDatabaseSchemaOwnership(state, randomUUID()) + for (const [index, pin] of pins.slice(0, count).entries()) { + const id = randomUUID() + state = model.transitionDatabaseSchemaControl(state, state.owner!, { type: "begin", id, kind: "migration", manifestSha256: manifestForIndex(index), artifactSha256: pin.sha256 }) + const intent = state.intent! + const observationSha256 = fixtureMac(fixtureFingerprint({ format: "controlled-migration-file-ack-v1", intent, file: pin.file, + transaction: pin.transaction, selfMarker: pin.selfMarker, fileCommitAck: true, markerAck: true }), acknowledgementDomain) + state = model.transitionDatabaseSchemaControl(state, state.owner!, { type: "terminal", id, outcome: "observed-success", observationSha256 }) + history.set(id, structuredClone({ id, intent, terminal: state.terminal })) + markerIds.add(pin.file.slice(0, -4)) + } + publicPresent = count > 0 + } + return { events, faults, appReleases, reads, run: exports.executeControlledSqlMigrations, history, seedCompletedHistory, state: () => structuredClone(state), markerIds: () => new Set(markerIds), legacy: () => { publicPresent = true; markerIds.add("0000_postgresql") }, corruptMarker: () => markerIds.add("9999_unknown") } +} +test("actual dispatcher executes exact current files with durable intents and correct transaction contracts", async () => { + const f = fixture(), result = await f.run() + assert.equal(result.completedFiles, pins.length); assert.equal(result.ready, false) + assert.equal(f.events.filter(event => event.startsWith("file:")).length, pins.length) + assert.equal(f.events.filter(event => event === "BEGIN").length, 17) + assert.equal(f.events.filter(event => event === "COMMIT").length, 17) + assert.equal(f.events.filter(event => event.startsWith("marker:")).length, 34) + assert.equal(f.events.filter(event => event === "exclusive lease").length, 1) + assert(f.events.indexOf("maintenance lease") < f.events.indexOf("exclusive lease")) + assert(f.events.indexOf("request drain") < f.events.indexOf("maintenance lease")) + for (const pin of pins) { assert(f.events.indexOf("intent:" + pin.file) < f.events.indexOf("file:" + pin.file)); assert.equal(f.reads.get(pin.file), 1) } + assert.deepEqual(f.appReleases, [false]) + assert.equal(f.history.size, pins.length) +}) + +for (const previousLength of [39, 40, 41, 42, 43]) test("old controlled" + previousLength + " manifest history is refused by the current build", async () => { + const f = fixture(); f.faults.beginFile = pins[previousLength].file + await assert.rejects(f.run()); assert.equal(f.history.size, previousLength) + const oldDigest = createHash("sha256").update(JSON.stringify(pins.slice(0, previousLength))).digest("hex") + for (const row of f.history.values()) { + row.intent.manifestSha256 = oldDigest + row.terminal!.intent.manifestSha256 = oldDigest + } + f.faults.beginFile = ""; f.events.length = 0 + await assert.rejects(f.run()) + assert.equal(f.events.some(event => event.startsWith("file:")), false) + assert.equal(f.history.size, previousLength) +}) +test("known completed prefix resumes after fresh ownership without replaying earlier bytes", async () => { + const f = fixture(); f.faults.beginFile = pins[7].file + await assert.rejects(f.run()); assert.equal(f.history.size, 7) + f.faults.beginFile = ""; f.events.length = 0 + const result = await f.run() + assert.equal(result.completedFiles, pins.length) + assert.deepEqual(f.events.filter(event => event.startsWith("file:")), pins.slice(7).map(pin => "file:" + pin.file)) +}) + +const reviewed44 = "fffae3381732e6f276ba276aca7185f6a87587edf8272ba42763b7f49bc59ef5" +const reviewed56 = "b371d8d261f7dd69cf0e2ebc48d4316e3ad804360046bbaaf20e715da8812521" +test("reviewed 44 migration history keeps its original keyed receipts and resumes only the current tail", async () => { + const f = fixture() + f.seedCompletedHistory(46, index => index < 44 ? reviewed44 : manifest.migrationManifestDigest()) + const previous = structuredClone([...f.history.values()]) + const result = await f.run() + assert.equal(result.completedFiles, pins.length) + assert.equal(result.ready, false) + assert.deepEqual(f.events.filter(event => event.startsWith("file:")), pins.slice(46).map(pin => "file:" + pin.file)) + assert.deepEqual([...f.history.values()].slice(0, 46), previous) + f.events.length = 0 + assert.equal((await f.run()).completedFiles, pins.length) + assert(f.events.every(event => !event.startsWith("file:"))) +}) + +test("complete 56 receipts with or without the inherited 44 prefix dispatch only index 0057", async () => { + for (const mixed of [false, true]) { + const f = fixture() + f.seedCompletedHistory(56, index => mixed && index < 44 ? reviewed44 : reviewed56) + const previous = structuredClone([...f.history.values()]) + assert.equal((await f.run()).completedFiles, pins.length) + assert.deepEqual(f.events.filter(event => event.startsWith("file:")), ["file:0057_mcp_routing_charge_audit_index.sql"]) + assert.deepEqual([...f.history.values()].slice(0, 56), previous) + f.events.length = 0 + assert.equal((await f.run()).completedFiles, pins.length) + assert(!f.events.some(event => event.startsWith("file:"))) + } +}) + +test("history 56 cannot omit a receipt or interleave another manifest", async () => { + for (const invalid of ["incomplete", "interleaved"] as const) { + const f = fixture() + f.seedCompletedHistory(invalid === "incomplete" ? 55 : 56, index => + invalid === "interleaved" && index === 12 ? manifest.migrationManifestDigest() : reviewed56) + await assert.rejects(f.run(), (error: Error) => error.cause instanceof Error && /complete and ordered/.test(error.cause.message)) + assert(!f.events.some(event => event.startsWith("file:") || event.startsWith("intent:"))) + } +}) + +test("reviewed 44 migration history refuses incomplete, mixed, out-of-range and tampered receipts", async () => { + for (const fault of ["incomplete", "mixed", "out-of-range", "tampered"] as const) { + const f = fixture() + f.seedCompletedHistory(fault === "incomplete" ? 43 : fault === "out-of-range" ? 45 : 44, + index => fault === "mixed" && index === 23 ? manifest.migrationManifestDigest() : reviewed44) + if (fault === "tampered") f.history.values().next().value!.terminal!.observationSha256 = "f".repeat(64) + await assert.rejects(f.run()) + assert(f.events.every(event => !event.startsWith("file:") && !event.startsWith("intent:")), fault) + } +}) + +test("reviewed marker guards are rechecked after each file and a change stops the next intent", async () => { + const prepare = () => { + const f = fixture() + f.seedCompletedHistory(44, () => reviewed44) + f.faults.guarded = true + return f + } + const reviewed = prepare() + assert.equal((await reviewed.run()).completedFiles, pins.length) + assert.equal(reviewed.events.filter(event => event === "reviewed marker guards").length, 15) + const damaged = prepare() + damaged.faults.guardDrift = true + await assert.rejects(damaged.run()) + assert(!damaged.events.some(event => event.startsWith("intent:"))) + const removed = prepare() + removed.faults.removeGuardAfterFile = true + await assert.rejects(removed.run()) + assert.deepEqual(removed.events.filter(event => event.startsWith("file:")), ["file:" + pins[44].file]) + assert.equal(removed.state().terminal?.outcome, "interrupted-unknown") +}) +test("fully completed restart validates ledger/markers and executes zero application files", async () => { + const f = fixture(); await f.run(); f.events.length = 0 + assert.equal((await f.run()).completedFiles, pins.length) + assert.equal(f.events.filter(event => event.startsWith("file:")).length, 0) +}) +test("lost control terminal ACK reconciles signed durable success and skips that file", async () => { + const f = fixture(); f.faults.terminal = true + await assert.rejects(f.run()) + assert.equal(f.state().terminal?.outcome, "observed-success") + assert.equal(f.history.size, 1) + f.faults.terminal = false; f.events.length = 0 + assert.equal((await f.run()).completedFiles, pins.length) + assert.deepEqual(f.events.filter(event => event.startsWith("file:")), pins.slice(1).map(pin => "file:" + pin.file)) +}) +test("legacy markers or source drift/new files never dispatch application SQL", async () => { + for (const kind of ["legacy", "drift", "new"] as const) { + const f = fixture() + if (kind === "legacy") f.legacy() + if (kind === "drift") f.faults.driftFile = pins[0].file + if (kind === "new") f.faults.extraFile = true + await assert.rejects(f.run()) + assert.equal(f.events.filter(event => event.startsWith("file:")).length, 0) + assert.equal(f.history.size, 0) + } +}) +test("application failure or lost COMMIT ACK records unknown and forbids restart replay", async () => { + for (const kind of ["file", "commit"] as const) { + const f = fixture() + if (kind === "file") f.faults.file = pins[0].file; else f.faults.lostCommit = true + await assert.rejects(f.run()) + assert.equal(f.state().terminal?.outcome, "interrupted-unknown") + assert.equal(f.appReleases.at(-1), true) + f.faults.file = ""; f.faults.lostCommit = false; f.events.length = 0 + await assert.rejects(f.run()) + assert.equal(f.events.filter(event => event.startsWith("file:")).length, 0) + } +}) +test("self-managed external-marker gap remains unknown and never retries file", async () => { + const f = fixture(); f.faults.marker = pins[7].file.slice(0, -4) + await assert.rejects(f.run()) + assert.equal(f.state().terminal?.outcome, "interrupted-unknown") + assert.equal(f.events.filter(event => event === "file:" + pins[7].file).length, 1) + f.faults.marker = ""; f.events.length = 0 + await assert.rejects(f.run()); assert(!f.events.some(event => event.startsWith("file:"))) +}) +test("resolved ROLLBACK command tag never becomes a successful COMMIT receipt", async () => { + const f = fixture(); f.faults.rollbackTag = true + await assert.rejects(f.run()) + assert.equal(f.state().terminal?.outcome, "interrupted-unknown") + assert.equal(f.history.size, 1) +}) +test("marker inconsistency rejects before issuing a successful file receipt", async () => { + const f = fixture(); f.faults.forgeMarker = true + await assert.rejects(f.run()) + assert.equal(f.state().terminal?.outcome, "interrupted-unknown") +}) +test("caller-forged receipt hash or marker mismatch cannot authorize prefix skipping", async () => { + for (const kind of ["hash", "marker"] as const) { + const f = fixture(); f.faults.beginFile = pins[1].file + await assert.rejects(f.run()); f.faults.beginFile = "" + if (kind === "hash") f.history.values().next().value!.terminal!.observationSha256 = "a".repeat(64) + else f.corruptMarker() + f.events.length = 0 + await assert.rejects(f.run()); assert(!f.events.some(event => event.startsWith("file:"))) + } +}) diff --git a/backend/src/db/controlled-migrations.ts b/backend/src/db/controlled-migrations.ts new file mode 100644 index 00000000..055c051d --- /dev/null +++ b/backend/src/db/controlled-migrations.ts @@ -0,0 +1,146 @@ +import { createHash, randomUUID } from "node:crypto" +import { readdir, readFile } from "node:fs/promises" +import type { PoolClient } from "pg" +import { acquireDatabaseSchemaControl } from "./database-schema-control.js" +import type { DatabaseSchemaIntent, DatabaseSchemaTerminal } from "./database-schema-control-model.js" +import { canonical, exactHistory, observation, REVIEWED_PREDECESSORS, REVIEWED_SUCCESSOR_MANIFEST } from "./controlled-migration-history.js" +import { analyzeMigration, MIGRATION_MANIFEST, migrationManifestDigest, validateMigrationFileSet, type MigrationReview } from "./migration-manifest.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js" +import { acquireDatabaseRequestDrain, releaseDatabaseRequestDrain, SYSTEM_DATABASE_OPERATION_LOCK } from "./operation-lock.js" +import { assertReviewedMigrationMarkerGuard } from "./partition-guard-inheritance.js" +import { assertDatabaseRestoreQuarantineOpen } from "./database-restore-quarantine.js" + +/** Opt-in internal dispatcher only. Default startup remains on the old path. + * Control DDL/enrollment, imports, data verification and READY are absent. */ +export const CONTROLLED_MIGRATIONS_SQL = Object.freeze({ + history: `SELECT id::text AS id, intent, terminal FROM mmapp_control.database_schema_intents ORDER BY id LIMIT ${MIGRATION_MANIFEST.length + 1}`, + namespaces: "SELECT nspname AS name FROM pg_catalog.pg_namespace WHERE nspname !~ '^pg_' AND nspname <> 'information_schema' ORDER BY nspname", + publicObjects: "SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_depend WHERE refclassid = 'pg_catalog.pg_namespace'::pg_catalog.regclass AND refobjid = 'public'::pg_catalog.regnamespace) OR EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extnamespace = 'public'::pg_catalog.regnamespace) AS present", + markerTable: "SELECT c.relkind::text AS kind, c.relpersistence::text AS persistence, c.relrowsecurity AS rls, c.relforcerowsecurity AS force_rls, EXISTS (SELECT 1 FROM pg_catalog.pg_trigger t WHERE t.tgrelid = c.oid AND NOT t.tgisinternal) AS user_triggers FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace WHERE n.nspname = 'public' AND c.relname = 'schema_migrations'", + markers: `SELECT id FROM public.schema_migrations ORDER BY id LIMIT ${MIGRATION_MANIFEST.length + 1}`, + marker: "INSERT INTO public.schema_migrations (id) VALUES ($1)", +}) +export class ControlledMigrationsError extends Error { + readonly code = "CONTROLLED_MIGRATIONS_REFUSED" + constructor(message: string, readonly outcomeUnknown = false, options?: ErrorOptions) { super(message, options) } +} +const refuse = (message: string): never => { throw new ControlledMigrationsError(message) } +const digest = (value: string) => createHash("sha256").update(value).digest("hex") +function requireCommitAck(result: unknown): void { + const terminal = Array.isArray(result) ? result.at(-1) : result + if (!terminal || typeof terminal !== "object" || Reflect.get(terminal, "command") !== "COMMIT") refuse("Application COMMIT command acknowledgement unavailable") +} +interface ExactFile { pin: MigrationReview; sql: string } +async function exactFiles(): Promise { + const directory = new URL("../../drizzle/", import.meta.url) + const entries = (await readdir(directory, { withFileTypes: true })).filter(entry => entry.name.toLowerCase().endsWith(".sql")) + if (entries.some(entry => !entry.isFile())) refuse("Migration SQL must be ordinary pinned files") + validateMigrationFileSet(entries.map(entry => entry.name)) + // Read ONCE and validate those SAME bytes. Never execute a second unchecked + // read after inventory validation, and never expose the source to a caller. + return Promise.all(MIGRATION_MANIFEST.map(async pin => { + const bytes = await readFile(new URL(pin.file, directory)) + analyzeMigration(pin.file, bytes, pin) + return { pin, sql: new TextDecoder("utf-8", { fatal: true }).decode(bytes) } + })) +} +async function baseline(client: PoolClient, prefix: number, expectedGuarded?: boolean): Promise { + const namespaces = (await client.query(CONTROLLED_MIGRATIONS_SQL.namespaces)).rows + if (canonical(namespaces) !== canonical([{ name: "mmapp_control" }, { name: "public" }])) refuse("Unexpected database namespace scope") + if (prefix === 0) { + const rows = (await client.query(CONTROLLED_MIGRATIONS_SQL.publicObjects)).rows + if (rows.length !== 1 || rows[0].present !== false) refuse("Legacy or unmarked application state cannot be enrolled") + return false + } + const table = (await client.query(CONTROLLED_MIGRATIONS_SQL.markerTable)).rows + const ordinary = { kind: "r", persistence: "p", rls: false, force_rls: false, user_triggers: false } + const guarded = canonical(table) !== canonical([ordinary]) + if (expectedGuarded !== undefined && expectedGuarded !== guarded) refuse("Migration marker protection changed") + if (guarded) { + if (prefix < REVIEWED_PREDECESSORS[0].length || migrationManifestDigest() !== REVIEWED_SUCCESSOR_MANIFEST || + canonical(table) !== canonical([{ ...ordinary, user_triggers: true }])) { + refuse("Ordinary or exact reviewed guarded migration marker table required") + } + await assertReviewedMigrationMarkerGuard(client) + } + const markers = (await client.query(CONTROLLED_MIGRATIONS_SQL.markers)).rows + const expected = MIGRATION_MANIFEST.slice(0, prefix).map(pin => ({ id: pin.file.slice(0, -4) })) + if (canonical(markers) !== canonical(expected)) refuse("Migration markers disagree with exact controlled prefix") + return guarded +} + +/** Fixed production pool + fixed pinned source path; no caller-supplied SQL, + * pool, execution callback, receipt, or verifier. Tests may replace module I/O + * in their isolated loader; that is not a production admission interface. */ +export async function executeControlledSqlMigrations() { + const files = await exactFiles(), manifest = migrationManifestDigest() + const { pool } = await import("./index.js") + const control = await acquireDatabaseSchemaControl(pool) + let app: PoolClient | undefined, released = false, activeIntent: DatabaseSchemaIntent | undefined + let applicationSqlSent = false, completedFiles = 0 + try { + app = await pool.connect() + await app.query("SET lock_timeout = '10s'") + await app.query("SET statement_timeout = '120s'") + await app.query("SET synchronous_commit = on") + await acquireDatabaseRequestDrain(app) + // Take the global key before the partition key: restore holds the former + // while its SQL backend takes the latter. + const maintenance = await app.query<{ acquired: boolean }>( + "SELECT pg_catalog.pg_try_advisory_lock($1::bigint) AS acquired", [SYSTEM_DATABASE_OPERATION_LOCK]) + if (maintenance.rows.length !== 1 || maintenance.rows[0].acquired !== true) refuse("Application database maintenance lease unavailable") + // Exclusive SESSION lock fences every adopted partition writer for the + // whole DDL sequence, including self-managed files and marker gaps. + await app.query("SELECT pg_catalog.pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + await app.query("SET search_path = public, pg_catalog") + const current = (await control.read()).state + const history = (await app.query(CONTROLLED_MIGRATIONS_SQL.history)).rows + completedFiles = await exactHistory(history, current, manifest) + const fileHashes = new Set(MIGRATION_MANIFEST.map(pin => pin.sha256)) + const legacy = history.find(row => !fileHashes.has((row.intent as DatabaseSchemaIntent).artifactSha256)) + if (legacy) { + if (!(legacy.terminal as DatabaseSchemaTerminal).legacyRelease) refuse("Authenticated legacy migration release required") + await assertDatabaseRestoreQuarantineOpen(app) + } + const guarded = await baseline(app, completedFiles) + for (const { pin, sql } of files.slice(completedFiles)) { + const id = randomUUID() + const started = await control.transition({ type: "begin", id, kind: "migration", manifestSha256: manifest, artifactSha256: pin.sha256 }) + activeIntent = started.state.intent! + // These writes are reached only after the durable intent COMMIT ACK. + applicationSqlSent = true + if (pin.transaction === "owner-transaction") { + await app.query("BEGIN") + await app.query(sql) + await app.query(CONTROLLED_MIGRATIONS_SQL.marker, [pin.file.slice(0, -4)]) + requireCommitAck(await app.query("COMMIT")) + } else { + // Do not wrap, split, strip or rewrite the audited BEGIN...COMMIT. + requireCommitAck(await app.query(sql)) + if (!pin.selfMarker) await app.query(CONTROLLED_MIGRATIONS_SQL.marker, [pin.file.slice(0, -4)]) + } + await baseline(app, completedFiles + 1, guarded) + await control.transition({ type: "terminal", id, outcome: "observed-success", observationSha256: await observation(activeIntent, pin) }) + activeIntent = undefined; completedFiles++ + } + await baseline(app, completedFiles, guarded) + const unlocked = await app.query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY]) + if (unlocked.rows.length !== 1 || unlocked.rows[0].unlocked !== true) refuse("Application schema lease release unconfirmed") + const maintenanceUnlocked = await app.query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", [SYSTEM_DATABASE_OPERATION_LOCK]) + if (maintenanceUnlocked.rows.length !== 1 || maintenanceUnlocked.rows[0].unlocked !== true) refuse("Application database maintenance lease release unconfirmed") + await releaseDatabaseRequestDrain(app) + for (const setting of ["search_path", "synchronous_commit", "lock_timeout", "statement_timeout"]) await app.query("RESET " + setting) + app.release(false); released = true + await control.close() + return Object.freeze({ completedFiles, manifestSha256: manifest, ready: false as const, schemaAndDataVerified: false as const }) + } catch (cause) { + if (activeIntent) { + // If intent/terminal COMMIT itself lost ACK the store is poisoned; this + // best-effort record fails closed and the unresolved intent survives. + try { await control.transition({ type: "terminal", id: activeIntent.id, outcome: "interrupted-unknown", observationSha256: digest(JSON.stringify({ format: "controlled-migration-unknown-v1", intent: activeIntent })) }) } catch { /* never retry application SQL */ } + } + if (app && !released) { released = true; app.release(true) } + try { await control.close() } catch { /* preserve unknown outcome */ } + throw new ControlledMigrationsError("Controlled migration stopped; automatic replay forbidden", applicationSqlSent, { cause }) + } +} diff --git a/backend/src/db/controlled-schema-reference-core.ts b/backend/src/db/controlled-schema-reference-core.ts new file mode 100644 index 00000000..d0efdb10 --- /dev/null +++ b/backend/src/db/controlled-schema-reference-core.ts @@ -0,0 +1,254 @@ +import { createPublicKey, randomUUID, verify } from "node:crypto" +import { mkdir, open, readFile, readdir, lstat } from "node:fs/promises" +import path from "node:path" +import { Pool, type PoolClient, type PoolConfig } from "pg" +import { z } from "zod" +import { fingerprint, mcpStateDirectory, privateSecretFingerprint } from "../modules/mcp/security.js" +import { analyzeMigration, MIGRATION_MANIFEST, migrationManifestDigest, validateMigrationFileSet } from "./migration-manifest.js" +import { captureMigrationSchema, compareMigrationSchemas, type CapturedMigrationSchema } from "./migration-schema-inventory.js" +import { sourceBindingBytes, sourceBindingCanonical, sourceBindingSha256 } from "./database-source-binding.js" + +const maximum = 32 * 1048576, domain = "database-controlled-schema-reference-v1" +const filename = () => path.join(mcpStateDirectory(), "database-controlled-schema-reference-v1.json") +export class ControlledSchemaReferenceProvisioningError extends Error { + readonly automaticRetry = false + constructor(readonly referenceDatabase: string, readonly phase: string, readonly publicationAttempted: boolean, options: ErrorOptions) { + super("Controlled reference provisioning unconfirmed; inspect the exact owned database before manual cleanup", options) + } +} +const scope = { applicationNamespaces: ["public"], excludedControlNamespaces: ["mmapp_control"] } +const refuse = (reason: string): never => { throw new Error("Controlled schema reference refused: " + reason) } +const sha = (value: unknown) => typeof value === "string" && /^[a-f0-9]{64}$/.test(value) +const identifier = (value: string) => '"' + value.replaceAll('"', '""') + '"' +const literal = (value: string) => "'" + value.replaceAll("'", "''") + "'" +interface Envelope { + format: "controlled-schema-reference-v1"; manifestSha256: string; prefixLength: number; profileSha256: string; + schema: CapturedMigrationSchema; canonicalSha256: string; signature: string; + splitControlApprovalSha256?: string +} +const profile = (schema: CapturedMigrationSchema) => fingerprint({ serverVersion: schema.serverVersion, scope: schema.scope, database: schema.sections.database, namespaces: schema.sections.namespaces }) +export const LEGACY_CONTROL_REFERENCE_DOMAIN = "mmapp:legacy-control-reference:ed25519:v1\n" +const digest = z.string().regex(/^[a-f0-9]{64}$/) +const role = z.object({ name: z.string().regex(/^(?!pg_)[a-z_][a-z0-9_]{0,62}$/), signedOid: z.string().regex(/^[1-9][0-9]{0,9}$/) }).strict() +const splitApproval = z.object({ body: z.object({ format: z.literal("legacy-control-reference-approval-v1"), manifestSha256: digest, + sourceHeadSha256: digest, sourceTlsSpkiSha256: digest, publisher: role, operator: role, control: role, + expiresAt: z.number().int().positive() }).strict(), signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }).strict() +const splitApprovals = new WeakSet() +export function verifyLegacyControlReferenceApproval(raw: Buffer, publicKey: Buffer, pins: { + coldSpkiSha256: string; sourceSpkiSha256: string; sourceHeadSha256: string; sourceTlsSpkiSha256: string; + roles: Record<"publisher" | "operator" | "control", z.infer> +}) { + const parsed: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(raw)), { body, signature } = splitApproval.parse(parsed) + const key = createPublicKey(publicKey), sig = Buffer.from(signature, "base64url"), roles = [body.publisher, body.operator, body.control] + if (!sourceBindingBytes(parsed).equals(raw) || sig.toString("base64url") !== signature || publicKey.includes("PRIVATE KEY") || + key.asymmetricKeyType !== "ed25519" || pins.coldSpkiSha256 === pins.sourceSpkiSha256 || + sourceBindingSha256(key.export({ format: "der", type: "spki" })) !== pins.coldSpkiSha256 || + body.manifestSha256 !== migrationManifestDigest() || body.sourceHeadSha256 !== pins.sourceHeadSha256 || + body.sourceTlsSpkiSha256 !== pins.sourceTlsSpkiSha256 || body.expiresAt <= Date.now() || body.expiresAt - Date.now() > 900000 || + new Set(roles.map(entry => entry.name)).size !== 3 || new Set(roles.map(entry => entry.signedOid)).size !== 3 || + (["publisher", "operator", "control"] as const).some(scope => canonicalRole(body[scope]) !== canonicalRole(pins.roles[scope])) || + !verify(null, Buffer.from(LEGACY_CONTROL_REFERENCE_DOMAIN + sourceBindingCanonical(body)), key, sig)) { + return refuse("independent split-control approval") + } + const result = Object.freeze({ publisher: Object.freeze(body.publisher), operator: Object.freeze(body.operator), control: Object.freeze(body.control), + expiresAt: body.expiresAt, approvalSha256: sourceBindingSha256(raw) }) + splitApprovals.add(result) + return result +} +const canonicalRole = (entry: z.infer) => sourceBindingCanonical({ name: entry.name, signedOid: entry.signedOid }) +interface SplitReference { + approval: ReturnType + liveApplication: Pool + application: PoolConfig + publisher: PoolConfig + systemIdentifier: string + sourceDatabaseOid: string + retain: (client: PoolClient) => () => void + beforePublish: () => Promise +} +interface StandardReference { liveApplication: Pool; connectionString: string } + +/** Fixed path and fixed issuer only. Supplied JSON or a digest never becomes a + * reference. This proves provenance of a generated DDL reference, not data. */ +export async function readControlledSchemaReference(): Promise { + let handle + try { handle = await open(filename(), "r") } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; throw error } + let text: string + try { + const stat = await handle.stat() + if (!stat.isFile() || stat.size < 1 || stat.size > maximum) return refuse("artifact bound") + const bytes = Buffer.alloc(stat.size + 1); let length = 0 + while (length < bytes.length) { const read = await handle.read(bytes, length, bytes.length - length, length); if (!read.bytesRead) break; length += read.bytesRead } + if (length !== stat.size) return refuse("artifact changed while reading") + text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)) + } finally { await handle.close() } + const value = JSON.parse(text) as Envelope + if (!value || Object.getPrototypeOf(value) !== Object.prototype || !["canonicalSha256 format manifestSha256 prefixLength profileSha256 schema signature", + "canonicalSha256 format manifestSha256 prefixLength profileSha256 schema signature splitControlApprovalSha256"].includes(Object.keys(value).sort().join(" ")) || + (Object.hasOwn(value, "splitControlApprovalSha256") && !sha(value.splitControlApprovalSha256)) || + value.format !== "controlled-schema-reference-v1" || value.manifestSha256 !== migrationManifestDigest() || value.prefixLength !== MIGRATION_MANIFEST.length || !sha(value.profileSha256) || !sha(value.canonicalSha256) || !sha(value.signature)) return refuse("artifact envelope") + compareMigrationSchemas(value.schema, value.schema) + if (JSON.stringify(value.schema.scope) !== JSON.stringify(scope) || value.profileSha256 !== profile(value.schema)) return refuse("artifact profile") + const { signature, canonicalSha256, ...payload } = value + if (fingerprint(payload) !== canonicalSha256 || signature !== await privateSecretFingerprint(canonicalSha256, domain)) return refuse("artifact issuer proof") + return structuredClone(value.schema) +} + +async function snapshot(client: PoolClient): Promise { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + try { + await client.query("SET LOCAL search_path=pg_catalog") + await client.query("SET LOCAL TimeZone='UTC'") + await client.query("SET LOCAL DateStyle='ISO, YMD'") + await client.query("SET LOCAL statement_timeout='5s'") + const result = await captureMigrationSchema(client, { scope }) + await client.query("ROLLBACK"); return result + } catch (error) { await client.query("ROLLBACK").catch(() => {}); throw error } +} +const databaseProfileSql = `SELECT current_user AS role,pg_catalog.current_setting('server_version_num') AS version, + pg_catalog.pg_encoding_to_char(d.encoding) AS encoding,d.datcollate AS collate,d.datctype AS ctype,d.datlocprovider::text AS provider, + d.datlocale AS locale,d.daticurules AS rules,d.datcollversion AS collation_version,ts.spcname AS tablespace, + (SELECT t.spcname FROM pg_catalog.pg_database b JOIN pg_catalog.pg_tablespace t ON t.oid=b.dattablespace WHERE b.datname='template0') AS template_tablespace, + d.datacl IS NULL AS default_acl,pg_catalog.shobj_description(d.oid,'pg_database') IS NULL AS no_comment, + pg_catalog.pg_get_userbyid(d.datdba)=current_user AS owned_by_current + FROM pg_catalog.pg_database d JOIN pg_catalog.pg_tablespace ts ON ts.oid=d.dattablespace WHERE d.datname=current_database()` + +/** Separate explicit provisioning workflow. Creates ONLY its own fresh UUID + * database from template0, executes fixed pinned source there, closes it and + * drops exactly that verified database BEFORE publishing an exclusive artifact. + * No application data, application schema or control rows are changed. + * This is never called by startup, restore, imports or a public MCP tool. */ +export async function provisionControlledSchemaReferenceOn(access: StandardReference | SplitReference) { + const split = "approval" in access ? access : null + if (split && (!splitApprovals.has(split.approval) || Date.now() >= split.approval.expiresAt)) return refuse("verified split-control approval required") + try { await lstat(filename()); return refuse("artifact already exists") } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error } + const directory = new URL("../../drizzle/", import.meta.url) + const entries = (await readdir(directory, { withFileTypes: true })).filter(entry => entry.name.toLowerCase().endsWith(".sql")) + if (entries.some(entry => !entry.isFile())) return refuse("non-file migration source") + validateMigrationFileSet(entries.map(entry => entry.name)) + const files = await Promise.all(MIGRATION_MANIFEST.map(async pin => { const bytes = await readFile(new URL(pin.file, directory)); analyzeMigration(pin.file, bytes, pin); return { pin, sql: new TextDecoder("utf-8", { fatal: true }).decode(bytes) } })) + const control = await access.liveApplication.connect() + const name = "mm_schema_reference_" + randomUUID().replaceAll("-", "") + if (!/^mm_schema_reference_[a-f0-9]{32}$/.test(name)) return refuse("owned reference name") + let reference: Pool | undefined, captured: CapturedMigrationSchema | undefined + let phase = "profile", publicationAttempted = false + try { + const controlPeer = split?.retain(control) + const bind = async (client: PoolClient, database: string, oid: string, scope: "operator" | "publisher") => { + if (!split) return + const retained = split.retain(client) + const { rows: [identity] } = await client.query(`SELECT d.oid::text AS database_oid,d.datname AS database, + r.oid::text AS role_oid,(pg_catalog.pg_control_system()).system_identifier::text AS system_identifier, + pg_catalog.pg_get_userbyid(d.datdba) AS owner + FROM pg_catalog.pg_database d JOIN pg_catalog.pg_roles r ON r.rolname=CURRENT_USER AND r.rolname=SESSION_USER + WHERE d.datname=pg_catalog.current_database() AND NOT pg_catalog.pg_is_in_recovery()`) + retained() + if (identity?.database !== database || identity.database_oid !== oid || identity.role_oid !== split.approval[scope].signedOid || + identity.system_identifier !== split.systemIdentifier || identity.owner !== split.approval.operator.name || + Date.now() >= split.approval.expiresAt) return refuse("reference connection scope") + } + if (split) await bind(control, String(split.application.database), split.sourceDatabaseOid, "operator") + await control.query("SET standard_conforming_strings=on") + await control.query("SET statement_timeout='120s'") + const rows = (await control.query(databaseProfileSql)).rows + if (rows.length !== 1) return refuse("database profile missing") + const meta = rows[0] + if (!/^18\d{4}$/.test(meta.version) || !["b", "c", "i"].includes(meta.provider) || !meta.default_acl || !meta.no_comment || !meta.owned_by_current || + [meta.role, meta.encoding, meta.collate, meta.ctype, meta.tablespace, meta.template_tablespace].some(value => typeof value !== "string" || !value || value.length > 256 || /[\x00-\x1f\x7f]/.test(value)) || + [meta.locale, meta.rules, meta.collation_version].some(value => value !== null && (typeof value !== "string" || value.length > 4096 || /[\x00-\x1f\x7f]/.test(value)))) return refuse("unsupported database profile") + const provider = { b: "builtin", c: "libc", i: "icu" }[meta.provider as "b" | "c" | "i"] + let sql = `CREATE DATABASE ${identifier(name)} TEMPLATE template0 OWNER ${identifier(meta.role)} ENCODING ${literal(meta.encoding)} LOCALE_PROVIDER ${provider} LC_COLLATE ${literal(meta.collate)} LC_CTYPE ${literal(meta.ctype)}` + if (meta.tablespace !== meta.template_tablespace) sql += ` TABLESPACE ${identifier(meta.tablespace)}` + if (meta.provider === "b" || meta.provider === "i") { if (!meta.locale) return refuse("provider locale missing"); sql += ` ${meta.provider === "b" ? "BUILTIN_LOCALE" : "ICU_LOCALE"} ${literal(meta.locale)}` } + if (meta.provider === "i" && meta.rules !== null) sql += " ICU_RULES " + literal(meta.rules) + phase = "create-unknown" + await control.query(sql); phase = "created" + const referenceOid = async () => { + const { rows: [owned] } = await control.query("SELECT oid::text AS oid FROM pg_catalog.pg_database WHERE datname=$1 AND pg_catalog.pg_get_userbyid(datdba)=current_user", [name]) + controlPeer?.() + if (!owned || typeof owned.oid !== "string" || owned.oid === split?.sourceDatabaseOid) return refuse("reference ownership unavailable") + return owned.oid + } + const oid = split ? await referenceOid() : undefined + let referenceOptions: PoolConfig + if ("connectionString" in access) { + const url = new URL(access.connectionString); url.pathname = "/" + name + referenceOptions = { connectionString: url.toString() } + } else referenceOptions = { ...access.application, database: name } + reference = new Pool({ ...referenceOptions, max: 1, connectionTimeoutMillis: 8000, statement_timeout: 120000 }) + const client = await reference.connect() + let clean = false + try { + if (fingerprint((await client.query(databaseProfileSql)).rows[0]) !== fingerprint(meta)) return refuse("generated database profile mismatch") + await client.query("SET search_path=public,pg_catalog") + if (split) { + await bind(client, name, oid!, "operator") + const owner = new Pool({ ...split.publisher, database: name, max: 1 }) + owner.on("error", () => {}) + try { + const publisher = await owner.connect() + try { + await bind(publisher, name, oid!, "publisher") + const peer = split.retain(publisher) + await publisher.query("BEGIN") + try { + await publisher.query("SET LOCAL search_path=pg_catalog") + const entries = await publisher.query("SELECT rolname AS name,oid::text AS oid FROM pg_catalog.pg_roles WHERE rolname=ANY($1::text[]) ORDER BY rolname", + [[split.approval.publisher.name, split.approval.operator.name, split.approval.control.name]]) + const expected = [split.approval.publisher, split.approval.operator, split.approval.control].map(role => ({ name: role.name, oid: role.signedOid })).sort((a, b) => a.name.localeCompare(b.name, "en")) + if (fingerprint(entries.rows.sort((a, b) => a.name.localeCompare(b.name, "en"))) !== fingerprint(expected)) return refuse("reference role OID mismatch") + await publisher.query("CREATE SCHEMA mmapp_control") + await publisher.query(`GRANT USAGE ON SCHEMA mmapp_control TO ${identifier(split.approval.operator.name)}, ${identifier(split.approval.control.name)}`) + peer() + if ((await publisher.query("COMMIT")).command !== "COMMIT") return refuse("reference namespace COMMIT unknown") + } catch (error) { await publisher.query("ROLLBACK").catch(() => {}); throw error } + } finally { publisher.release(true) } + } finally { await owner.end() } + } else await client.query("CREATE SCHEMA mmapp_control") + for (const { pin, sql } of files) { + if (pin.transaction === "owner-transaction") await client.query("BEGIN") + const executed = await client.query(sql) + if (!pin.selfMarker) await client.query("INSERT INTO public.schema_migrations(id) VALUES($1)", [pin.file.slice(0, -4)]) + const committed = pin.transaction === "owner-transaction" ? await client.query("COMMIT") : Array.isArray(executed) ? executed.at(-1) : executed + if (committed?.command !== "COMMIT") return refuse("reference file COMMIT not acknowledged") + } + const markers = (await client.query(`SELECT id FROM public.schema_migrations ORDER BY id LIMIT ${MIGRATION_MANIFEST.length + 1}`)).rows + if (fingerprint(markers) !== fingerprint(MIGRATION_MANIFEST.map(pin => ({ id: pin.file.slice(0, -4) })))) return refuse("reference markers mismatch") + captured = await snapshot(client); clean = true + if (split) { + const owner = split.approval.publisher.name, namespace = captured.sections.namespaces.find(row => row.namespace === "mmapp_control") + const acl = [`${owner}=UC/${owner}`, `${split.approval.operator.name}=U/${owner}`, `${split.approval.control.name}=U/${owner}`].sort() + if (namespace?.owner !== owner || namespace.comment !== null || !Array.isArray(namespace.acl) || + JSON.stringify([...namespace.acl].sort()) !== JSON.stringify(acl)) return refuse("exact approved control namespace owner and ACL") + } + } finally { client.release(!clean) } + await reference.end(); reference = undefined + const owned = (await control.query("SELECT datname,pg_catalog.pg_get_userbyid(datdba)=current_user AS owned FROM pg_catalog.pg_database WHERE datname=$1", [name])).rows + if (owned.length !== 1 || owned[0].datname !== name || owned[0].owned !== true) return refuse("reference cleanup ownership changed") + if (split && await referenceOid() !== oid) return refuse("reference cleanup OID changed") + phase = "drop-unknown" + await control.query("DROP DATABASE " + identifier(name)); phase = "dropped" + await split?.beforePublish() + if (split && Date.now() >= split.approval.expiresAt) return refuse("split-control approval expired") + const payload = { format: "controlled-schema-reference-v1" as const, manifestSha256: migrationManifestDigest(), prefixLength: MIGRATION_MANIFEST.length, profileSha256: profile(captured), schema: captured, + ...(split ? { splitControlApprovalSha256: split.approval.approvalSha256 } : {}) } + const canonicalSha256 = fingerprint(payload), signature = await privateSecretFingerprint(canonicalSha256, domain) + const bytes = Buffer.from(JSON.stringify({ ...payload, canonicalSha256, signature }) + "\n") + if (bytes.length > maximum) return refuse("generated artifact too large") + await mkdir(mcpStateDirectory(), { recursive: true, mode: 0o700 }) + publicationAttempted = true + const output = await open(filename(), "wx", 0o600) + try { await output.writeFile(bytes); await output.sync() } finally { await output.close() } + return Object.freeze({ manifestSha256: payload.manifestSha256, logicalSchemaSha256: captured.sha256, referenceDatabaseDropped: true, + ...(split ? { splitControlApprovalSha256: split.approval.approvalSha256 } : {}), readyAuthorized: false }) + } catch (cause) { + throw new ControlledSchemaReferenceProvisioningError(name, phase, publicationAttempted, { cause }) + } finally { + if (reference) await reference.end().catch(() => {}) + // A failed or uncertain CREATE/DROP is never retried or force-dropped. The + // exact owned UUID may remain for an operator; no artifact is published. + control.release(true) + } +} diff --git a/backend/src/db/controlled-schema-reference.sql.test.ts b/backend/src/db/controlled-schema-reference.sql.test.ts new file mode 100644 index 00000000..e629e75f --- /dev/null +++ b/backend/src/db/controlled-schema-reference.sql.test.ts @@ -0,0 +1,164 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { fingerprint, privateSecretFingerprint } from "../modules/mcp/security.js" +import { basename, dirname, resolve } from "node:path" +import { readFile, writeFile } from "node:fs/promises" +assert.equal(process.env.MMAPP_OWN_CONTROLLED_SCHEMA_REFERENCE, "true") +const databaseUrl = new URL(process.env.DATABASE_URL ?? "") +assert.equal(databaseUrl.hostname, "127.0.0.1"); assert.equal(databaseUrl.pathname, "/postgres") +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const { Pool } = await import("pg") +const { pool, closePool } = await import("./index.js") +const { applySqlMigrations } = await import("./migrate.js") +const { MIGRATION_MANIFEST } = await import("./migration-manifest.js") +const { initialDatabaseSchemaControlState } = await import("./database-schema-control-model.js") +const { executeControlledSqlMigrations } = await import("./controlled-migrations.js") +const { provisionControlledSchemaReference, readControlledSchemaReference, ControlledSchemaReferenceProvisioningError } = await import("./controlled-schema-reference.js") +const { verifyControlledSchemaCurrentState, CONTROLLED_SCHEMA_VERIFICATION_SQL } = await import("./controlled-schema-verification.js") +const originalConnect = pool.connect.bind(pool), cases: string[] = [] +let application: InstanceType | undefined +async function owned(work: () => Promise) { + assert(application) + pool.connect = application.connect.bind(application) as typeof pool.connect + try { return await work() } finally { pool.connect = originalConnect as typeof pool.connect } +} +try { + await applySqlMigrations(pool) + const name = "mm_reference_app_" + randomUUID().replaceAll("-", "") + assert(/^mm_reference_app_[a-f0-9]{32}$/.test(name)) + await pool.query('CREATE DATABASE "' + name + '" TEMPLATE template0') + const url = new URL(databaseUrl); url.pathname = "/" + name + application = new Pool({ connectionString: url.toString(), max: 6 }) + await application.query("CREATE SCHEMA mmapp_control; CREATE TABLE mmapp_control.database_schema_state(id smallint PRIMARY KEY CHECK(id=1),revision bigint NOT NULL CHECK(revision>=0),state jsonb NOT NULL); CREATE TABLE mmapp_control.database_schema_intents(id uuid PRIMARY KEY,intent jsonb NOT NULL,terminal jsonb)") + await application.query("INSERT INTO mmapp_control.database_schema_state VALUES(1,0,$1::jsonb)", [JSON.stringify(initialDatabaseSchemaControlState())]) + assert.equal((await owned(executeControlledSqlMigrations)).completedFiles, MIGRATION_MANIFEST.length) + assert.equal(await readControlledSchemaReference(), null) + const missing = await owned(verifyControlledSchemaCurrentState) + assert.equal(missing.referenceStatus, "missing"); assert.equal(missing.ready, false); assert.equal(missing.dataFacts.length, 0) + cases.push("missing reference never promotes a controlled40 ledger or probes data as trusted schema") + const before = (await application.query("SELECT count(*)::int AS n FROM pg_database WHERE datname LIKE 'mm_schema_reference_%'")).rows[0].n + const generated = await owned(provisionControlledSchemaReference) + assert.equal(generated.referenceDatabaseDropped, true); assert.equal(generated.readyAuthorized, false) + assert.equal((await application.query("SELECT count(*)::int AS n FROM pg_database WHERE datname LIKE 'mm_schema_reference_%'")).rows[0].n, before) + const reference = await readControlledSchemaReference(); assert(reference) + assert.equal(reference.sha256, generated.logicalSchemaSha256) + assert.equal(reference.format, "pg18-application-schema-v3") + // Reconstruct the retired v2 algorithm from genuine current raw catalogs. + // This is test-only: numeric indkey/tombstones and old fast-default-only + // projection are intentional. Production never upgrades or accepts v2. + const canonical = (value: unknown): string => Array.isArray(value) ? "[" + value.map(canonical).join(",") + "]" + : value && typeof value === "object" ? "{" + Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, item]) => JSON.stringify(key) + ":" + canonical(item)).join(",") + "}" + : JSON.stringify(value) ?? "null" + const sortRows = (rows: T[]): T[] => rows.map(row => canonical(row)).sort().map(row => JSON.parse(row) as T) + const legacySections = structuredClone(reference.sections) + legacySections.constraints = sortRows(legacySections.constraints.map(row => Object.fromEntries(Object.entries(row).filter(([key]) => key !== "restore_definition")))) + const legacyLogical = { ...legacySections, columns: sortRows(legacySections.columns.map(row => Object.fromEntries(Object.entries(row).filter(([key]) => key !== "has_missing" && key !== "missing_value")))) } + const legacyBody = { format: "pg18-application-schema-v2", scope: reference.scope, serverVersion: reference.serverVersion, sections: legacyLogical } + const legacySchema = { ...legacyBody, sections: legacySections, observedPhysical: reference.observedPhysical, canonical: canonical(legacyBody), sha256: fingerprint(legacyBody) } + const compatibilityFile = resolve(process.env.MCP_STATE_DIR!, "database-controlled-schema-reference-v1.json"), v3Bytes = await readFile(compatibilityFile) + const { signature: _v3Signature, canonicalSha256: _v3Canonical, ...v3Payload } = JSON.parse(v3Bytes.toString("utf8")) + const v2Payload = { ...v3Payload, schema: legacySchema }, v2Canonical = fingerprint(v2Payload) + const v2Bytes = Buffer.from(JSON.stringify({ ...v2Payload, canonicalSha256: v2Canonical, signature: await privateSecretFingerprint(v2Canonical, "database-controlled-schema-reference-v1") }) + "\n") + try { + await writeFile(compatibilityFile, v2Bytes) + await assert.rejects(readControlledSchemaReference(), /format\/version/) + await assert.rejects(owned(provisionControlledSchemaReference)) + assert.deepEqual(await readFile(compatibilityFile), v2Bytes) + } finally { await writeFile(compatibilityFile, v3Bytes) } + assert.equal((await readControlledSchemaReference())?.sha256, reference.sha256) + cases.push("genuine raw-catalog legacy-v2 algorithm and fixed-issuer HMAC cannot bypass v3 refusal or overwrite existing reference; fresh v3 producer remains valid") + + cases.push("fixed producer executes current pins in independently created database and drops only its verified owned UUID before publishing HMAC artifact") + await assert.rejects(owned(provisionControlledSchemaReference)) + assert.equal((await readControlledSchemaReference())?.sha256, reference.sha256) + cases.push("existing trusted reference is not overwritten or regenerated by a repeat invocation") + const matching = await owned(verifyControlledSchemaCurrentState) + assert.equal(matching.referenceStatus, "matched") + assert.equal(matching.currentPristineDataInvariants, true, JSON.stringify({ data: matching.dataFacts.filter(f => !f.valid), sequences: matching.sequenceFacts.filter(f => !f.valid) })) + assert.equal(matching.ready, false); assert.equal(matching.historicalDataEquivalent, false) + cases.push("actual PG18 current schema matches independent issuer reference; exact pristine data and sequence invariants are checked without READY") + // Run the genuine fixed predicate on typed, temporary copies of already + // verified pristine rows. No production trigger is bypassed or altered. + const epochProbe = await application.connect() + try { + for (const table of ["traffic_settings", "uptime_settings", "internet_path_settings"]) { + await epochProbe.query('CREATE TEMP TABLE mm_owned_epoch_seed AS SELECT * FROM public."' + table + '"') + const timeFields = ["created_at", "updated_at"] + const seed = (await epochProbe.query("SELECT pg_catalog.to_jsonb(t)-ARRAY['created_at','updated_at','poll_configuration_epoch'] AS seed FROM pg_temp.mm_owned_epoch_seed t")).rows[0].seed + const original = CONTROLLED_SCHEMA_VERIFICATION_SQL.seed(table) + const relation = 'FROM public."' + table + '"' + assert.equal(original.split(relation).length, 2) + const probe = original.replace(relation, 'FROM pg_temp.mm_owned_epoch_seed') + const check = async () => (await epochProbe.query(probe, [JSON.stringify(seed), timeFields])).rows[0].valid + assert.equal(await check(), true) + for (const epoch of [null, "00000000-0000-0000-0000-000000000000", "12345678-1234-1234-9234-123456789abc", "12345678-1234-4234-7234-123456789abc"]) { + await epochProbe.query("UPDATE pg_temp.mm_owned_epoch_seed SET poll_configuration_epoch=$1::uuid", [epoch]) + assert.equal(await check(), false) + } + await epochProbe.query("UPDATE pg_temp.mm_owned_epoch_seed SET poll_configuration_epoch=pg_catalog.gen_random_uuid()") + assert.equal(await check(), true) + await epochProbe.query("UPDATE pg_temp.mm_owned_epoch_seed SET enabled=false") + assert.equal(await check(), false) + await epochProbe.query("UPDATE pg_temp.mm_owned_epoch_seed SET enabled=true") + await epochProbe.query("ALTER TABLE pg_temp.mm_owned_epoch_seed ADD COLUMN unrelated_field integer DEFAULT 1") + assert.equal(await check(), false) + await epochProbe.query("DROP TABLE pg_temp.mm_owned_epoch_seed") + } + } finally { epochProbe.release(true) } + assert(matching.gaps.includes("NONDETERMINISTIC_POLL_EPOCH_ORIGIN_UNATTESTED")) + cases.push("actual typed SQL epoch predicates accept UUIDv4 only for three0042 seeds; null/nil/version/variant, changed settings and unknown columns refuse without historical-origin authority") + await application.query("INSERT INTO public.servers(host) VALUES('192.0.2.1')") + const changedData = await owned(verifyControlledSchemaCurrentState) + assert.equal(changedData.referenceStatus, "matched"); assert.equal(changedData.currentPristineDataInvariants, false) + assert(changedData.dataFacts.some(f => f.table === "servers" && !f.valid)); assert(changedData.sequenceFacts.some(f => !f.valid)) + cases.push("actual added application row and advanced identity sequence fail pristine data evidence despite equal DDL") + await application.query("ALTER TABLE public.servers ADD COLUMN mm_owned_schema_drift integer") + const drift = await owned(verifyControlledSchemaCurrentState) + assert.equal(drift.referenceStatus, "different"); assert.equal(drift.dataFacts.length, 0) + cases.push("actual schema drift refuses trusted match before any data-probe classification") + const file = resolve(process.env.MCP_STATE_DIR!, "database-controlled-schema-reference-v1.json"), original = await readFile(file) + const oldEnvelope = JSON.parse(original.toString("utf8")); oldEnvelope.prefixLength = 39 + await writeFile(file, JSON.stringify(oldEnvelope)) + await assert.rejects(readControlledSchemaReference(), /artifact envelope/) + await writeFile(file, original) + cases.push("historical39 reference envelope cannot be promoted to current40 by startup or restore") + const fake = JSON.parse(original.toString("utf8")); fake.signature = "f".repeat(64) + await writeFile(file, JSON.stringify(fake)) + await assert.rejects(readControlledSchemaReference()) + const untrusted = await owned(verifyControlledSchemaCurrentState) + assert.equal(untrusted.referenceStatus, "invalid"); assert.equal(untrusted.dataFacts.length, 0); assert.equal(untrusted.ready, false) + await writeFile(file, original) + cases.push("forged artifact signature is refused by fixed reader and never authorizes data verification") + const initialStateDirectory = process.env.MCP_STATE_DIR!, faultDirectory = resolve(initialStateDirectory, "owned-reference-create-ack-loss") + let uncertainName = "", creates = 0, drops = 0 + process.env.MCP_STATE_DIR = faultDirectory + pool.connect = (async () => { + const client = await application!.connect() + return { release(destroy?: boolean) { client.release(destroy) }, async query(text: string, values?: unknown[]) { + if (text.startsWith("DROP DATABASE ")) drops++ + const result = await client.query(text, values) + if (text.startsWith('CREATE DATABASE "mm_schema_reference_')) { + const match = /^CREATE DATABASE "(mm_schema_reference_[a-f0-9]{32})"/.exec(text); assert(match) + uncertainName = match[1]; creates++ + throw Error("Owned fixture lost actual reference CREATE acknowledgement") + } + return result + } } + }) as typeof pool.connect + try { + await assert.rejects(provisionControlledSchemaReference(), (error: unknown) => error instanceof ControlledSchemaReferenceProvisioningError && error.referenceDatabase === uncertainName && error.phase === "create-unknown" && !error.publicationAttempted && !error.automaticRetry) + assert.equal(creates, 1); assert.equal(drops, 0); assert.equal(await readControlledSchemaReference(), null) + } finally { pool.connect = originalConnect as typeof pool.connect; process.env.MCP_STATE_DIR = initialStateDirectory } + assert(/^mm_schema_reference_[a-f0-9]{32}$/.test(uncertainName)) + const orphan = (await application.query("SELECT datname,pg_get_userbyid(datdba)=current_user AS owned FROM pg_database WHERE datname=$1", [uncertainName])).rows + assert.deepEqual(orphan, [{ datname: uncertainName, owned: true }]) + // Explicit test-owner cleanup after independently proving the exact UUID. + // The production producer itself never retries or force-cleans unknown CREATE. + await application.query('DROP DATABASE "' + uncertainName + '"') + cases.push("actual CREATE with lost ACK publishes no reference and never retries or drops uncertain database; test owner independently cleans verified UUID") + console.log("Owned controlled schema reference SQL:", JSON.stringify({ cases: cases.length, names: cases, actualSql: true, referenceGeneratedByRealProducer: true, independentPinnedFiles: MIGRATION_MANIFEST.length, applicationPinnedFiles: MIGRATION_MANIFEST.length, readyAuthorized: false, startupWired: false, productionDatabaseTouched: false })) +} finally { + pool.connect = originalConnect as typeof pool.connect + await application?.end() + await closePool() +} diff --git a/backend/src/db/controlled-schema-reference.ts b/backend/src/db/controlled-schema-reference.ts new file mode 100644 index 00000000..6813c179 --- /dev/null +++ b/backend/src/db/controlled-schema-reference.ts @@ -0,0 +1,9 @@ +import { env } from "../config.js" +import { provisionControlledSchemaReferenceOn } from "./controlled-schema-reference-core.js" + +export { ControlledSchemaReferenceProvisioningError, readControlledSchemaReference } from "./controlled-schema-reference-core.js" + +export async function provisionControlledSchemaReference() { + const { pool } = await import("./index.js") + return provisionControlledSchemaReferenceOn({ liveApplication: pool, connectionString: env.DATABASE_URL }) +} diff --git a/backend/src/db/controlled-schema-verification.test.ts b/backend/src/db/controlled-schema-verification.test.ts new file mode 100644 index 00000000..aa51f3dc --- /dev/null +++ b/backend/src/db/controlled-schema-verification.test.ts @@ -0,0 +1,157 @@ +import assert from "node:assert/strict" +import { createHash, createHmac, randomUUID } from "node:crypto" +import { readFile } from "node:fs/promises" +import { fileURLToPath } from "node:url" +import { test } from "node:test" +import { runInNewContext } from "node:vm" +import ts from "typescript" +import * as model from "./database-schema-control-model.js" +import * as manifest from "./migration-manifest.js" +import { CONTROLLED_MIGRATIONS_SQL } from "./controlled-migrations.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js" + +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item) +const fingerprint = (value: unknown) => createHash("sha256").update(canonical(value)).digest("hex") +const privateSecretFingerprint = async (value: string, domain: string) => createHmac("sha256", "inert-only-key:" + domain).update(value).digest("hex") +const source = await readFile(new URL("./controlled-schema-verification.ts", import.meta.url), "utf8") +const compiled = ts.transpileModule(source.replaceAll("import.meta.url", JSON.stringify(new URL("./controlled-schema-verification.ts", import.meta.url).href)), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022, esModuleInterop: true } }).outputText +const historySource = await readFile(new URL("./controlled-migration-history.ts", import.meta.url), "utf8") +const compiledHistory = ts.transpileModule(historySource, { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText + +/** Actual verifier source, inert module I/O. Synthetic capture/reference here + * checks control flow only; it is not a production reference or SQL acceptance. + * Root's producer→verifier private PG tests exercise genuine schema/data. */ +async function fixture() { + const owner = randomUUID(), epoch = 1, digest = manifest.migrationManifestDigest() + const history = await Promise.all(manifest.MIGRATION_MANIFEST.map(async pin => { + const intent = { id: randomUUID(), epoch, kind: "migration" as const, manifestSha256: digest, artifactSha256: pin.sha256 } + const observationSha256 = await privateSecretFingerprint(fingerprint({ format: "controlled-migration-file-ack-v1", intent, file: pin.file, transaction: pin.transaction, selfMarker: pin.selfMarker, fileCommitAck: true, markerAck: true }), "database-controlled-migration-file-ack-v1") + return { id: intent.id, intent, terminal: { intent, outcome: "observed-success" as const, observationSha256 } } + })) + let state: model.DatabaseSchemaControlState = { format: model.DATABASE_SCHEMA_CONTROL_FORMAT, owner, epoch, phase: "TERMINAL", intent: null, terminal: history.at(-1)!.terminal } + const facts = { reference: "valid" as "valid" | "missing" | "invalid" | "different", badTable: "", badSequence: false, badContext: false, badMarkers: false, forgedLedger: false, markerView: false, loseCommit: false } + const events: string[] = [], releases: boolean[] = [] + const context = { backend_pid: "42", transaction_started: "2026-10-25 10:00:00+00" } + const schema = { sha256: "a".repeat(64), sections: { + relations: [{ namespace: "public", name: "schema_migrations", kind: "r", persistence: "p", row_security: false, force_row_security: false }], + columns: [{ namespace: "public", relation: "schema_migrations", name: "id", dropped: false, type: "text" }], + sequences: [{ namespace: "public", name: "servers_id_seq", start: "1" }], + } } + const app = { + query: async (query: string, values: unknown[] = []) => { + events.push(query) + if (query === CONTROLLED_MIGRATIONS_SQL.history) { + const rows = structuredClone(history) + if (facts.forgedLedger) rows[0].terminal.observationSha256 = "a".repeat(64) + return { rows } + } + if (query === CONTROLLED_MIGRATIONS_SQL.markers) return { rows: manifest.MIGRATION_MANIFEST.slice(0, facts.badMarkers ? manifest.MIGRATION_MANIFEST.length - 1 : manifest.MIGRATION_MANIFEST.length).map(pin => ({ id: pin.file.slice(0, -4) })) } + if (query === sql.context) return { rows: [{ ...context }] } + if (query.startsWith("SET ") || query.startsWith("RESET ") || query.startsWith("BEGIN ")) return { rows: [] } + if (query.includes("pg_catalog.pg_advisory_lock(")) { assert.deepEqual(Array.from(values), [PARTITION_WRITER_ADVISORY_KEY]); return { rows: [] } } + if (query.includes("pg_catalog.pg_advisory_unlock(")) return { rows: [{ unlocked: true }] } + if (query === "COMMIT") { if (facts.loseCommit) throw Error("lost readonly COMMIT ACK"); return { rows: [], command: "COMMIT" } } + const data = /\/\* controlled-schema-data:(seed|empty|sequence):([a-z_0-9]+) \*\//.exec(query) + assert(data, "Only fixed production data SQL may execute") + if (data[1] === "seed") { + assert(query.includes("LIMIT 2")); assert(query.includes("- $2::text[]")) + const expected = JSON.parse(String(values[0])); if (data[2] === "server_probe_storage_usage") assert.deepEqual(expected, { scope: "global", admissions: 0, reserved_files: 0, reserved_bytes: 0 }); else if (data[2] === "server_poll_storage_usage") assert.deepEqual(expected, { scope: "global", admissions: 0, reserved_rows: 0, reserved_bytes: 0, compacted_admissions: 0, archive_bytes: 0 }); else assert.equal(expected.id, 1) + } + return { rows: [{ ...context, ...(facts.badContext ? { backend_pid: "43" } : {}), valid: data[1] === "sequence" ? !facts.badSequence : data[2] !== facts.badTable }] } + }, + release: (destroy: boolean) => { releases.push(destroy) }, + } + const modules: Record = { + "node:crypto": { createHash }, + "node:url": { fileURLToPath }, + "./database-schema-control.js": { acquireDatabaseSchemaControl: async () => { + events.push("acquire control"); state = model.claimDatabaseSchemaOwnership(state, randomUUID()) + return { read: async () => ({ state: structuredClone(state) }), close: async () => { events.push("close control") } } + } }, + "./database-schema-control-model.js": { ...model, checkDatabaseSchemaControlState: (value: unknown) => model.checkDatabaseSchemaControlState(structuredClone(value)) }, + "./controlled-migrations.js": { CONTROLLED_MIGRATIONS_SQL }, + "./migration-manifest.js": manifest, + "./controlled-schema-reference.js": { readControlledSchemaReference: async () => { if (facts.reference === "invalid") throw Error("bad artifact signature"); return facts.reference === "missing" ? null : { ...schema, sha256: facts.reference === "different" ? "b".repeat(64) : schema.sha256 } } }, + "./migration-schema-inventory.js": { + captureMigrationSchema: async (_client: unknown, options: unknown) => { events.push("capture schema"); assert.deepEqual(JSON.parse(JSON.stringify(options)), { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["mmapp_control"] } }); return { ...schema, sections: { ...schema.sections, relations: [{ ...schema.sections.relations[0], kind: facts.markerView ? "v" : "r" }] } } }, + compareMigrationSchemas: (expected: typeof schema, observed: typeof schema) => { events.push("compare schema"); if (expected.sha256 !== observed.sha256) throw Error("schema differs") }, + }, + "../modules/mcp/security.js": { fingerprint, privateSecretFingerprint, privateFingerprinterExisting: async (domain: string) => (value: string) => privateSecretFingerprint(value, domain) }, + "../services/partition-write-transaction.js": { PARTITION_WRITER_ADVISORY_KEY }, + "./index.js": { pool: { connect: async () => app } }, + } + const exports = {} as { verifyControlledSchemaCurrentState(): Promise<{ ready: false; schemaAndDataVerified: false; historicalDataEquivalent: false; currentPristineDataInvariants: boolean; referenceStatus: string; gaps: string[]; dataFacts: unknown[]; sequenceFacts: unknown[] }>; CONTROLLED_SCHEMA_VERIFICATION_SQL: typeof import("./controlled-schema-verification.js").CONTROLLED_SCHEMA_VERIFICATION_SQL } + const globals = { require: (name: string) => { assert(name in modules, "Unapproved import: " + name); return modules[name] }, URL, Promise, Error, Object, Map, Set, Date, structuredClone } + modules["./controlled-migration-history.js"] = {} + runInNewContext(compiledHistory, { ...globals, exports: modules["./controlled-migration-history.js"] }, { filename: "controlled-migration-history.ts" }) + runInNewContext(compiled, { ...globals, exports }, { filename: "controlled-schema-verification.ts" }) + const sql = exports.CONTROLLED_SCHEMA_VERIFICATION_SQL + return { exports, facts, events, releases, run: exports.verifyControlledSchemaCurrentState } +} +test("fixed current snapshot authenticates full ledger and checks bounded data after reference equality", async () => { + const f = await fixture(), result = await f.run() + assert.equal(result.referenceStatus, "matched"); assert.equal(result.currentPristineDataInvariants, true) + assert.equal(result.ready, false); assert.equal(result.schemaAndDataVerified, false); assert.equal(result.historicalDataEquivalent, false) + assert.equal(result.dataFacts.length, manifest.CURRENT_REQUIRED_TABLES.length) + assert.equal(f.events.filter(query => query.startsWith("/* controlled-schema-data:seed:")).length, 18) + assert.equal(result.sequenceFacts.length, 1) + assert(f.events.indexOf("compare schema") < f.events.findIndex(query => query.startsWith("/* controlled-schema-data:"))) + assert(f.events.indexOf("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") < f.events.indexOf("capture schema")) + assert.deepEqual(f.releases, [false]) +}) +test("missing, invalid or different reference never reads application data or grants READY", async () => { + for (const status of ["missing", "invalid", "different"] as const) { + const f = await fixture(); f.facts.reference = status + const result = await f.run() + assert.equal(result.referenceStatus, status); assert.equal(result.ready, false); assert.equal(result.currentPristineDataInvariants, false) + assert(!f.events.some(query => query.startsWith("/* controlled-schema-data:"))) + assert(result.gaps.includes("TRUSTED_SCHEMA_REFERENCE_" + status.toUpperCase())) + } +}) +test("real data predicate failures remain observed failures, never hidden by matching schema", async () => { + for (const table of ["data_migration", "traffic_flow_settings", "servers"]) { + const f = await fixture(); f.facts.badTable = table + const result = await f.run() + assert.equal(result.referenceStatus, "matched"); assert.equal(result.currentPristineDataInvariants, false); assert.equal(result.ready, false) + } + const f = await fixture(); f.facts.badSequence = true + assert.equal((await f.run()).currentPristineDataInvariants, false) +}) +test("forged ACK ledger or marker mismatch refuses rather than returning verification", async () => { + for (const field of ["forgedLedger", "badMarkers"] as const) { + const f = await fixture(); f.facts[field] = true + await assert.rejects(f.run()); assert.deepEqual(f.releases, [true]) + assert(!f.events.some(query => query.startsWith("/* controlled-schema-data:"))) + } +}) +test("marker schema drift refuses before reading a forged marker view", async () => { + const f = await fixture(); f.facts.markerView = true + await assert.rejects(f.run()); assert(!f.events.includes(CONTROLLED_MIGRATIONS_SQL.markers)) +}) +test("snapshot backend changes or lost read-only COMMIT cannot publish accepted evidence", async () => { + for (const field of ["badContext", "loseCommit"] as const) { + const f = await fixture(); f.facts[field] = true + await assert.rejects(f.run()); assert.deepEqual(f.releases, [true]) + } +}) +test("data SQL permits only known seed tables and bounded lowercase identifiers", async () => { + const f = await fixture() + assert.throws(() => f.exports.CONTROLLED_SCHEMA_VERIFICATION_SQL.seed("servers")) + assert.throws(() => f.exports.CONTROLLED_SCHEMA_VERIFICATION_SQL.seed('mcp_settings"; SELECT 1')) +}) + + +test("only the three0042 seeds validate current UUIDv4 epochs separately without attesting origin", async () => { + const f = await fixture() + for (const table of ["traffic_settings", "uptime_settings", "internet_path_settings"]) { + const query = f.exports.CONTROLLED_SCHEMA_VERIFICATION_SQL.seed(table) + assert(query.includes("t.poll_configuration_epoch IS NOT NULL")) + assert(query.includes("4[0-9a-f]{3}-[89ab]")) + assert(query.includes("to_jsonb(t) - 'poll_configuration_epoch'")) + f.facts.badTable = table; assert.equal((await f.run()).currentPristineDataInvariants, false) + } + assert(!f.exports.CONTROLLED_SCHEMA_VERIFICATION_SQL.seed("servers_api_ping_settings").includes("poll_configuration_epoch")) + f.facts.badTable = "" + const evidence = await f.run() + assert(evidence.gaps.includes("NONDETERMINISTIC_POLL_EPOCH_ORIGIN_UNATTESTED")); assert.equal(evidence.ready, false) +}) diff --git a/backend/src/db/controlled-schema-verification.ts b/backend/src/db/controlled-schema-verification.ts new file mode 100644 index 00000000..52250ce0 --- /dev/null +++ b/backend/src/db/controlled-schema-verification.ts @@ -0,0 +1,142 @@ +import { fileURLToPath } from "node:url" +import type { PoolClient } from "pg" +import { acquireDatabaseSchemaControl } from "./database-schema-control.js" +import type { DatabaseSchemaIntent, DatabaseSchemaTerminal } from "./database-schema-control-model.js" +import { CONTROLLED_MIGRATIONS_SQL } from "./controlled-migrations.js" +import { exactHistory } from "./controlled-migration-history.js" +import { CURRENT_REQUIRED_TABLES, inventoryMigrationDirectory, MIGRATION_MANIFEST, migrationManifestDigest } from "./migration-manifest.js" +import { captureMigrationSchema, compareMigrationSchemas, type CapturedMigrationSchema } from "./migration-schema-inventory.js" +import { readControlledSchemaReference } from "./controlled-schema-reference.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js" + +/** Current pristine-bootstrap invariants derived from the exact pinned INSERTs. + * Timestamps are checked separately as CURRENT finite/equal values, never as + * proof of the nondeterministic time originally assigned by a migration. */ +const seeds: Readonly>>> = Object.freeze({ + server_probe_storage_usage: { scope: "global", admissions: 0, reserved_files: 0, reserved_bytes: 0 }, + traffic_settings: { id: 1, enabled: true, interval_sec: 30, retention_days: 14, last_collected_at: null, last_duration_ms: null, last_error: null }, + servers_api_ping_settings: { id: 1, enabled: false, interval_sec: 120, last_collected_at: null, last_duration_ms: null, last_error: null }, + traffic_flow_settings: { id: 1, enabled: false, collector_ip: "10.255.254.1", flow_listen_port: 4739, wg_listen_port: 51821, prefix: "10.255.254.0/24", public_endpoint: "", host_public_key: "", host_private_key: "", hub_server_id: null, retention_hours: 24, top_n: 200, map_service_min_share_pct: 5, last_datagram_at: null, last_exporter_ip: null, last_error: null, packets_received: 0, peers_json: [] }, + uptime_settings: { id: 1, enabled: true, resources_enabled: true, ping_enabled: true, speed_enabled: true, interval_sec: 15, probe_interval_sec: 15, speed_interval_sec: 60, retention_days: 14, last_collected_at: null, last_duration_ms: null, last_error: null }, + evobgp_settings: { id: 1, base_url: "", api_key: "", enabled: false }, + alert_telegram_settings: { id: 1, bot_token: "", chat_id: "", message_thread_id: null }, + acme_settings: { id: 1, directory_url: "https://acme-v02.api.letsencrypt.org/directory", cloudflare_api_token: "", default_zone_id: "", account_private_key: "" }, + certificate_renew_settings: { id: 1, enabled: true, interval_sec: 21600, renew_before_days: 30, last_collected_at: null, last_duration_ms: null, last_error: null }, + backup_schedule_settings: { id: 1, enabled: true, frequency: "daily", hour: 3, minute: 0, week_day: 0, month_day: 1, keep_count: 7, format: "rsc", server_ids_json: [], last_run_at: null, last_duration_ms: null, last_error: null }, + internet_path_settings: { id: 1, enabled: true, interval_sec: 300, retention_days: 14, last_collected_at: null, last_duration_ms: null, last_error: null }, + alert_engine_cursor: { id: 1, last_source_finished_at: null }, + data_migration: { id: 1, sqlite_imported_at: null, sqlite_path: null, sqlite_sha256: null, report_json: null }, + backup_storage_settings: { id: 1, provider: "local", s3_endpoint: "", s3_region: "us-east-1", s3_bucket: "", s3_prefix: "mikrotik", s3_access_key_id: "", s3_secret_access_key: "", s3_force_path_style: true, keep_local_copy: true, last_test_at: null, last_test_error: null }, + geoip_settings: { id: 1, enabled: true, update_interval_sec: 604800, last_check_at: null, last_success_at: null, last_error: null, country_build_at: null, asn_build_at: null, etags_json: {} }, + router_log_settings: { id: 1, enabled: false, interval_sec: 10, retention_days: 14 }, + server_poll_storage_usage: { scope: "global", admissions: 0, reserved_rows: 0, reserved_bytes: 0, compacted_admissions: 0, archive_bytes: 0 }, + mcp_settings: { id: 1, enabled: false, requests_retention_days: 30, audit_retention_days: 180 }, +}) +// Migration0042 assigns fresh UUIDv4 epochs independently in each database. +// Check their current shape separately; this never attests their origin/history. +const epochSeeds = new Set(["traffic_settings", "uptime_settings", "internet_path_settings"]) +const bothTimes = new Set(["traffic_settings", "servers_api_ping_settings", "traffic_flow_settings", "uptime_settings", "internet_path_settings", "geoip_settings"]) +const noTimes = new Set(["data_migration", "router_log_settings", "server_probe_storage_usage", "server_poll_storage_usage"]) +const metadata = "pg_catalog.pg_backend_pid()::text AS backend_pid, pg_catalog.transaction_timestamp()::text AS transaction_started" +const identifier = (value: string) => { if (!/^[a-z_][a-z_0-9]{0,62}$/.test(value)) throw new ControlledSchemaVerificationError("Unsupported pinned data identifier"); return '"' + value + '"' } +export const CONTROLLED_SCHEMA_VERIFICATION_SQL = Object.freeze({ + context: `SELECT ${metadata}`, + empty: (table: string) => `/* controlled-schema-data:empty:${table} */ SELECT ${metadata}, NOT EXISTS (SELECT 1 FROM public.${identifier(table)} LIMIT 1) AS valid`, + seed: (table: string) => { + if (!Object.hasOwn(seeds, table)) throw new ControlledSchemaVerificationError("Unknown pinned seed table") + const times = noTimes.has(table) ? [] : bothTimes.has(table) ? ["created_at", "updated_at"] : ["updated_at"] + const timeCheck = times.length ? " AND " + times.map(name => `pg_catalog.isfinite(t.${identifier(name)})`).join(" AND ") + (times.length === 2 ? " AND t.created_at = t.updated_at" : "") : "" + const epoch = epochSeeds.has(table) + const projection = epoch ? "(pg_catalog.to_jsonb(t) - 'poll_configuration_epoch')" : "pg_catalog.to_jsonb(t)" + const epochCheck = epoch ? " AND t.poll_configuration_epoch IS NOT NULL AND t.poll_configuration_epoch::text ~ '^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$'" : "" + return `/* controlled-schema-data:seed:${table} */ SELECT ${metadata}, (SELECT count(*) = 1 AND COALESCE(pg_catalog.bool_and((${projection} - $2::text[]) = $1::jsonb${timeCheck}${epochCheck}), false) FROM (SELECT * FROM public.${identifier(table)} LIMIT 2) t) AS valid` + }, + sequence: (name: string) => `/* controlled-schema-data:sequence:${name} */ SELECT ${metadata}, last_value::text = $1::text AND is_called IS FALSE AS valid FROM public.${identifier(name)}`, +}) +export class ControlledSchemaVerificationError extends Error { readonly code = "CONTROLLED_SCHEMA_VERIFICATION_REFUSED" } +const refuse = (message: string): never => { throw new ControlledSchemaVerificationError(message) } +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item) + +/** Fixed production I/O, evidence only. No caller reference/provider/SQL, + * provisioning, application writes, READY publication, or data issuer exists. + * The exclusive lease fences adopted writers; all-replica quiescence remains + * an explicit runtime gap before any future durable READY grant. */ +export async function verifyControlledSchemaCurrentState() { + await inventoryMigrationDirectory(fileURLToPath(new URL("../../drizzle/", import.meta.url))) + const manifest = migrationManifestDigest() + let reference: CapturedMigrationSchema | null = null, referenceStatus: "missing" | "invalid" | "matched" | "different" = "missing" + try { reference = await readControlledSchemaReference() } catch { referenceStatus = "invalid" } + const { pool } = await import("./index.js") + const control = await acquireDatabaseSchemaControl(pool) + let app: PoolClient | undefined, released = false + try { + app = await pool.connect() + await app.query("SET lock_timeout = '10s'") + await app.query("SELECT pg_catalog.pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + const state = (await control.read()).state + await app.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + for (const sql of ["SET LOCAL search_path = pg_catalog", "SET LOCAL TimeZone = 'UTC'", "SET LOCAL DateStyle = 'ISO, YMD'", "SET LOCAL standard_conforming_strings = on", "SET LOCAL statement_timeout = '5s'"]) await app.query(sql) + const contextRows = (await app.query(CONTROLLED_SCHEMA_VERIFICATION_SQL.context)).rows + if (contextRows.length !== 1 || Object.keys(contextRows[0]).sort().join(" ") !== "backend_pid transaction_started" || typeof contextRows[0].backend_pid !== "string" || !/^\d+$/.test(contextRows[0].backend_pid) || typeof contextRows[0].transaction_started !== "string" || !Number.isFinite(Date.parse(contextRows[0].transaction_started))) refuse("Read-only evidence transaction context unavailable") + const context = contextRows[0] + const history = (await app.query(CONTROLLED_MIGRATIONS_SQL.history)).rows + if (await exactHistory(history, state, manifest, true) !== MIGRATION_MANIFEST.length) refuse("Complete controlled file ledger required") + const fileHashes = new Set(MIGRATION_MANIFEST.map(pin => pin.sha256)) + const legacy = history.find(row => !fileHashes.has((row.intent as DatabaseSchemaIntent).artifactSha256)) + if (legacy) { + const { rows: quarantine } = await app.query("SELECT operation_id FROM mmapp_control.database_restore_quarantine WHERE id=1") + if (!(legacy.terminal as DatabaseSchemaTerminal).legacyRelease || quarantine.length !== 1 || quarantine[0].operation_id !== null) { + refuse("Authenticated legacy migration release and open quarantine required") + } + } + const schema = await captureMigrationSchema(app, { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["mmapp_control"] } }) + const markerTable = schema.sections.relations.find(row => row.namespace === "public" && row.name === "schema_migrations") + const markerId = schema.sections.columns.find(row => row.namespace === "public" && row.relation === "schema_migrations" && row.name === "id" && row.dropped === false) + if (!markerTable || markerTable.kind !== "r" || markerTable.persistence !== "p" || markerTable.row_security !== false || markerTable.force_row_security !== false || !markerId || markerId.type !== "text") refuse("Ordinary text migration marker relation required") + const markers = (await app.query(CONTROLLED_MIGRATIONS_SQL.markers)).rows + if (canonical(markers) !== canonical(MIGRATION_MANIFEST.map(pin => ({ id: pin.file.slice(0, -4) })))) refuse("Markers disagree with complete controlled ledger") + if (reference) { + try { compareMigrationSchemas(reference, schema); referenceStatus = "matched" } catch { referenceStatus = "different" } + } + const dataFacts: { table: string; valid: boolean; profile: "seed" | "empty" | "marker" }[] = [] + const sequenceFacts: { sequence: string; valid: boolean }[] = [] + const checked = async (query: string, values: unknown[] = []) => { + const rows = (await app!.query(query, values)).rows + if (rows.length !== 1 || Object.keys(rows[0]).sort().join(" ") !== "backend_pid transaction_started valid" || rows[0].backend_pid !== context.backend_pid || rows[0].transaction_started !== context.transaction_started || typeof rows[0].valid !== "boolean") refuse("Data observation escaped the owned evidence snapshot") + return rows[0].valid as boolean + } + // An untrusted schema could turn a name into a view/function or custom + // output type. Read application data only after trusted reference equality. + if (referenceStatus === "matched") { + for (const table of CURRENT_REQUIRED_TABLES) { + if (table === "schema_migrations") { dataFacts.push({ table, valid: true, profile: "marker" }); continue } + const seed = seeds[table] + const times = noTimes.has(table) ? [] : bothTimes.has(table) ? ["created_at", "updated_at"] : ["updated_at"] + const valid = seed ? await checked(CONTROLLED_SCHEMA_VERIFICATION_SQL.seed(table), [JSON.stringify(seed), times]) : await checked(CONTROLLED_SCHEMA_VERIFICATION_SQL.empty(table)) + dataFacts.push({ table, valid, profile: seed ? "seed" : "empty" }) + } + for (const sequence of schema.sections.sequences) { + if (sequence.namespace !== "public" || typeof sequence.name !== "string" || typeof sequence.start !== "string") return refuse("Unsupported pinned sequence state") + sequenceFacts.push({ sequence: sequence.name, valid: await checked(CONTROLLED_SCHEMA_VERIFICATION_SQL.sequence(sequence.name), [sequence.start]) }) + } + } + const committed = await app.query("COMMIT") + if (committed.command !== "COMMIT") refuse("Read-only evidence COMMIT unconfirmed") + const unlock = await app.query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY]) + if (unlock.rows.length !== 1 || unlock.rows[0].unlocked !== true) refuse("Evidence lease release unconfirmed") + await app.query("RESET lock_timeout") + app.release(false); released = true + await control.close() + return Object.freeze({ format: "controlled-schema-current-evidence-v1" as const, manifestSha256: manifest, authenticatedFiles: MIGRATION_MANIFEST.length, + referenceStatus, schemaSha256: schema.sha256, dataFacts, sequenceFacts, + currentPristineDataInvariants: referenceStatus === "matched" && dataFacts.every(fact => fact.valid) && sequenceFacts.every(fact => fact.valid), + ready: false as const, schemaAndDataVerified: false as const, historicalDataEquivalent: false as const, + gaps: [...(referenceStatus !== "matched" ? ["TRUSTED_SCHEMA_REFERENCE_" + referenceStatus.toUpperCase()] : []), + "HISTORICAL_DATA_AND_NONDETERMINISTIC_TIMESTAMP_RECEIPT_MISSING", "NONDETERMINISTIC_POLL_EPOCH_ORIGIN_UNATTESTED", "IMPORT_RESTORE_DATA_EQUIVALENCE_UNPROVEN", "ALL_REPLICA_QUIESCENCE_UNATTESTED", "DURABLE_READY_PROTOCOL_UNIMPLEMENTED"], + }) + } catch (cause) { + if (app && !released) { released = true; app.release(true) } + try { await control.close() } catch { /* preserve primary evidence failure */ } + throw new ControlledSchemaVerificationError("Controlled current-state verification refused", { cause }) + } +} diff --git a/backend/src/db/database-restore-data.ts b/backend/src/db/database-restore-data.ts new file mode 100644 index 00000000..4b52ba87 --- /dev/null +++ b/backend/src/db/database-restore-data.ts @@ -0,0 +1,44 @@ +import { createHash } from "node:crypto" +import type { Pool } from "pg" +import { withRuntimeCatalogSnapshot } from "./database-runtime-schema.js" +import { captureMigrationData } from "./migration-data-inventory.js" +import type { DatabaseOperationLockClient } from "./operation-lock.js" + +export class DatabaseRestoreDataRefused extends Error { + readonly code = "DATABASE_RESTORE_DATA_REFUSED" + readonly statusCode = 503 + constructor() { super("Архив или восстановленные данные PostgreSQL не соответствуют утверждённому снимку") } +} + +const sha256 = /^[a-f0-9]{64}$/ + +export function approvedRestoreDataDigest(archive: Buffer, quarantine: boolean) { + const archiveDigest = process.env.DATABASE_RESTORE_ARCHIVE_SHA256 + const dataDigest = process.env.DATABASE_RESTORE_DATA_SHA256 + if (!quarantine || !archiveDigest || !sha256.test(archiveDigest) || !dataDigest || !sha256.test(dataDigest) || + createHash("sha256").update(archive).digest("hex") !== archiveDigest) throw new DatabaseRestoreDataRefused() + return { archiveSha256: archiveDigest, dataSha256: dataDigest } +} + +export async function captureRuntimeRestoreData( + client: Pick, discard: () => void, targetCheckSql?: string, +) { + try { + return await withRuntimeCatalogSnapshot(client, discard, (sql, hasControl) => captureMigrationData(sql, { + scope: { applicationNamespaces: hasControl ? ["public", "mmapp_control"] : ["public"] }, + dataNamespaces: ["public"], + includeControlEventTriggers: hasControl, + }), targetCheckSql) + } catch { throw new DatabaseRestoreDataRefused() } +} + +export async function assertRuntimeRestoreData(pool: Pick, expected: string, targetCheckSql: string): Promise { + try { + const client = await pool.connect() + let discard = false + try { + const snapshot = await captureRuntimeRestoreData(client, () => { discard = true }, targetCheckSql) + if (snapshot.sha256 !== expected) throw new DatabaseRestoreDataRefused() + } finally { client.release(discard) } + } catch { throw new DatabaseRestoreDataRefused() } +} diff --git a/backend/src/db/database-restore-quarantine-profile.ts b/backend/src/db/database-restore-quarantine-profile.ts new file mode 100644 index 00000000..581d0d1f --- /dev/null +++ b/backend/src/db/database-restore-quarantine-profile.ts @@ -0,0 +1,70 @@ +import { DATABASE_RESTORE_QUARANTINE_DDL } from "./database-restore-quarantine.js" + +export const RESTORE_WRITER_PROFILE_DEFINITION_SHA256 = "cb0dc0b71b291e25fdd1e6f071f7b55f53b9710b98f89dea90fd3d9d69250628" +export const RESTORE_WRITER_PROFILE_MANIFEST_SHA256 = "7dbd678e59740c14a29594180c86b85b3b2dd53fd5aa7a72da6d14a513760dbc" + +const relation = "mmapp_control.database_restore_quarantine" +const registry = "mmapp_control.application_writer_profile_current, mmapp_control.application_writer_profile_history" + +interface Sql { + query(sql: string, values?: unknown[]): Promise<{ rows: Record[] }> +} + +const identifier = (name: string) => { + if (!/^[a-z_][a-z0-9_]{0,62}$/.test(name)) throw new Error("Invalid restore profile role") + return `"${name}"` +} + +/** Run only as the control schema owner in an offline, retained operator transaction. */ +export function restoreProfileEnrollmentSql(runtime: string, operator: string): string { + return `${DATABASE_RESTORE_QUARANTINE_DDL} +REVOKE ALL ON ${relation} FROM PUBLIC; +GRANT USAGE ON SCHEMA mmapp_control TO ${identifier(runtime)}; +GRANT SELECT ON ${relation}, ${registry} TO ${identifier(runtime)}; +GRANT SELECT, UPDATE (operation_id, journal_prefix_bytes, journal_prefix_sha256) ON ${relation} TO ${identifier(operator)};` +} + +/** Fixed control relation DATA evidence. Table contents do not enter the signed profile. */ +export async function assertRestoreProfileQuarantine(sql: Sql, runtime: string, operator: string, control: string): Promise { + const { rows } = await sql.query(`WITH q AS MATERIALIZED ( + SELECT c.*,n.nspowner FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='mmapp_control' AND c.relname='database_restore_quarantine' + ) SELECT ( + q.relkind='r' AND q.relpersistence='p' AND q.relowner=$3::pg_catalog.regrole AND q.nspowner=q.relowner + AND NOT (q.relrowsecurity OR q.relforcerowsecurity OR q.relispartition OR q.relhasrules OR q.relhastriggers) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_inherits i WHERE q.oid IN (i.inhrelid,i.inhparent)) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_policy p WHERE p.polrelid=q.oid) + AND ARRAY(SELECT a.attname::text||':'||a.atttypid::pg_catalog.text||':'||a.attnotnull::text + FROM pg_catalog.pg_attribute a WHERE a.attrelid=q.oid AND a.attnum>0 ORDER BY a.attnum) + =ARRAY['id:21:true','operation_id:2950:false','journal_prefix_bytes:23:false','journal_prefix_sha256:25:false'] + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_attribute a WHERE a.attrelid=q.oid AND a.attnum>0 AND + (a.atthasdef OR a.attisdropped OR a.attidentity<>'' OR a.attgenerated<>'')) + AND ARRAY(SELECT pg_catalog.pg_get_constraintdef(k.oid,true) FROM pg_catalog.pg_constraint k + WHERE k.conrelid=q.oid AND k.contype<>'n' AND k.convalidated AND NOT k.condeferrable ORDER BY k.contype,k.conname) + =ARRAY['CHECK (id = 1)',$4::text,'PRIMARY KEY (id)'] + AND (SELECT count(*) FROM pg_catalog.pg_constraint k WHERE k.conrelid=q.oid AND k.contype<>'n')=3 + AND (SELECT count(*) FROM pg_catalog.pg_index i WHERE i.indrelid=q.oid)=1 + AND ARRAY(SELECT pg_catalog.format('%s:%s', a.grantee, a.privilege_type) + FROM pg_catalog.aclexplode(q.relacl) a WHERE a.grantor=q.relowner AND NOT a.is_grantable AND a.grantee<>q.relowner + ORDER BY 1) = ARRAY(SELECT x FROM pg_catalog.unnest(ARRAY[ + ($1::pg_catalog.regrole::pg_catalog.oid::text||':SELECT'),($2::pg_catalog.regrole::pg_catalog.oid::text||':SELECT')]) x ORDER BY x) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.aclexplode(q.relacl) a + WHERE a.grantor<>q.relowner OR a.is_grantable) + AND (SELECT count(*) FROM pg_catalog.pg_attribute c CROSS JOIN LATERAL pg_catalog.aclexplode(c.attacl) a + WHERE c.attrelid=q.oid AND c.attnum>0)=3 + AND (SELECT count(*) FROM pg_catalog.pg_attribute c CROSS JOIN LATERAL pg_catalog.aclexplode(c.attacl) a + WHERE c.attrelid=q.oid AND c.attnum IN(2,3,4) AND a.grantor=q.relowner AND a.grantee=$2::pg_catalog.regrole + AND a.privilege_type='UPDATE' AND NOT a.is_grantable)=3 + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_class c WHERE c.relnamespace=q.relnamespace AND c.oid<>q.oid + AND CASE WHEN c.relkind='S' THEN pg_catalog.has_sequence_privilege($1,c.oid,'SELECT,UPDATE,USAGE') + WHEN c.relkind IN ('r','p','v','m','f') THEN + pg_catalog.has_table_privilege($1,c.oid,'INSERT,UPDATE,DELETE,TRUNCATE,REFERENCES,TRIGGER,MAINTAIN') + OR (c.relname NOT IN ('application_writer_profile_current','application_writer_profile_history') + AND pg_catalog.has_any_column_privilege($1,c.oid,'SELECT,INSERT,UPDATE,REFERENCES')) + ELSE false END) + AND NOT EXISTS(SELECT 1 FROM pg_catalog.pg_proc p WHERE p.pronamespace=q.relnamespace + AND pg_catalog.has_function_privilege($1,p.oid,'EXECUTE')) + ) AS valid FROM q`, [runtime, operator, control, + "CHECK ((journal_prefix_bytes IS NULL) = (journal_prefix_sha256 IS NULL) AND journal_prefix_bytes >= 1 AND journal_prefix_bytes <= 8388608 AND journal_prefix_sha256 ~ '^[a-f0-9]{64}$'::text)"]) + if (rows.length !== 1 || rows[0].valid !== true) throw new Error("Protected restore quarantine structure or ACL refused") +} diff --git a/backend/src/db/database-restore-quarantine.test.ts b/backend/src/db/database-restore-quarantine.test.ts new file mode 100644 index 00000000..8eb573ac --- /dev/null +++ b/backend/src/db/database-restore-quarantine.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { + assertDatabaseRestoreQuarantineOpen, beginDatabaseRestoreQuarantine, + completeDatabaseRestoreQuarantine, readDatabaseRestoreJournalHead, +} from "./database-restore-quarantine.js" + +const id = "cce2a99d-bd85-4311-9d0a-6e7022fab650" +const otherId = "6a8ad2ec-dfcd-4d44-8f93-7234e3853b78" +const refusal = { code: "DATABASE_RESTORE_QUARANTINE_UNAVAILABLE", statusCode: 503 } +const clientReturning = (...rows: Record[]) => ({ query: async () => ({ rows }) }) + +test("admission requires exactly one open row and hides connection details", async () => { + const open = { operation_id: null } + await assertDatabaseRestoreQuarantineOpen(clientReturning(open)) + for (const rows of [[], [{}], [{ operation_id: id }], [{ ...open, unexpected: true }], [open, open]]) { + await assert.rejects(assertDatabaseRestoreQuarantineOpen(clientReturning(...rows)), refusal) + } + await assert.rejects(assertDatabaseRestoreQuarantineOpen({ query: async () => { throw new Error("private SQL detail") } }), error => { + assert.deepEqual({ code: (error as { code?: string }).code, statusCode: (error as { statusCode?: number }).statusCode }, refusal) + assert.doesNotMatch(String(error), /private SQL detail/) + return true + }) +}) + +test("begin and completion compare against the previous claim and require an exact acknowledgement", async () => { + const calls: (string | null)[][] = [] + const client = { query: async (sql: string, values: (string | null)[]) => { + assert.match(sql, /operation_id IS NOT DISTINCT FROM \$2::pg_catalog.uuid/) + calls.push(values) + return { rows: [{ operation_id: values[0] }] } + } } + await beginDatabaseRestoreQuarantine(client, id) + await completeDatabaseRestoreQuarantine(client, id) + await assert.rejects(beginDatabaseRestoreQuarantine(client, "invalid"), refusal) + await assert.rejects(completeDatabaseRestoreQuarantine(client, "invalid"), refusal) + assert.deepEqual(calls, [[id, null], [null, id]]) + for (const rows of [[], [{ operation_id: null }], [{ operation_id: otherId }], [{ operation_id: id }, { operation_id: id }]]) { + await assert.rejects(beginDatabaseRestoreQuarantine(clientReturning(...rows), id), refusal) + } + for (const rows of [[], [{ operation_id: id }], [{ operation_id: null }, { operation_id: null }]]) { + await assert.rejects(completeDatabaseRestoreQuarantine(clientReturning(...rows), id), refusal) + } +}) + +test("journal head distinguishes the old schema from a provisioned empty head", async () => { + const read = (columns: Record) => readDatabaseRestoreJournalHead(clientReturning({ quarantine: { id: 1, ...columns } })) + const sha256 = "a".repeat(64) + assert.equal(await read({ operation_id: null }), undefined) + assert.equal(await read({ journal_prefix_bytes: null, journal_prefix_sha256: null }), null) + assert.deepEqual(await read({ journal_prefix_bytes: 42, journal_prefix_sha256: sha256 }), { bytes: 42, sha256 }) + for (const columns of [ + { journal_prefix_bytes: null }, + { journal_prefix_bytes: null, journal_prefix_sha256: sha256 }, + { journal_prefix_bytes: 42, journal_prefix_sha256: null }, + { journal_prefix_bytes: 0, journal_prefix_sha256: sha256 }, + { journal_prefix_bytes: 8_388_609, journal_prefix_sha256: sha256 }, + { journal_prefix_bytes: "42", journal_prefix_sha256: sha256 }, + { journal_prefix_bytes: 42, journal_prefix_sha256: sha256 + "\n" }, + ]) await assert.rejects(read(columns), refusal) + await assert.rejects(readDatabaseRestoreJournalHead(clientReturning()), refusal) + const client = { query: async () => { throw Object.assign(new Error("missing relation"), { code: "42P01" }) } } + await assert.rejects(readDatabaseRestoreJournalHead(client), refusal) +}) + +test("signed transitions compare and acknowledge both the operation and journal head", async () => { + const first = { bytes: 42, sha256: "a".repeat(64) } + const applied = { bytes: 76, sha256: "b".repeat(64) } + const calls: (string | null)[][] = [] + const client = { query: async (sql: string, values: (string | null)[]) => { + assert.match(sql, /operation_id IS NOT DISTINCT FROM \$2::pg_catalog.uuid/) + assert.match(sql, /journal_prefix_bytes IS NOT DISTINCT FROM \$5::pg_catalog.int4/) + assert.match(sql, /journal_prefix_sha256 IS NOT DISTINCT FROM \$6::pg_catalog.text/) + calls.push(values) + return { rows: [{ operation_id: values[0], journal_prefix_bytes: Number(values[2]), journal_prefix_sha256: values[3] }] } + } } + await beginDatabaseRestoreQuarantine(client, id, { expected: null, next: first }) + await completeDatabaseRestoreQuarantine(client, id, { expected: first, next: applied }) + await assert.rejects(completeDatabaseRestoreQuarantine(client, id, { expected: applied, next: first }), refusal) + assert.deepEqual(calls, [[id, null, "42", first.sha256, null, null], [null, id, "76", applied.sha256, "42", first.sha256]]) + await assert.rejects(completeDatabaseRestoreQuarantine(clientReturning(), id, { expected: first, next: applied }), refusal) + await assert.rejects(completeDatabaseRestoreQuarantine(clientReturning({ + operation_id: null, journal_prefix_bytes: applied.bytes, journal_prefix_sha256: first.sha256, + }), id, { expected: first, next: applied }), refusal) +}) diff --git a/backend/src/db/database-restore-quarantine.ts b/backend/src/db/database-restore-quarantine.ts new file mode 100644 index 00000000..eeed2ee5 --- /dev/null +++ b/backend/src/db/database-restore-quarantine.ts @@ -0,0 +1,141 @@ +/** Provision outside application migrations, after the control schema exists. */ +export const DATABASE_RESTORE_QUARANTINE_DDL = ` +CREATE TABLE mmapp_control.database_restore_quarantine ( + id smallint PRIMARY KEY CHECK (id = 1), + operation_id uuid, + journal_prefix_bytes integer, + journal_prefix_sha256 text, + CONSTRAINT database_restore_quarantine_journal_head CHECK ( + (journal_prefix_bytes IS NULL) = (journal_prefix_sha256 IS NULL) + AND journal_prefix_bytes BETWEEN 1 AND 8388608 + AND journal_prefix_sha256 ~ '^[a-f0-9]{64}$' + ) +); +INSERT INTO mmapp_control.database_restore_quarantine (id) VALUES (1); +` + +interface QuarantineClient { + query(sql: string, values: (string | null)[]): Promise +} + +export interface RestoreJournalHead { + bytes: number + sha256: string +} + +interface RestoreJournalHeadAdvance { + expected: RestoreJournalHead | null + next: RestoreJournalHead +} + +const operationId = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/ +// A transaction holding a writer key must see the currently committed claim. +const readSql = `SELECT operation_id::pg_catalog.text AS operation_id +FROM mmapp_control.database_restore_quarantine +WHERE id = 1 AND pg_catalog.current_setting('transaction_isolation') = 'read committed'` +// The restore caller commits the change with synchronous_commit enabled. +const updateSql = `UPDATE mmapp_control.database_restore_quarantine +SET operation_id = $1::pg_catalog.uuid +WHERE id = 1 AND operation_id IS NOT DISTINCT FROM $2::pg_catalog.uuid +RETURNING operation_id::pg_catalog.text AS operation_id` + +const readHeadSql = `SELECT pg_catalog.to_jsonb(quarantine) AS quarantine +FROM mmapp_control.database_restore_quarantine AS quarantine +WHERE id = 1 AND pg_catalog.current_setting('transaction_isolation') = 'read committed'` +const updateHeadSql = `UPDATE mmapp_control.database_restore_quarantine +SET operation_id = $1::pg_catalog.uuid, journal_prefix_bytes = $3::pg_catalog.int4, journal_prefix_sha256 = $4::pg_catalog.text +WHERE id = 1 AND operation_id IS NOT DISTINCT FROM $2::pg_catalog.uuid + AND journal_prefix_bytes IS NOT DISTINCT FROM $5::pg_catalog.int4 + AND journal_prefix_sha256 IS NOT DISTINCT FROM $6::pg_catalog.text +RETURNING operation_id::pg_catalog.text AS operation_id, journal_prefix_bytes, journal_prefix_sha256` + +function unavailable() { + return Object.assign(new Error("Состояние базы после восстановления не подтверждено"), { + code: "DATABASE_RESTORE_QUARANTINE_UNAVAILABLE", statusCode: 503, + }) +} + +export function databaseRestoreQuarantineEnabled(): boolean { + return process.env.DATABASE_RESTORE_QUARANTINE === "true" +} + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value) +} + +async function readSingleRow(client: QuarantineClient, sql: string, values: (string | null)[]): Promise> { + let result + try { result = await client.query(sql, values) } + catch { throw unavailable() } + if (!isRecord(result) || !Array.isArray(result.rows) || result.rows.length !== 1 || !isRecord(result.rows[0])) throw unavailable() + return result.rows[0] +} + +function validHead(bytes: unknown, sha256: unknown): boolean { + return typeof bytes === "number" && Number.isInteger(bytes) && bytes >= 1 && bytes <= 8_388_608 && + typeof sha256 === "string" && sha256.length === 64 && /^[a-f0-9]{64}$/.test(sha256) +} + +async function requireState(client: QuarantineClient, nextId: string | null, previousId: string | null, heads?: RestoreJournalHeadAdvance): Promise { + const values = [nextId, previousId] + if (!heads) { + const row = await readSingleRow(client, updateSql, values) + if (row.operation_id !== nextId || Object.keys(row).length !== 1) throw unavailable() + return + } + const { expected, next } = heads + if (!validHead(next.bytes, next.sha256) || (expected && !validHead(expected.bytes, expected.sha256)) || + next.bytes <= (expected?.bytes ?? 0)) throw unavailable() + const row = await readSingleRow(client, updateHeadSql, [ + ...values, String(next.bytes), next.sha256, expected ? String(expected.bytes) : null, expected?.sha256 ?? null, + ]) + if (row.operation_id !== nextId || row.journal_prefix_bytes !== next.bytes || + row.journal_prefix_sha256 !== next.sha256 || Object.keys(row).length !== 3) throw unavailable() +} + +export async function assertDatabaseRestoreQuarantineOpen(client: QuarantineClient): Promise { + const row = await readSingleRow(client, readSql, []) + if (row.operation_id !== null || Object.keys(row).length !== 1) throw unavailable() +} + +/** Undefined is the original schema. Null is a provisioned head before its first write. */ +async function readQuarantineRow(client: QuarantineClient) { + const row = await readSingleRow(client, readHeadSql, []) + const { quarantine } = row + if (Object.keys(row).length !== 1 || !isRecord(quarantine) || quarantine.id !== 1) throw unavailable() + return quarantine +} + +function journalHead(quarantine: Record): RestoreJournalHead | null | undefined { + const hasBytes = Object.hasOwn(quarantine, "journal_prefix_bytes") + const hasDigest = Object.hasOwn(quarantine, "journal_prefix_sha256") + if (!hasBytes && !hasDigest) return undefined + const { journal_prefix_bytes: bytes, journal_prefix_sha256: sha256 } = quarantine + if (bytes === null && sha256 === null) return null + if (!validHead(bytes, sha256)) throw unavailable() + return { bytes: bytes as number, sha256: sha256 as string } +} + +export async function readDatabaseRestoreJournalHead(client: QuarantineClient, expectedOperationId?: string): Promise { + const quarantine = await readQuarantineRow(client) + if (expectedOperationId !== undefined && (!operationId.test(expectedOperationId) || quarantine.operation_id !== expectedOperationId)) throw unavailable() + return journalHead(quarantine) +} + +export async function readDatabaseRestoreQuarantineStatus(client: QuarantineClient) { + const quarantine = await readQuarantineRow(client) + const activeOperationId = quarantine.operation_id + const head = journalHead(quarantine) + if (head === undefined || (activeOperationId !== null && (typeof activeOperationId !== "string" || !operationId.test(activeOperationId)))) throw unavailable() + return { activeOperationId, head } +} + +export async function beginDatabaseRestoreQuarantine(client: QuarantineClient, id: string, heads?: RestoreJournalHeadAdvance): Promise { + if (typeof id !== "string" || !operationId.test(id)) throw unavailable() + await requireState(client, id, null, heads) +} + +export async function completeDatabaseRestoreQuarantine(client: QuarantineClient, id: string, heads?: RestoreJournalHeadAdvance): Promise { + if (typeof id !== "string" || !operationId.test(id)) throw unavailable() + await requireState(client, null, id, heads) +} diff --git a/backend/src/db/database-runtime-schema.ts b/backend/src/db/database-runtime-schema.ts new file mode 100644 index 00000000..1c7e0c57 --- /dev/null +++ b/backend/src/db/database-runtime-schema.ts @@ -0,0 +1,78 @@ +import { captureMigrationSchema, type SchemaInventoryClient } from "./migration-schema-inventory.js" +import { assertSqlMigrationsApplied } from "./migrate.js" +import type { DatabaseOperationLockClient } from "./operation-lock.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js" + +export class DatabaseRuntimeSchemaRefused extends Error { + readonly code = "DATABASE_RUNTIME_SCHEMA_REFUSED" + readonly statusCode = 503 + constructor() { super("Фактическая схема PostgreSQL не соответствует утверждённому состоянию") } +} + +/** The outer operation already owns the global maintenance locks. */ +export async function withRuntimeCatalogSnapshot( + client: Pick, + discard: () => void, + work: (sql: SchemaInventoryClient, hasControl: boolean) => Promise, + targetCheckSql?: string, +): Promise { + const query = (sql: string, values: unknown[] = []) => client.query(sql, values) + let acquired = false, inSnapshot = false + try { + await query("SET lock_timeout = '10s'") + await query("SELECT pg_catalog.pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + acquired = true + if (targetCheckSql) await query(targetCheckSql) + // Start the snapshot after the writer drain so the wait cannot freeze stale catalog state. + if ((await query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY")).command !== "BEGIN") throw new DatabaseRuntimeSchemaRefused() + inSnapshot = true + for (const setting of [ + "SET LOCAL search_path = pg_catalog", "SET LOCAL TimeZone = 'UTC'", "SET LOCAL DateStyle = 'ISO, YMD'", + "SET LOCAL standard_conforming_strings = on", "SET LOCAL statement_timeout = '5s'", + "SET LOCAL quote_all_identifiers = off", "SET LOCAL IntervalStyle = 'postgres'", + "SET LOCAL extra_float_digits = 3", "SET LOCAL bytea_output = 'hex'", + "SET LOCAL row_security = off", "SET LOCAL client_encoding = 'UTF8'", + "SET LOCAL lc_monetary = 'C'", "SET LOCAL lc_numeric = 'C'", + ]) await query(setting) + const { rows } = await query("SELECT pg_catalog.to_regnamespace('mmapp_control') IS NOT NULL AS present") + if (rows.length !== 1 || typeof rows[0].present !== "boolean") throw new DatabaseRuntimeSchemaRefused() + const result = await work({ query }, rows[0].present) + if (targetCheckSql) await query(targetCheckSql) + return result + } catch { + discard() + throw new DatabaseRuntimeSchemaRefused() + } finally { + try { + if (inSnapshot && (await query("ROLLBACK")).command !== "ROLLBACK") throw new DatabaseRuntimeSchemaRefused() + if (acquired) { + const { rows } = await query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS released", [PARTITION_WRITER_ADVISORY_KEY]) + if (rows.length !== 1 || rows[0].released !== true) throw new DatabaseRuntimeSchemaRefused() + await query("RESET lock_timeout") + } + } catch { + discard() + throw new DatabaseRuntimeSchemaRefused() + } + } +} + +export async function captureRuntimeDatabaseSchema(client: Pick, discard: () => void, expected?: string) { + if (expected !== undefined && !/^[a-f0-9]{64}$/.test(expected)) throw new DatabaseRuntimeSchemaRefused() + return withRuntimeCatalogSnapshot(client, discard, async (sql, hasControl) => { + const snapshot = await captureMigrationSchema(sql, { + scope: { applicationNamespaces: hasControl ? ["public", "mmapp_control"] : ["public"] }, + includeControlEventTriggers: hasControl, + }) + if (expected !== undefined && snapshot.sha256 !== expected) throw new DatabaseRuntimeSchemaRefused() + const { relations, columns, inheritance } = snapshot.sections + const marker = relations.find(row => row.namespace === "public" && row.name === "schema_migrations") + const markerId = columns.find(row => row.namespace === "public" && row.relation === "schema_migrations" && row.name === "id") + if (!marker || marker.kind !== "r" || marker.persistence !== "p" || marker.row_security !== false || + marker.force_row_security !== false || marker.is_partition !== false || markerId?.type !== "text" || markerId.dropped !== false || + inheritance.some(row => row.parent_namespace === "public" && row.parent === "schema_migrations" || + row.namespace === "public" && row.child === "schema_migrations")) throw new DatabaseRuntimeSchemaRefused() + await assertSqlMigrationsApplied(client) + return snapshot + }) +} diff --git a/backend/src/db/database-runtime-source.ts b/backend/src/db/database-runtime-source.ts new file mode 100644 index 00000000..dd814e88 --- /dev/null +++ b/backend/src/db/database-runtime-source.ts @@ -0,0 +1,257 @@ +import { createHash, X509Certificate } from "node:crypto" +import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, type BigIntStats } from "node:fs" +import path from "node:path" +import { checkServerIdentity, TLSSocket, type PeerCertificate } from "node:tls" +import type { ClientBase, PoolConfig, QueryConfig } from "pg" +import { z } from "zod" +import { DATABASE_SOURCE_ACTIVATION_FILENAME, MAX_DATABASE_SOURCE_ACTIVATION_BYTES, parseDatabaseSourceActivationRecord } from "./database-source-activation-record.js" +import { approvedRuntimeWriterProfile } from "./database-runtime-writer-profile.js" + +const sha256 = (bytes: Buffer) => createHash("sha256").update(bytes).digest("hex") +const digest = z.string().regex(/^[a-f0-9]{64}$/) +const oid = z.string().regex(/^[1-9][0-9]{0,9}$/).refine(value => Number(value) <= 4294967295) +const roleBinding = z.object({ name: z.string().regex(/^[a-z_][a-z0-9_]{0,62}$/), signedOid: oid }) +const requestSchema = z.object({ + format: z.literal("application-writer-bootstrap-request-v1"), + installationId: z.string().uuid(), + sourceHeadSha256: digest, + source: z.object({ + systemIdentifier: z.string().regex(/^[1-9][0-9]{0,19}$/), + databaseOid: oid, + databaseName: z.string().regex(/^[^\x00-\x1f\x7f]{1,63}$/), + serverVersionNum: z.number().int().min(180000).max(189999), + inRecovery: z.literal(false), + }), + roles: z.object({ runtime: roleBinding, capture: roleBinding.optional(), operator: roleBinding.optional(), publisher: roleBinding.optional(), control: roleBinding.optional() }), + tls: z.object({ + serverName: z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}$/).transform(value => value.toLowerCase()), + caSha256: digest, spkiSha256: digest, + }), +}) + +export class DatabaseRuntimeSourceRefused extends Error { + readonly code = "DATABASE_RUNTIME_SOURCE_REFUSED" + readonly statusCode = 503 + constructor() { super("Источник PostgreSQL для приложения не подтверждён") } +} + +function sameFile(left: BigIntStats, right: BigIntStats): boolean { + return left.dev === right.dev && left.ino === right.ino && left.size === right.size && + left.mtimeNs === right.mtimeNs && left.ctimeNs === right.ctimeNs +} + +function pinnedFile(file: string, limit: number, expected: string, entry?: BigIntStats): Buffer { + if (!path.isAbsolute(file) || !digest.safeParse(expected).success) throw new DatabaseRuntimeSourceRefused() + const handle = openSync(file, constants.O_RDONLY | constants.O_NONBLOCK) + try { + const stat = fstatSync(handle) + if (!stat.isFile() || stat.size < 1 || stat.size > limit) throw new DatabaseRuntimeSourceRefused() + if (entry && !sameFile(entry, fstatSync(handle, { bigint: true }))) throw new DatabaseRuntimeSourceRefused() + const buffer = Buffer.alloc(stat.size + 1) + let length = 0 + while (length < buffer.length) { + const read = readSync(handle, buffer, length, buffer.length - length, null) + if (!read) break + length += read + } + const bytes = buffer.subarray(0, length) + if (length !== stat.size || sha256(bytes) !== expected) throw new DatabaseRuntimeSourceRefused() + return bytes + } finally { closeSync(handle) } +} + +// The pinned runtime receives direct grants and cannot assume or be assumed by another role. +const identitySql = `SELECT SESSION_USER::text AS session_user, CURRENT_USER::text AS current_user, + r.oid::text AS role_oid, d.datname::text AS database_name, d.oid::text AS database_oid, + (pg_catalog.pg_control_system()).system_identifier::text AS system_identifier, + pg_catalog.current_setting('server_version_num') AS version_num, + pg_catalog.pg_is_in_recovery() AS in_recovery +FROM pg_catalog.pg_database d CROSS JOIN pg_catalog.pg_roles r +WHERE d.datname=pg_catalog.current_database() AND r.rolname=SESSION_USER + AND r.rolcanlogin AND NOT (r.rolsuper OR r.rolreplication OR r.rolbypassrls OR r.rolcreaterole OR r.rolcreatedb) + AND r.oid<>d.datdba + AND NOT pg_catalog.has_database_privilege(r.oid,d.oid,'CREATE,TEMP') + AND pg_catalog.current_setting('session_replication_role')='origin' + AND pg_catalog.current_setting('event_triggers')='on' + AND NOT pg_catalog.has_parameter_privilege(r.oid,'session_replication_role','SET') + AND NOT pg_catalog.has_parameter_privilege(r.oid,'event_triggers','SET') + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid + ) + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace n WHERE pg_catalog.has_schema_privilege(r.oid,n.oid,'CREATE') + ) + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_shdepend dependency + WHERE dependency.dbid=d.oid AND dependency.refclassid='pg_catalog.pg_authid'::pg_catalog.regclass + AND dependency.refobjid=r.oid AND dependency.deptype='o' + )` + +interface RuntimeSourceFiles { + databaseUrl: string + requestFile?: string + requestSha256?: string + caFile?: string + activationDirectory?: string + activationSha256?: string + writerApprovalFile?: string + writerApprovalSha256?: string +} + +export type DatabaseArchivePeer = { + host: string; port: number; database: string; capture: z.output; leaf: string +} + +function activationCheck(input: RuntimeSourceFiles, request: z.output): (() => void) | undefined { + const { activationDirectory, activationSha256 } = input + if (activationDirectory === undefined && activationSha256 === undefined) return + if (!activationDirectory || !path.isAbsolute(activationDirectory) || !activationSha256 || !digest.safeParse(activationSha256).success) { + throw new DatabaseRuntimeSourceRefused() + } + const directory = path.resolve(activationDirectory) + const file = path.join(directory, DATABASE_SOURCE_ACTIVATION_FILENAME) + const assertNoPending = () => { + try { lstatSync(file + ".pending") } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return; throw error } + throw new DatabaseRuntimeSourceRefused() + } + return () => { + try { + const root = lstatSync(directory, { bigint: true }) + if (!root.isDirectory()) throw new DatabaseRuntimeSourceRefused() + assertNoPending() + const before = lstatSync(file, { bigint: true }) + if (!before.isFile()) throw new DatabaseRuntimeSourceRefused() + const record = parseDatabaseSourceActivationRecord(pinnedFile(file, MAX_DATABASE_SOURCE_ACTIVATION_BYTES, activationSha256, before)) + const head = record.generations.at(-1) + if (record.status !== "active" || record.installationId !== request.installationId || + head?.headSha256 !== request.sourceHeadSha256 || head.tlsSpkiSha256 !== request.tls.spkiSha256) { + throw new DatabaseRuntimeSourceRefused() + } + assertNoPending() + const after = lstatSync(file, { bigint: true }), rootAfter = lstatSync(directory, { bigint: true }) + if (!after.isFile() || !sameFile(before, after) || + !rootAfter.isDirectory() || rootAfter.ino !== root.ino || rootAfter.dev !== root.dev) throw new DatabaseRuntimeSourceRefused() + } catch { throw new DatabaseRuntimeSourceRefused() } + } +} + +function runtimeSocket(client: ClientBase): TLSSocket { + const socket = (client as ClientBase & { connection?: { stream?: unknown } }).connection?.stream + if (!(socket instanceof TLSSocket) || !socket.encrypted || !socket.authorized || socket.destroyed || !socket.writable) { + throw new DatabaseRuntimeSourceRefused() + } + return socket +} + +/** The externally pinned request is public input for verifying one connection. + * It conveys neither generation acceptance nor application READY. */ +export function readRuntimeDatabasePoolOptions(input: RuntimeSourceFiles): PoolConfig & { + checkActivation?: () => void + checkCheckout?: (client: ClientBase) => Promise + archivePeer?: (client: ClientBase) => DatabaseArchivePeer + readWriterHead?: (client: ClientBase) => Promise<{ epoch: string; sha256: string; issuedAt: number }> +} { + if (input.requestFile === undefined && input.requestSha256 === undefined && input.caFile === undefined && + input.activationDirectory === undefined && input.activationSha256 === undefined && + input.writerApprovalFile === undefined && input.writerApprovalSha256 === undefined) { + return { connectionString: input.databaseUrl } + } + try { + if (!input.requestFile || !input.requestSha256 || !input.caFile || + process.env.NODE_PG_FORCE_NATIVE || process.env.PGREPLICATION) throw new DatabaseRuntimeSourceRefused() + const bytes = pinnedFile(input.requestFile, 16_384, input.requestSha256) + const request = requestSchema.parse(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))) + const checkActivation = activationCheck(input, request) + const { source, roles: { runtime: role }, tls } = request + const writer = (() => { + if (input.writerApprovalFile === undefined && input.writerApprovalSha256 === undefined) return undefined + const { capture, operator, publisher, control } = request.roles + if (!input.writerApprovalFile || !input.writerApprovalSha256 || !checkActivation || !capture || !operator || !publisher || !control) { + throw new DatabaseRuntimeSourceRefused() + } + const roles = { runtime: role.signedOid, capture: capture.signedOid, operator: operator.signedOid, publisher: publisher.signedOid, control: control.signedOid } + if (new Set(Object.values(roles)).size !== 5) throw new DatabaseRuntimeSourceRefused() + return approvedRuntimeWriterProfile(pinnedFile(input.writerApprovalFile, 8192, input.writerApprovalSha256), { + installationId: request.installationId, source, roles, + }) + })() + const checkLive = checkActivation && (() => { + checkActivation() + try { writer?.assertLive() } catch { throw new DatabaseRuntimeSourceRefused() } + }) + const readWriterHead = writer && (async (client: ClientBase) => { + const socket = runtimeSocket(client) + checkActivation?.() + const head = await writer.check({ query: (text, values) => client.query({ text, values, query_timeout: 5000 } as QueryConfig) }) + if (runtimeSocket(client) !== socket) throw new DatabaseRuntimeSourceRefused() + checkLive?.() + return head + }) + const ca = pinnedFile(input.caFile, 65_536, tls.caSha256) + new X509Certificate(ca) + const uri = new URL(input.databaseUrl) + const user = decodeURIComponent(uri.username), password = decodeURIComponent(uri.password) + const database = decodeURIComponent(uri.pathname.slice(1)), port = Number(uri.port || 5432) + if (!/^postgres(?:ql)?:\/\/[^/?#]+\/[^/?#]+$/.test(input.databaseUrl) || /[\x00-\x20\x7f\\]/.test(input.databaseUrl) || + uri.hostname.toLowerCase() !== tls.serverName || user !== role.name || database !== source.databaseName || + !password || password.length > 4096 || /[\x00-\x1f\x7f]/.test(password) || !Number.isInteger(port) || port < 1 || port > 65535) { + throw new DatabaseRuntimeSourceRefused() + } + const verifyPeer = (host: string, certificate: PeerCertificate): Error | undefined => { + try { + if (host === tls.serverName && !checkServerIdentity(tls.serverName, certificate) && + sha256(new X509Certificate(certificate.raw).publicKey.export({ format: "der", type: "spki" })) === tls.spkiSha256) return + } catch { /* TLS and certificate diagnostics are internal. */ } + return new DatabaseRuntimeSourceRefused() + } + const checkCheckout = async (client: ClientBase) => { + const socket = runtimeSocket(client) + const probe: QueryConfig & { query_timeout: number } = { text: identitySql, query_timeout: 5_000 } + const { rows } = await client.query(probe) + if (runtimeSocket(client) !== socket) throw new DatabaseRuntimeSourceRefused() + const observed = rows[0] + if (rows.length !== 1 || observed.session_user !== role.name || observed.current_user !== role.name || + observed.role_oid !== role.signedOid || observed.database_name !== source.databaseName || + observed.database_oid !== source.databaseOid || observed.system_identifier !== source.systemIdentifier || + observed.version_num !== String(source.serverVersionNum) || observed.in_recovery !== false) { + throw new DatabaseRuntimeSourceRefused() + } + checkActivation?.() + await readWriterHead?.(client) + if (runtimeSocket(client) !== socket) throw new DatabaseRuntimeSourceRefused() + checkLive?.() + } + return { + checkActivation: checkLive, + checkCheckout, + readWriterHead, + archivePeer(client: ClientBase) { + try { + checkLive?.() + const socket = runtimeSocket(client), certificate = socket.getPeerCertificate() + const { capture } = request.roles + if (!capture || capture.name === role.name || capture.signedOid === role.signedOid || + socket.servername !== tls.serverName || verifyPeer(tls.serverName, certificate)) throw new DatabaseRuntimeSourceRefused() + const leaf = new X509Certificate(certificate.raw) + // libpq trusts this leaf directly. Reject a CA that could authorize a substitute. + if (leaf.ca || leaf.subject !== leaf.issuer || !leaf.verify(leaf.publicKey)) throw new DatabaseRuntimeSourceRefused() + return { host: tls.serverName, port, database, capture, leaf: leaf.toString() } + } catch { throw new DatabaseRuntimeSourceRefused() } + }, + host: tls.serverName, port, user, password, database, + options: "-c search_path=public", + sslnegotiation: "postgres", + client_encoding: "UTF8", + ssl: { ca, servername: tls.serverName, rejectUnauthorized: true, checkServerIdentity: verifyPeer }, + async onConnect(client: ClientBase) { + try { + const peer = runtimeSocket(client) + if (!peer.servername || verifyPeer(peer.servername, peer.getPeerCertificate())) throw new DatabaseRuntimeSourceRefused() + peer.disableRenegotiation() + await checkCheckout(client) + } catch { throw new DatabaseRuntimeSourceRefused() } + }, + } + } catch { throw new DatabaseRuntimeSourceRefused() } +} diff --git a/backend/src/db/database-runtime-writer-profile.test.ts b/backend/src/db/database-runtime-writer-profile.test.ts new file mode 100644 index 00000000..f71e6c28 --- /dev/null +++ b/backend/src/db/database-runtime-writer-profile.test.ts @@ -0,0 +1,37 @@ +import assert from "node:assert/strict" +import { HEAD_DOMAIN, POLICY_DOMAIN, sha } from "./application-writer-profile-format.js" +import { stagedModelClient, syntheticFixture } from "./application-writer-profile-registry-read-model.test.js" +import { approvedRuntimeWriterProfile } from "./database-runtime-writer-profile.js" +import { RESTORE_WRITER_PROFILE_DEFINITION_SHA256, RESTORE_WRITER_PROFILE_MANIFEST_SHA256 } from "./database-restore-quarantine-profile.js" + +const { subject, policy: basePolicy, state, signBody } = syntheticFixture("1") +const policy = { ...basePolicy, definitionSha256: RESTORE_WRITER_PROFILE_DEFINITION_SHA256, manifestSha256: RESTORE_WRITER_PROFILE_MANIFEST_SHA256 } +const expected = { installationId: subject.installationId, source: policy.source, roles: policy.roles } + +function publish(next: typeof policy, previous = state.latest[0]) { + const policyBytes = signBody(next, POLICY_DOMAIN) + const policyHash = sha(policyBytes) + const headBytes = signBody({ format: "application-writer-head-v1", installationId: next.installationId, + epoch: next.epoch, policySha256: policyHash, previousHeadSha256: next.epoch === "1" ? null : previous.head_sha256 }, HEAD_DOMAIN) + const top = { ...previous, epoch: next.epoch, head_sha256: sha(headBytes), head_bytes: headBytes, + policy_sha256: policyHash, previous_head_sha256: next.epoch === "1" ? null : previous.head_sha256, + previous_epoch: next.epoch === "1" ? null : previous.epoch, previous_installation_id: next.epoch === "1" ? null : previous.installation_id } + const current = { installation_id: top.installation_id, epoch: top.epoch, head_sha256: top.head_sha256 } + return { ...state, current: [current], latest: next.epoch === "1" ? [top] : [top, previous], + policy: [{ policy_sha256: policyHash, policy_bytes: policyBytes }] } +} + +const initial = publish(policy) +const approval = Buffer.from(JSON.stringify({ format: "database-runtime-writer-approval-v1", + publicKeyPem: subject.publicKeyPem, publicKeySpkiSha256: subject.publicKeySpkiSha256, + minimumEpoch: "1", minimumHeadSha256: initial.current[0].head_sha256 })) +const guard = approvedRuntimeWriterProfile(approval, expected) +assert.throws(guard.assertLive, /истёк/) +await guard.check(stagedModelClient(initial).client) +assert.doesNotThrow(guard.assertLive) +await assert.rejects(guard.check(stagedModelClient(publish({ ...policy, definitionSha256: basePolicy.definitionSha256 })).client), /не подтверждён/) +await assert.rejects(guard.check(stagedModelClient(publish({ ...policy, catalogSha256: "e".repeat(64) })).client), /откатилось/) +await assert.rejects(guard.check(stagedModelClient(publish({ ...policy, epoch: "2", roles: { ...policy.roles, capture: "99" } }, initial.latest[0])).client), /не подтверждён/) +await guard.check(stagedModelClient(publish({ ...policy, epoch: "2" }, initial.latest[0])).client) +await assert.rejects(guard.check(stagedModelClient(initial).client), /откатилось/) +console.log("Runtime writer profile: pinned genesis, role binding, renewal and rollback refusal passed") diff --git a/backend/src/db/database-runtime-writer-profile.ts b/backend/src/db/database-runtime-writer-profile.ts new file mode 100644 index 00000000..319e8a51 --- /dev/null +++ b/backend/src/db/database-runtime-writer-profile.ts @@ -0,0 +1,47 @@ +import { z } from "zod" +import { canonical, epoch, publicKey, type ApplicationWriterPolicy } from "./application-writer-profile-format.js" +import { readCurrentApplicationWriterProfile, type ApplicationWriterRegistryReadSql } from "./application-writer-profile-registry-read.js" +import { RESTORE_WRITER_PROFILE_DEFINITION_SHA256, RESTORE_WRITER_PROFILE_MANIFEST_SHA256 } from "./database-restore-quarantine-profile.js" + +const digest = z.string().regex(/^[a-f0-9]{64}$/) +const approvalSchema = z.object({ + format: z.literal("database-runtime-writer-approval-v1"), + publicKeyPem: z.string().min(1).max(4096), + publicKeySpkiSha256: digest, + minimumEpoch: z.string().refine(value => { try { return epoch(value) === value } catch { return false } }), + minimumHeadSha256: digest, +}).strict() + +export function approvedRuntimeWriterProfile( + bytes: Buffer, + expected: Pick, +) { + const approval = approvalSchema.parse(JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes))) + const subject = { ...approval, installationId: expected.installationId } + publicKey(subject) + let expiresAt = 0 + let acceptedEpoch = BigInt(approval.minimumEpoch) + let acceptedHead = approval.minimumHeadSha256 + return { + async check(client: ApplicationWriterRegistryReadSql) { + const current = await readCurrentApplicationWriterProfile(client, subject) + const policy = current?.policy + if (!current || !policy || policy.definitionSha256 !== RESTORE_WRITER_PROFILE_DEFINITION_SHA256 || + policy.manifestSha256 !== RESTORE_WRITER_PROFILE_MANIFEST_SHA256 || + canonical(policy.source) !== canonical(expected.source) || canonical(policy.roles) !== canonical(expected.roles)) { + throw new Error("Текущий подписанный профиль записи PostgreSQL не подтверждён") + } + const observedEpoch = BigInt(current.epoch) + if (observedEpoch < acceptedEpoch || (observedEpoch === acceptedEpoch && current.headSha256 !== acceptedHead)) { + throw new Error("Состояние подписанного профиля записи PostgreSQL откатилось") + } + acceptedEpoch = observedEpoch + acceptedHead = current.headSha256 + expiresAt = policy.notAfter + return { epoch: current.epoch, sha256: current.headSha256, issuedAt: policy.issuedAt } + }, + assertLive() { + if (Date.now() >= expiresAt) throw new Error("Срок подтверждённого профиля записи PostgreSQL истёк") + }, + } +} diff --git a/backend/src/db/database-schema-control-model.test.ts b/backend/src/db/database-schema-control-model.test.ts new file mode 100644 index 00000000..693d5c88 --- /dev/null +++ b/backend/src/db/database-schema-control-model.test.ts @@ -0,0 +1,54 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { checkDatabaseSchemaControlState, claimDatabaseSchemaOwnership, databaseSchemaAdmission, initialDatabaseSchemaControlState, transitionDatabaseSchemaControl, type DatabaseSchemaControlState } from "./database-schema-control-model.js" + +const owner = "00000000-0000-4000-8000-000000000001", nextOwner = "00000000-0000-4000-8000-000000000002" +const id = "00000000-0000-4000-8000-000000000003", hash = "a".repeat(64) +const claimed = () => claimDatabaseSchemaOwnership(initialDatabaseSchemaControlState(), owner) +const begin = (state = claimed()) => transitionDatabaseSchemaControl(state, owner, { type: "begin", id, kind: "restore", manifestSha256: hash, artifactSha256: hash }) +test("initial and freshly claimed state remain quarantined", () => { + assert.equal(databaseSchemaAdmission(initialDatabaseSchemaControlState()).ready, false) + assert.equal(claimed().epoch, 1) + assert.equal(databaseSchemaAdmission(claimed()).ready, false) +}) +test("success observation never opens readiness or proves data migration", () => { + const state = transitionDatabaseSchemaControl(begin(), owner, { type: "terminal", id, outcome: "observed-success", observationSha256: hash }) + assert.equal(state.phase, "TERMINAL") + assert.equal(databaseSchemaAdmission(state).reason, "SCHEMA_AND_DATA_VERIFIER_REQUIRED") +}) +test("restart preserves pending intent and cannot infer outcome from matching schema", () => { + const state = claimDatabaseSchemaOwnership(begin(), nextOwner) + assert.equal(state.intent?.epoch, 1) + assert.equal(state.epoch, 2) + assert.equal(databaseSchemaAdmission(state).reason, "UNRESOLVED_INTENT") + for (const outcome of ["observed-success", "observed-failure"] as const) assert.throws(() => transitionDatabaseSchemaControl(state, nextOwner, { type: "terminal", id, outcome, observationSha256: hash })) + const terminal = transitionDatabaseSchemaControl(state, nextOwner, { type: "terminal", id, outcome: "interrupted-unknown", observationSha256: hash }) + assert.equal(databaseSchemaAdmission(terminal).ready, false) +}) +test("fresh owner retains terminal for future independent verification", () => { + const terminal = transitionDatabaseSchemaControl(begin(), owner, { type: "terminal", id, outcome: "observed-success", observationSha256: hash }) + const restarted = claimDatabaseSchemaOwnership(terminal, nextOwner) + assert.deepEqual(restarted.terminal, terminal.terminal) + assert.equal(restarted.phase, "QUARANTINED") +}) +test("owner mismatch, unresolved intent, duplicate terminal and malformed commands refuse", () => { + assert.throws(() => transitionDatabaseSchemaControl(begin(), nextOwner, { type: "terminal", id, outcome: "observed-success", observationSha256: hash })) + assert.throws(() => begin(begin())) + const terminal = transitionDatabaseSchemaControl(begin(), owner, { type: "terminal", id, outcome: "observed-failure", observationSha256: hash }) + assert.throws(() => begin(terminal)) + assert.throws(() => transitionDatabaseSchemaControl(terminal, owner, { type: "terminal", id, outcome: "observed-failure", observationSha256: hash })) + assert.throws(() => transitionDatabaseSchemaControl(claimed(), owner, { type: "begin", id, kind: "migration", manifestSha256: "true", artifactSha256: hash })) +}) +test("unknown durable fields, forged READY, unsafe epochs and inconsistent phases refuse", () => { + for (const changes of [{ ready: true }, { phase: "READY" }, { epoch: Number.MAX_SAFE_INTEGER + 1 }, { phase: "INTENT" }, { phase: "TERMINAL" }, { owner: "a" }]) assert.throws(() => checkDatabaseSchemaControlState({ ...claimed(), ...changes })) + assert.throws(() => claimDatabaseSchemaOwnership({ ...claimed(), epoch: Number.MAX_SAFE_INTEGER }, nextOwner)) + assert.throws(() => claimDatabaseSchemaOwnership(claimed(), owner)) +}) +test("returned states are detached from untrusted caller mutation", () => { + const original = begin(), checked = checkDatabaseSchemaControlState(original) + checked.intent!.artifactSha256 = "b".repeat(64) + assert.equal(original.intent!.artifactSha256, hash) + const frozen = databaseSchemaAdmission(original) + assert(Object.isFrozen(frozen)) + assert.equal((original as DatabaseSchemaControlState).phase, "INTENT") +}) diff --git a/backend/src/db/database-schema-control-model.ts b/backend/src/db/database-schema-control-model.ts new file mode 100644 index 00000000..0125bc1b --- /dev/null +++ b/backend/src/db/database-schema-control-model.ts @@ -0,0 +1,115 @@ +/** Durable coordination only. Neither caller receipts nor schema equality are + * data-migration proof. There is deliberately no READY transition here until a + * fixed, independently reviewed schema AND data verifier is implemented. */ +export const DATABASE_SCHEMA_CONTROL_FORMAT = "database-schema-control-v1" as const +export type DatabaseSchemaOperation = "bootstrap" | "migration" | "import" | "restore" +export interface DatabaseSchemaIntent { + id: string + epoch: number + kind: DatabaseSchemaOperation + manifestSha256: string + artifactSha256: string +} +export interface DatabaseSchemaTerminal { + intent: DatabaseSchemaIntent + outcome: "observed-success" | "observed-failure" | "interrupted-unknown" + /** Observation content address or fixed issuer's SHA256-based keyed digest. + * The generic store never treats this field as an authorization capability; + * a specific dispatcher must independently verify its fixed issuer. */ + observationSha256: string + legacyRelease?: { approvalSha256: string; observationSha256: string } +} +export interface DatabaseSchemaControlState { + format: typeof DATABASE_SCHEMA_CONTROL_FORMAT + epoch: number + owner: string | null + phase: "QUARANTINED" | "INTENT" | "TERMINAL" + intent: DatabaseSchemaIntent | null + terminal: DatabaseSchemaTerminal | null +} +export class DatabaseSchemaControlError extends Error { + readonly code = "DATABASE_SCHEMA_CONTROL_REFUSED" + constructor(message: string, readonly outcomeUnknown = false, options?: ErrorOptions) { super(message, options) } +} +const fail = (message: string): never => { throw new DatabaseSchemaControlError(message) } +const uuid = (value: unknown): value is string => typeof value === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value) +const sha = (value: unknown): value is string => typeof value === "string" && /^[0-9a-f]{64}$/.test(value) +function shape(value: unknown, fields: string): asserts value is Record { + if (!value || typeof value !== "object" || Array.isArray(value) || Object.getPrototypeOf(value) !== Object.prototype || Object.keys(value).sort().join(" ") !== fields.split(" ").sort().join(" ")) fail("Invalid control record shape") +} +function checkIntent(value: unknown): asserts value is DatabaseSchemaIntent { + shape(value, "id epoch kind manifestSha256 artifactSha256") + if (!uuid(value.id) || !Number.isSafeInteger(value.epoch) || Number(value.epoch) < 1 || !["bootstrap", "migration", "import", "restore"].includes(String(value.kind)) || !sha(value.manifestSha256) || !sha(value.artifactSha256)) fail("Invalid exact operation intent") +} +export function checkDatabaseSchemaControlState(value: unknown): DatabaseSchemaControlState { + shape(value, "format epoch owner phase intent terminal") + if (value.format !== DATABASE_SCHEMA_CONTROL_FORMAT || !Number.isSafeInteger(value.epoch) || Number(value.epoch) < 0 || !(value.owner === null && value.epoch === 0 || uuid(value.owner) && Number(value.epoch) > 0) || !["QUARANTINED", "INTENT", "TERMINAL"].includes(String(value.phase))) fail("Invalid control owner/epoch/phase") + if (value.intent !== null) { checkIntent(value.intent); if (value.intent.epoch > Number(value.epoch)) fail("Intent epoch exceeds owner epoch") } + if (value.terminal !== null) { + const hasRelease = !!value.terminal && Object.hasOwn(value.terminal, "legacyRelease") + shape(value.terminal, hasRelease ? "intent outcome observationSha256 legacyRelease" : "intent outcome observationSha256"); checkIntent(value.terminal.intent) + if (value.terminal.intent.epoch > Number(value.epoch) || !["observed-success", "observed-failure", "interrupted-unknown"].includes(String(value.terminal.outcome)) || !sha(value.terminal.observationSha256)) fail("Invalid terminal observation") + if (hasRelease) { + const release = value.terminal.legacyRelease + shape(release, "approvalSha256 observationSha256") + if (value.terminal.intent.kind !== "migration" || value.terminal.outcome !== "observed-success" || + !sha(release.approvalSha256) || !sha(release.observationSha256)) fail("Invalid migration release observation") + } + } + if (value.phase === "INTENT" && value.intent === null || value.phase === "TERMINAL" && (value.intent !== null || value.terminal === null)) fail("Inconsistent control phase") + if (value.owner === null && (value.intent !== null || value.terminal !== null || value.phase !== "QUARANTINED")) fail("Invalid initial state") + return structuredClone(value) as unknown as DatabaseSchemaControlState +} +export function initialDatabaseSchemaControlState(): DatabaseSchemaControlState { + return { format: DATABASE_SCHEMA_CONTROL_FORMAT, epoch: 0, owner: null, phase: "QUARANTINED", intent: null, terminal: null } +} +export function claimDatabaseSchemaOwnership(raw: unknown, owner: string): DatabaseSchemaControlState { + const state = checkDatabaseSchemaControlState(raw) + if (!uuid(owner) || state.owner === owner || state.epoch === Number.MAX_SAFE_INTEGER) fail("Fresh owner and bounded monotonic epoch required") + // Preserve the interrupted intent and last terminal observation. A restart + // must not erase uncertainty, including a server-side COMMIT with lost ACK. + return { ...state, owner, epoch: state.epoch + 1, phase: "QUARANTINED" } +} +export type DatabaseSchemaControlCommand = + | { type: "begin"; id: string; kind: DatabaseSchemaOperation; manifestSha256: string; artifactSha256: string } + | { type: "terminal"; id: string; outcome: DatabaseSchemaTerminal["outcome"]; observationSha256: string } +/** Exact flat primitive contract: reject accessors/proxies through clone at + * the IO boundary; no nested policy bytes or arbitrary caller JSON accepted. */ +export function checkDatabaseSchemaControlCommand(raw: unknown): DatabaseSchemaControlCommand { + if (!raw || typeof raw !== "object") return fail("Missing control command") + // Inspect a bounded flat shape before copying. Do not walk arbitrary policy + // JSON or invoke accessor properties while awaiting a transaction. + const descriptors = Object.getOwnPropertyDescriptors(raw) + const names = Object.keys(descriptors).sort().join(" ") + if (Object.getPrototypeOf(raw) !== Object.prototype || Reflect.ownKeys(raw).length !== Object.keys(descriptors).length || !["artifactSha256 id kind manifestSha256 type", "id observationSha256 outcome type"].includes(names) || Object.values(descriptors).some(item => !("value" in item) || !item.enumerable || typeof item.value !== "string" || item.value.length > 64)) fail("Control command must be a bounded flat primitive record") + const command = structuredClone(raw) as Record + if (command.type === "begin") { + shape(command, "type id kind manifestSha256 artifactSha256") + checkIntent({ id: command.id, epoch: 1, kind: command.kind, manifestSha256: command.manifestSha256, artifactSha256: command.artifactSha256 }) + } else { + shape(command, "type id outcome observationSha256") + if (command.type !== "terminal" || !uuid(command.id) || !["observed-success", "observed-failure", "interrupted-unknown"].includes(String(command.outcome)) || !sha(command.observationSha256)) fail("Invalid terminal command") + } + return command as DatabaseSchemaControlCommand +} +export function transitionDatabaseSchemaControl(raw: unknown, owner: string, command: DatabaseSchemaControlCommand): DatabaseSchemaControlState { + const state = checkDatabaseSchemaControlState(raw) + if (!uuid(owner) || state.owner !== owner) fail("Control owner mismatch") + if (command.type === "begin") { + shape(command, "type id kind manifestSha256 artifactSha256") + if (state.intent !== null || state.terminal?.intent.id === command.id) fail("Unresolved or reused intent") + const intent = { id: command.id, epoch: state.epoch, kind: command.kind, manifestSha256: command.manifestSha256, artifactSha256: command.artifactSha256 } + checkIntent(intent) + return { ...state, phase: "INTENT", intent } + } + if (command.type !== "terminal") fail("Unknown control command") + shape(command, "type id outcome observationSha256") + if (!state.intent || state.intent.id !== command.id || !sha(command.observationSha256) || !["observed-success", "observed-failure", "interrupted-unknown"].includes(command.outcome)) return fail("Missing matching intent/terminal observation") + if (state.intent.epoch !== state.epoch && command.outcome !== "interrupted-unknown") fail("Restart cannot claim old execution outcome") + return checkDatabaseSchemaControlState({ ...state, phase: "TERMINAL", intent: null, terminal: { intent: state.intent, outcome: command.outcome, observationSha256: command.observationSha256 } }) +} +export function databaseSchemaAdmission(raw: unknown) { + const state = checkDatabaseSchemaControlState(raw) + return Object.freeze({ ready: false as const, quarantined: true as const, epoch: state.epoch, + reason: state.intent ? "UNRESOLVED_INTENT" as const : "SCHEMA_AND_DATA_VERIFIER_REQUIRED" as const }) +} diff --git a/backend/src/db/database-schema-control.sql.test.ts b/backend/src/db/database-schema-control.sql.test.ts new file mode 100644 index 00000000..c14ac460 --- /dev/null +++ b/backend/src/db/database-schema-control.sql.test.ts @@ -0,0 +1,197 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { basename, dirname, resolve } from "node:path" +import { execFile } from "node:child_process" +import { writeFile } from "node:fs/promises" + +assert.equal(process.env.MMAPP_OWN_DATABASE_SCHEMA_CONTROL, "true") +const databaseUrl = new URL(process.env.DATABASE_URL ?? "") +assert.equal(databaseUrl.hostname, "127.0.0.1"); assert.equal(databaseUrl.pathname, "/postgres") +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const { Pool } = await import("pg") +const { pool, closePool } = await import("./index.js") +const { applySqlMigrations } = await import("./migrate.js") +const { acquireDatabaseSchemaControl } = await import("./database-schema-control.js") +const { initialDatabaseSchemaControlState, DatabaseSchemaControlError } = await import("./database-schema-control-model.js") +const { runPrivateDatabaseDump } = await import("../services/database-dump-connection.js") +const { restorePartitionedDatabase } = await import("../services/system-database-partition-restore.js") +type ControlPool = Parameters[0] +const names: string[] = [], cases: string[] = [], handles: Awaited>[] = [] +const connections: InstanceType[] = [] +async function database() { + const name = "mm_control_" + randomUUID().replaceAll("-", "") + assert(/^mm_control_[a-f0-9]{32}$/.test(name)) + await pool.query('CREATE DATABASE "' + name + '" TEMPLATE template0') + const url = new URL(databaseUrl); url.pathname = "/" + name + const connection = new Pool({ connectionString: url.toString(), max: 6 }) + names.push(name); connections.push(connection); return connection +} +async function seed(connection: InstanceType) { + await connection.query("CREATE SCHEMA mmapp_control; CREATE TABLE mmapp_control.database_schema_state(id smallint PRIMARY KEY CHECK(id=1),revision bigint NOT NULL CHECK(revision>=0),state jsonb NOT NULL); CREATE TABLE mmapp_control.database_schema_intents(id uuid PRIMARY KEY,intent jsonb NOT NULL,terminal jsonb)") + await connection.query("INSERT INTO mmapp_control.database_schema_state VALUES(1,0,$1::jsonb)", [JSON.stringify(initialDatabaseSchemaControlState())]) +} +async function acquire(connection: ControlPool) { const owner = await acquireDatabaseSchemaControl(connection); handles.push(owner); return owner } +const command = () => ({ type: "begin" as const, id: randomUUID(), kind: "migration" as const, manifestSha256: "a".repeat(64), artifactSha256: "b".repeat(64) }) +const terminal = (id: string, outcome: "observed-success" | "observed-failure" | "interrupted-unknown" = "observed-success") => ({ type: "terminal" as const, id, outcome, observationSha256: "c".repeat(64) }) +const row = async (connection: InstanceType) => (await connection.query("SELECT revision::text,state FROM mmapp_control.database_schema_state WHERE id=1")).rows[0] +const history = async (connection: InstanceType, id: string) => (await connection.query("SELECT intent,terminal FROM mmapp_control.database_schema_intents WHERE id=$1", [id])).rows[0] +function lostCommit(connection: InstanceType, number: number): ControlPool { + return { async connect() { + const client = await connection.connect(); let commits = 0 + return { release(destroy?: boolean) { client.release(destroy) }, async query(sql: string, values?: unknown[]) { + const result = await client.query(sql, values) + if (sql === "COMMIT" && ++commits === number) throw Error("owned fixture lost actual COMMIT acknowledgement") + return { rows: result.rows, command: result.command } + } } + } } +} +function abortedCommit(connection: InstanceType, number: number): ControlPool { + return { async connect() { + const client = await connection.connect(); let commits = 0 + return { release(destroy?: boolean) { client.release(destroy) }, async query(sql: string, values?: unknown[]) { + if (sql === "COMMIT" && ++commits === number) { + // A real aborted PostgreSQL transaction acknowledges COMMIT as ROLLBACK. + await assert.rejects(client.query("SELECT 1/0")) + } + const result = await client.query(sql, values) + return { rows: result.rows, command: result.command } + } } + } } +} +async function commandIo(program: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + execFile(program, args, { encoding: "buffer", timeout: 30000, maxBuffer: 32 * 1048576, windowsHide: true }, (error, stdout) => error ? reject(Error("Owned PostgreSQL archive command failed: " + program)) : resolve(stdout)) + }) +} +try { + // The runner's separate terminal restore fixture uses this owned database. + // Control protocol scenarios below each use their own independent database. + await applySqlMigrations(pool) + const missing = await database() + await assert.rejects(acquire(missing), DatabaseSchemaControlError) + assert.equal((await missing.query("SELECT count(*)::int AS n FROM pg_namespace WHERE nspname='mmapp_control'")).rows[0].n, 0) + cases.push("missing external control refuses without startup DDL or enrollment") + + const regular = await database(); await seed(regular) + let owner = await acquire(regular) + const initial = await owner.read(); assert.equal(initial.state.epoch, 1); assert.equal(initial.revision, "1") + assert.equal((await owner.admission()).ready, false) + await assert.rejects(acquire(regular), DatabaseSchemaControlError) + assert.deepEqual(await owner.read(), initial) + cases.push("one checked-out session owns exclusive lease; another owner changes no epoch or revision") + const begin = command(), intended = await owner.transition(begin) + assert.equal(intended.revision, "2"); assert.equal(intended.state.phase, "INTENT") + assert.deepEqual((await history(regular, begin.id)).intent, intended.state.intent) + assert.equal((await owner.admission()).reason, "UNRESOLVED_INTENT") + cases.push("intent singleton and unique immutable history publish atomically before application dispatch") + await owner.close(); owner = await acquire(regular) + assert.equal((await owner.read()).state.intent?.id, begin.id) + await assert.rejects(owner.transition(terminal(begin.id)), DatabaseSchemaControlError) + assert.equal((await history(regular, begin.id)).terminal, null) + cases.push("restart preserves old intent and cannot claim an unobserved successful outcome") + owner = await acquire(regular) + const interrupted = await owner.transition(terminal(begin.id, "interrupted-unknown")) + assert.equal(interrupted.state.terminal?.outcome, "interrupted-unknown") + assert.equal((await owner.admission()).ready, false) + assert.deepEqual((await history(regular, begin.id)).terminal, interrupted.state.terminal) + cases.push("interrupted unknown can be recorded once but never enables consumers") + const second = command(); await owner.transition(second); await owner.transition(terminal(second.id)) + const prior = await row(regular) + await assert.rejects(owner.transition(begin), DatabaseSchemaControlError) + assert.deepEqual(await row(regular), prior) + assert.equal((await regular.query("SELECT count(*)::int AS n FROM mmapp_control.database_schema_intents")).rows[0].n, 2) + cases.push("reuse of an older completed operation is refused by actual unique history with no singleton advance") + + for (const stage of ["claim", "intent", "terminal"] as const) { + const connection = await database(); await seed(connection) + const wrapped = lostCommit(connection, stage === "claim" ? 1 : stage === "intent" ? 2 : 3) + const intent = command() + if (stage === "claim") await assert.rejects(acquire(wrapped), (error: unknown) => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + else { + const handle = await acquire(wrapped) + if (stage === "terminal") await handle.transition(intent) + await assert.rejects(handle.transition(stage === "intent" ? intent : terminal(intent.id)), (error: unknown) => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + await assert.rejects(handle.read(), DatabaseSchemaControlError) + } + const fresh = await acquire(connection), observed = await fresh.read() + assert.equal(observed.state.epoch, 2); assert.equal(observed.state.phase, "QUARANTINED") + assert.equal((await fresh.admission()).ready, false) + if (stage === "intent") { assert.equal(observed.state.intent?.id, intent.id); assert.equal((await history(connection, intent.id)).terminal, null) } + if (stage === "terminal") { assert.equal(observed.state.intent, null); assert.equal(observed.state.terminal?.intent.id, intent.id); assert.equal(observed.state.terminal?.outcome, "observed-success") } + await fresh.close() + cases.push("actual server COMMIT with lost " + stage + " ACK destroys owner; restart preserves quarantine and durable observation") + } + for (const stage of ["claim", "intent", "terminal"] as const) { + const connection = await database(); await seed(connection) + const wrapped = abortedCommit(connection, stage === "claim" ? 1 : stage === "intent" ? 2 : 3) + const intent = command() + if (stage === "claim") await assert.rejects(acquire(wrapped), (error: unknown) => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + else { + const handle = await acquire(wrapped) + if (stage === "terminal") await handle.transition(intent) + await assert.rejects(handle.transition(stage === "intent" ? intent : terminal(intent.id)), (error: unknown) => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + await assert.rejects(handle.read(), DatabaseSchemaControlError) + } + // Readback diagnoses the rolled-back publication without replaying it. + const saved = await row(connection) + assert.equal(saved.revision, stage === "claim" ? "0" : stage === "intent" ? "1" : "2") + assert.equal((await history(connection, intent.id))?.terminal ?? null, null) + assert.equal(saved.state.intent?.id ?? null, stage === "terminal" ? intent.id : null) + const fresh = await acquire(connection) + assert.equal((await fresh.admission()).ready, false) + if (stage === "terminal") await assert.rejects(fresh.transition(terminal(intent.id)), DatabaseSchemaControlError) + await fresh.close() + cases.push("actual aborted " + stage + " transaction returns ROLLBACK command; owner discarded without successful publication or replay") + } + const altered = await database(); await seed(altered) + const stale = await acquire(altered) + await altered.query("UPDATE mmapp_control.database_schema_state SET revision=revision+1") + await assert.rejects(stale.read(), DatabaseSchemaControlError) + cases.push("external revision drift invalidates a live owner before admission") + const corrupted = await database(); await seed(corrupted) + const corruptOwner = await acquire(corrupted), corruptIntent = command(); await corruptOwner.transition(corruptIntent) + await corrupted.query("UPDATE mmapp_control.database_schema_intents SET intent=jsonb_set(intent,'{artifactSha256}',to_jsonb($1::text)) WHERE id=$2", ["d".repeat(64), corruptIntent.id]) + await assert.rejects(corruptOwner.read(), DatabaseSchemaControlError) + await assert.rejects(acquire(corrupted), DatabaseSchemaControlError) + cases.push("immutable history mismatch invalidates both current owner and new owner claim") + const reset = await database(); await seed(reset) + const resetOwner = await acquire(reset), resetIntent = command(); await resetOwner.transition(resetIntent) + // This database was created by this test in the owned disposable cluster. + await reset.query("CREATE TABLE public.owned_fixture(id integer); DROP SCHEMA public CASCADE; CREATE SCHEMA public") + assert.equal((await resetOwner.read()).state.intent?.id, resetIntent.id) + assert.equal((await history(reset, resetIntent.id)).terminal, null) + cases.push("control epoch and unresolved history survive replacement of the public application namespace") + const archiveDb = await database(); await seed(archiveDb) + await archiveDb.query("CREATE TABLE public.servers(id integer PRIMARY KEY); CREATE TABLE public.schema_migrations(id text PRIMARY KEY); INSERT INTO public.servers VALUES(1)") + const archiveOwner = await acquire(archiveDb), archiveIntent = command(); await archiveOwner.transition(archiveIntent) + const originalState = await row(archiveDb), archiveUrl = new URL(databaseUrl); archiveUrl.pathname = "/" + names.at(-1)! + const safeFile = resolve(process.env.MCP_STATE_DIR!, "control-excluded.dump"), unsafeFile = resolve(process.env.MCP_STATE_DIR!, "control-full.dump") + const actualPrivateDump = await runPrivateDatabaseDump(archiveUrl.toString(), { authorize: async () => {} }) + await writeFile(safeFile, actualPrivateDump, { flag: "wx", mode: 0o600 }) + const toc = (await commandIo("pg_restore", ["--list", safeFile])).toString("utf8") + assert(!toc.includes("mmapp_control")) + await archiveDb.query("UPDATE public.servers SET id=2") + let replacements = 0 + await restorePartitionedDatabase(safeFile, archiveUrl.toString(), { + command: commandIo, prepareSql: file => writeFile(file, "", { flag: "wx", mode: 0o600 }), beforeTargetReplacement: () => { replacements++ }, + }) + assert.equal(replacements, 1) + assert.deepEqual((await archiveDb.query("SELECT id FROM public.servers")).rows, [{ id: 1 }]) + assert.deepEqual(await row(archiveDb), originalState) + assert.equal((await archiveOwner.read()).state.intent?.id, archiveIntent.id) + cases.push("actual private pg_dump excludes control; actual application restore changes public data while preserving live owner epoch and intent") + await commandIo("pg_dump", ["-Fc", "--file", unsafeFile, archiveUrl.toString()]) + let archiveCommands = 0, preparations = 0 + await assert.rejects(restorePartitionedDatabase(unsafeFile, archiveUrl.toString(), { + command: async (program, args) => { archiveCommands++; return commandIo(program, args) }, + prepareSql: async () => { preparations++ }, beforeTargetReplacement: () => { replacements++ }, + })) + assert.equal(archiveCommands, 1); assert.equal(preparations, 0); assert.equal(replacements, 1) + assert.deepEqual(await row(archiveDb), originalState) + cases.push("actual full archive containing control refuses immediately after TOC before SQL decode or target replacement") + console.log("Owned database schema control SQL:", JSON.stringify({ cases: cases.length, names: cases, databases: names.length, actualSql: true, lostActualCommitAcks: 3, startupWired: false, readyAuthorized: false, productionDatabaseTouched: false })) +} finally { + for (const handle of handles) await handle.close().catch(() => {}) + for (const connection of connections) await connection.end() + await closePool() +} diff --git a/backend/src/db/database-schema-control.test.ts b/backend/src/db/database-schema-control.test.ts new file mode 100644 index 00000000..24257c3f --- /dev/null +++ b/backend/src/db/database-schema-control.test.ts @@ -0,0 +1,197 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { acquireDatabaseSchemaControl, DATABASE_SCHEMA_CONTROL_SQL as sql, type DatabaseSchemaControlClient } from "./database-schema-control.js" +import { initialDatabaseSchemaControlState, DatabaseSchemaControlError, type DatabaseSchemaControlState } from "./database-schema-control-model.js" + +const id = "00000000-0000-4000-8000-000000000001", otherId = "00000000-0000-4000-8000-000000000002", hash = "a".repeat(64) +const command = (intentId = id) => ({ type: "begin" as const, id: intentId, kind: "restore" as const, manifestSha256: hash, artifactSha256: hash }) +/** Inert protocol emulator: does not claim PostgreSQL durability. Root's + * private PG suite must independently test these SQL strings/commit windows. */ +function fixture() { + let durable = { revision: "0", state: initialDatabaseSchemaControlState() }, intents = new Map() + let lock: number | undefined, sessions = 0 + const calls: string[] = [], releases: boolean[] = [] + const faults = { at: "", afterCommit: false, commitTag: "COMMIT" as string | undefined }, empty = { rows: [] as Record[] } + const pool = { connect: async (): Promise => { + const session = ++sessions + let tx: typeof durable | undefined, history: typeof intents | undefined + return { + query: async (query, values = []) => { + calls.push(query) + if (query === faults.at && !faults.afterCommit) throw new Error("injected SQL refusal") + if (query === sql.acquire) { if (lock !== undefined) return { rows: [{ acquired: false }] }; lock = session; return { rows: [{ acquired: true }] } } + if (query === sql.unlock) { assert.equal(lock, session); lock = undefined; return { rows: [{ unlocked: true }] } } + if (query === "BEGIN") { assert(!tx); tx = structuredClone(durable); history = structuredClone(intents); return empty } + if (query.startsWith("SET LOCAL ")) { assert(tx); return empty } + if (query === sql.read) { assert(tx); return { rows: [structuredClone(tx)] } } + if (query === sql.beginIntent) { assert(history); if (history.has(String(values[0]))) throw new Error("duplicate intent"); history.set(String(values[0]), { intent: JSON.parse(String(values[1])), terminal: null }); return { rows: [{ id: values[0] }] } } + if (query === sql.terminalIntent) { const row = history?.get(String(values[0])); if (!row || row.terminal) return empty; row.terminal = JSON.parse(String(values[1])); return { rows: [{ id: values[0] }] } } + if (query === sql.readIntent) { const row = history?.get(String(values[0])); return { rows: row ? [structuredClone(row)] : [] } } + if (query === sql.write) { assert(tx); if (values[1] !== tx.revision) return empty; tx = { revision: String(BigInt(tx.revision) + 1n), state: JSON.parse(String(values[0])) as DatabaseSchemaControlState }; return { rows: [structuredClone(tx)] } } + if (query === "COMMIT") { assert(tx && history); durable = tx; intents = history; tx = undefined; history = undefined; if (faults.at === query && faults.afterCommit) throw new Error("lost COMMIT ACK"); return { ...empty, command: faults.commitTag } } + throw new Error("Unexpected SQL: " + query) + }, + release: destroy => { releases.push(Boolean(destroy)); tx = undefined; history = undefined; if (lock === session) lock = undefined }, + } + } } + return { pool, calls, releases, faults, read: () => structuredClone(durable), intents: () => structuredClone(intents), corrupt: (change: (state: DatabaseSchemaControlState) => void) => change(durable.state) } +} +test("acquisition commits quarantine before returning, no DDL or application SQL", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + assert.equal(f.read().state.epoch, 1) + assert.equal(f.calls.at(-1), "COMMIT") + assert.equal((await handle.admission()).ready, false) + assert(f.calls.includes("SET LOCAL synchronous_commit = on")) + assert(!f.calls.some(query => /CREATE|DROP|public\./.test(query))) + await handle.close(); assert.deepEqual(f.releases, [false]) +}) +test("second owner cannot acquire a live session lease", async () => { + const f = fixture(), first = await acquireDatabaseSchemaControl(f.pool) + await assert.rejects(acquireDatabaseSchemaControl(f.pool), DatabaseSchemaControlError) + assert.equal(f.read().state.epoch, 1) + await first.close() + const second = await acquireDatabaseSchemaControl(f.pool) + assert.equal(f.read().state.epoch, 2) + await second.close() +}) +test("intent and terminal history are atomically published with control CAS", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + await handle.transition(command()) + assert(f.intents().has(id)); assert.equal(f.read().state.phase, "INTENT") + await handle.transition({ type: "terminal", id, outcome: "observed-success", observationSha256: hash }) + assert(f.intents().get(id)?.terminal) + assert.equal((await handle.admission()).ready, false) + await handle.close() +}) +test("historical id reuse refuses after subsequent operations", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + for (const current of [id, otherId]) { + await handle.transition(command(current)) + await handle.transition({ type: "terminal", id: current, outcome: "observed-success", observationSha256: hash }) + } + await assert.rejects(handle.transition(command())) + assert.equal(f.read().state.terminal?.intent.id, otherId) + assert.equal(f.releases.at(-1), true) +}) +test("failed CAS rolls back intent publication and discards owner", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + f.faults.at = sql.write + await assert.rejects(handle.transition(command())) + assert.equal(f.intents().size, 0); assert.equal(f.read().state.intent, null) + await assert.rejects(handle.read()) + assert.equal(f.releases.at(-1), true) +}) +test("lost intent COMMIT ACK persists unknown intent but poisons old handle", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + f.faults.at = "COMMIT"; f.faults.afterCommit = true + await assert.rejects(handle.transition(command()), error => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + assert.equal(f.read().state.intent?.id, id) + await assert.rejects(handle.admission()) + f.faults.at = "" + const restarted = await acquireDatabaseSchemaControl(f.pool) + assert.equal(f.read().state.epoch, 2) + assert.equal((await restarted.admission()).reason, "UNRESOLVED_INTENT") + await assert.rejects(restarted.transition({ type: "terminal", id, outcome: "observed-success", observationSha256: hash })) +}) +test("lost terminal ACK never produces READY after fresh ownership", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + await handle.transition(command()) + f.faults.at = "COMMIT"; f.faults.afterCommit = true + await assert.rejects(handle.transition({ type: "terminal", id, outcome: "observed-success", observationSha256: hash })) + f.faults.at = "" + const restarted = await acquireDatabaseSchemaControl(f.pool) + assert.equal(f.read().state.terminal?.outcome, "observed-success") + assert.equal((await restarted.admission()).ready, false) + await restarted.close() +}) +test("restarted owner may durably record interrupted-unknown but stays quarantined", async () => { + const f = fixture(), first = await acquireDatabaseSchemaControl(f.pool) + await first.transition(command()); await first.close() + const next = await acquireDatabaseSchemaControl(f.pool) + await next.transition({ type: "terminal", id, outcome: "interrupted-unknown", observationSha256: hash }) + assert.equal(f.read().state.intent, null) + assert.equal((await next.admission()).ready, false) + await next.close() +}) +test("concurrent use of one handle refuses without duplicate publication", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + const first = handle.transition(command()) + await assert.rejects(handle.transition(command(otherId))) + await first + assert.equal(f.intents().size, 1) + await handle.close(); await handle.close() + await assert.rejects(handle.read()) +}) +test("caller mutation after transition cannot change durable command or history", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool), mutable = command() + const pending = handle.transition(mutable) + mutable.id = otherId + await pending + assert.equal(f.read().state.intent?.id, id) + assert(f.intents().has(id)); assert(!f.intents().has(otherId)) + await handle.close() +}) +test("unknown command policy bytes refuse before SQL and leave owner usable", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool), before = f.calls.length + await assert.rejects(handle.transition({ ...command(), policy: { ready: true } } as ReturnType)) + assert.equal(f.calls.length, before) + assert.equal((await handle.admission()).ready, false) + await handle.close() +}) +test("tampered state cannot disagree with independently stored intent history", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + await handle.transition(command()) + f.corrupt(state => { state.intent!.artifactSha256 = "b".repeat(64) }) + await assert.rejects(handle.read()) + assert.equal(f.releases.at(-1), true) + await assert.rejects(acquireDatabaseSchemaControl(f.pool)) + assert.equal(f.read().state.epoch, 1) +}) +test("accessor commands refuse without invocation or database IO", async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool), before = f.calls.length + let invoked = false + const accessor = command() + Object.defineProperty(accessor, "id", { enumerable: true, get: () => { invoked = true; return id } }) + await assert.rejects(handle.transition(accessor)) + assert.equal(invoked, false); assert.equal(f.calls.length, before) + await handle.close() +}) + +for (const tag of ["ROLLBACK", "", undefined]) { + test(`unconfirmed COMMIT command ${String(tag)} destroys owner and never returns a grant`, async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + f.faults.commitTag = tag + await assert.rejects(handle.transition(command()), error => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + assert.equal(f.releases.at(-1), true) + await assert.rejects(handle.read(), DatabaseSchemaControlError) + // Even if the server committed, no readback replays or upgrades the ACK. + assert.equal(f.calls.filter(query => query === sql.beginIntent).length, 1) + f.faults.commitTag = "COMMIT" + const restarted = await acquireDatabaseSchemaControl(f.pool) + assert.equal((await restarted.admission()).ready, false) + await restarted.close() + }) + test(`claim COMMIT command ${String(tag)} refuses before a usable owner exists`, async () => { + const f = fixture() + f.faults.commitTag = tag + await assert.rejects(acquireDatabaseSchemaControl(f.pool), error => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + assert.deepEqual(f.releases, [true]) + assert.equal(f.calls.filter(query => query === sql.beginIntent).length, 0) + }) +} + +for (const tag of ["ROLLBACK", "", undefined]) { + test(`terminal COMMIT command ${String(tag)} cannot publish an acknowledged outcome`, async () => { + const f = fixture(), handle = await acquireDatabaseSchemaControl(f.pool) + await handle.transition(command()) + f.faults.commitTag = tag + await assert.rejects(handle.transition({ type: "terminal", id, outcome: "observed-success", observationSha256: hash }), error => error instanceof DatabaseSchemaControlError && error.outcomeUnknown) + assert.equal(f.releases.at(-1), true) + await assert.rejects(handle.admission(), DatabaseSchemaControlError) + assert.equal(f.calls.filter(query => query === sql.terminalIntent).length, 1) + f.faults.commitTag = "COMMIT" + const restarted = await acquireDatabaseSchemaControl(f.pool) + assert.equal((await restarted.admission()).ready, false) + await restarted.close() + }) +} diff --git a/backend/src/db/database-schema-control.ts b/backend/src/db/database-schema-control.ts new file mode 100644 index 00000000..f79212c6 --- /dev/null +++ b/backend/src/db/database-schema-control.ts @@ -0,0 +1,170 @@ +import { randomUUID } from "node:crypto" +import { checkDatabaseSchemaControlCommand, checkDatabaseSchemaControlState, claimDatabaseSchemaOwnership, databaseSchemaAdmission, DatabaseSchemaControlError, transitionDatabaseSchemaControl, type DatabaseSchemaControlCommand, type DatabaseSchemaControlState } from "./database-schema-control-model.js" +import { beginDatabaseRestoreQuarantine, readDatabaseRestoreQuarantineStatus } from "./database-restore-quarantine.js" +import { exactHistory, legacy0037AdoptionObservation, legacy0037ReleaseObservation } from "./controlled-migration-history.js" +import { assertHeldLegacyProbeOwner, assertLegacyHandoffAdoptionHeld, type HeldLegacyProbeOwner, type LegacyHandoffAdoption } from "../services/server-probe-legacy-handoff-inspection.js" + +/** Externally provisioned only. Do not include this namespace in application + * dumps/restores, public schema reset, or application schema fingerprints. + * Restoring the control namespace would roll back fencing epochs. */ +export const DATABASE_SCHEMA_CONTROL_NAMESPACE = "mmapp_control" +export const DATABASE_SCHEMA_CONTROL_LOCK = 8723103 +export const DATABASE_SCHEMA_CONTROL_SQL = Object.freeze({ + acquire: "SELECT pg_catalog.pg_try_advisory_lock($1::bigint) AS acquired", + read: "SELECT revision::text AS revision, state FROM mmapp_control.database_schema_state WHERE id = 1 FOR UPDATE", + write: "UPDATE mmapp_control.database_schema_state SET revision = revision + 1, state = $1::jsonb WHERE id = 1 AND revision = $2::bigint RETURNING revision::text AS revision, state", + beginIntent: "INSERT INTO mmapp_control.database_schema_intents (id, intent, terminal) VALUES ($1::uuid, $2::jsonb, NULL) RETURNING id::text AS id", + terminalIntent: "UPDATE mmapp_control.database_schema_intents SET terminal = $2::jsonb WHERE id = $1::uuid AND terminal IS NULL RETURNING id::text AS id", + readIntent: "SELECT intent, terminal FROM mmapp_control.database_schema_intents WHERE id = $1::uuid FOR SHARE", + unlock: "SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", +}) +export interface DatabaseSchemaControlClient { + query(sql: string, values?: unknown[]): Promise<{ rows: Record[]; command?: string }> + release(destroy?: boolean): void +} +export interface DatabaseSchemaControlPool { connect(): Promise } +export interface DatabaseSchemaControlSnapshot { revision: string; state: DatabaseSchemaControlState } +const canonical = (value: unknown): string => JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a.localeCompare(b))) : item) +function snapshot(rows: Record[]): DatabaseSchemaControlSnapshot { + if (rows.length !== 1 || Object.keys(rows[0]).sort().join(" ") !== "revision state" || typeof rows[0].revision !== "string" || !/^(0|[1-9][0-9]{0,18})$/.test(rows[0].revision) || BigInt(rows[0].revision) >= 9223372036854775807n) throw new DatabaseSchemaControlError("Missing/invalid externally provisioned control singleton") + return { revision: rows[0].revision, state: checkDatabaseSchemaControlState(rows[0].state) } +} +/** Owns a checked-out SESSION for its full lifetime. No pool.query, retries, + * background process, file state, DDL, automatic enrollment, or SQL dispatch. + * A successful returned transition means this client observed COMMIT success; + * any uncertain SQL/COMMIT destroys the handle and its session lock. */ +export async function acquireDatabaseSchemaControl(pool: DatabaseSchemaControlPool) { + const client = await pool.connect(), owner = randomUUID() + let closed = false, busy = false, revision: string | undefined + const destroy = () => { if (!closed) { closed = true; client.release(true) } } + async function checkHistory(state: DatabaseSchemaControlState) { + const entries = [state.intent && { intent: state.intent, terminal: null }, state.terminal && { intent: state.terminal.intent, terminal: state.terminal }].filter(item => item !== null) + for (const expected of entries) { + const rows = (await client.query(DATABASE_SCHEMA_CONTROL_SQL.readIntent, [expected.intent.id])).rows + if (rows.length !== 1 || Object.keys(rows[0]).sort().join(" ") !== "intent terminal" || canonical(rows[0]) !== canonical(expected)) throw new DatabaseSchemaControlError("Control state disagrees with durable intent history") + } + } + async function transaction(change?: (state: DatabaseSchemaControlState) => DatabaseSchemaControlState, command?: DatabaseSchemaControlCommand, + legacy?: HeldLegacyProbeOwner, adoption?: LegacyHandoffAdoption) { + if (closed || busy) throw new DatabaseSchemaControlError("Closed or concurrent control owner handle") + busy = true; let commitSent = false + try { + await client.query("BEGIN") + await client.query("SET LOCAL search_path = pg_catalog") + await client.query("SET LOCAL lock_timeout = '5s'") + await client.query("SET LOCAL statement_timeout = '10s'") + await client.query("SET LOCAL synchronous_commit = on") + let current = snapshot((await client.query(DATABASE_SCHEMA_CONTROL_SQL.read)).rows) + await checkHistory(current.state) + if (revision !== undefined && (current.revision !== revision || current.state.owner !== owner)) throw new DatabaseSchemaControlError("Control ownership/revision changed") + if (legacy) { + await assertHeldLegacyProbeOwner(client, legacy) + if (current.state.phase !== "QUARANTINED" || current.state.intent !== null || !current.state.terminal) throw new DatabaseSchemaControlError("Exact quarantined owner history required") + const history = await client.query("SELECT id::text AS id,intent,terminal FROM mmapp_control.database_schema_intents ORDER BY id LIMIT 37") + if (await exactHistory(history.rows, current.state, legacy.intent.manifestSha256, true) !== 36 || + (await readDatabaseRestoreQuarantineStatus(client)).activeOperationId !== null) throw new DatabaseSchemaControlError("Legacy handoff predecessor or quarantine mismatch") + } + let adopted: DatabaseSchemaControlState | undefined + if (adoption) { + await assertLegacyHandoffAdoptionHeld(client, adoption) + const { intent, release } = adoption + const original = release ? current.state.terminal?.intent : current.state.intent + const active = (await client.query(`SELECT id FROM mmapp_control.database_restore_quarantine + WHERE id=1 AND operation_id=$1::pg_catalog.uuid FOR UPDATE`, [intent.operationId])).rows + if (current.state.phase !== "QUARANTINED" || active.length !== 1 || !original || original.id !== intent.operationId || original.kind !== "migration" || + original.artifactSha256 !== intent.intentSha256 || original.manifestSha256 !== intent.manifestSha256) throw new DatabaseSchemaControlError("Exact pending legacy adoption required") + if (release) { + const terminal = current.state.terminal + const history = await client.query("SELECT id::text AS id,intent,terminal FROM mmapp_control.database_schema_intents ORDER BY id LIMIT 38") + if (current.state.intent || !terminal || terminal.legacyRelease || terminal.observationSha256 !== release.adoptionSha256 || + await exactHistory(history.rows, current.state, intent.manifestSha256, true) !== 37) { + throw new DatabaseSchemaControlError("Exact adopted legacy prefix required for release") + } + const observationSha256 = await legacy0037ReleaseObservation(terminal, release.approvalSha256) + const extended = { ...terminal, legacyRelease: { approvalSha256: release.approvalSha256, observationSha256 } } + const updated = await client.query(`UPDATE mmapp_control.database_schema_intents SET terminal=$2::jsonb + WHERE id=$1::uuid AND terminal=$3::jsonb RETURNING id::text AS id`, [intent.operationId, JSON.stringify(extended), JSON.stringify(terminal)]) + const cleared = await client.query(`UPDATE mmapp_control.database_restore_quarantine SET operation_id=NULL + WHERE id=1 AND operation_id=$1::uuid RETURNING id`, [intent.operationId]) + if (updated.rows.length !== 1 || updated.rows[0].id !== intent.operationId || cleared.rows.length !== 1 || cleared.rows[0].id !== 1) { + throw new DatabaseSchemaControlError("Legacy release CAS refused") + } + adopted = { ...current.state, phase: "TERMINAL", intent: null, terminal: extended } + } else { + const terminal = { intent: original, outcome: "observed-success" as const, observationSha256: await legacy0037AdoptionObservation(original) } + adopted = { ...current.state, phase: "TERMINAL", intent: null, terminal } + } + } + if (change || adopted) { + const next = checkDatabaseSchemaControlState(adopted ?? change!(current.state)) + const id = command?.id ?? (adoption?.release ? undefined : adopted?.terminal?.intent.id) + if (id) { + const result = command?.type === "begin" + ? await client.query(DATABASE_SCHEMA_CONTROL_SQL.beginIntent, [id, JSON.stringify(next.intent)]) + : await client.query(DATABASE_SCHEMA_CONTROL_SQL.terminalIntent, [id, JSON.stringify(next.terminal)]) + if (result.rows.length !== 1 || result.rows[0].id !== id) throw new DatabaseSchemaControlError("Intent history publication refused") + } + const updated = snapshot((await client.query(DATABASE_SCHEMA_CONTROL_SQL.write, [JSON.stringify(next), current.revision])).rows) + if (BigInt(updated.revision) !== BigInt(current.revision) + 1n || canonical(updated.state) !== canonical(next)) throw new DatabaseSchemaControlError("Control CAS publication mismatch") + current = updated + await checkHistory(current.state) + } + if (legacy) { + await beginDatabaseRestoreQuarantine(client, legacy.intent.operationId) + await assertHeldLegacyProbeOwner(client, legacy) + } + if (adoption) await assertLegacyHandoffAdoptionHeld(client, adoption) + commitSent = true + const committed = await client.query("COMMIT") + if (committed.command !== "COMMIT") throw new DatabaseSchemaControlError("Control COMMIT acknowledgement unavailable") + revision = current.revision + return structuredClone(current) + } catch (cause) { + // Discard the checked-out backend: COMMIT may have completed, and a + // ROLLBACK/retry cannot turn that uncertainty into a valid receipt. + destroy() + throw new DatabaseSchemaControlError("Control transaction unconfirmed; owner handle discarded", commitSent, { cause }) + } finally { busy = false } + } + try { + const lock = await client.query(DATABASE_SCHEMA_CONTROL_SQL.acquire, [DATABASE_SCHEMA_CONTROL_LOCK]) + if (lock.rows.length !== 1 || lock.rows[0].acquired !== true) throw new DatabaseSchemaControlError("Another database schema controller owns the session lease") + await transaction(state => claimDatabaseSchemaOwnership(state, owner)) + } catch (cause) { destroy(); throw cause } + return Object.freeze({ + /** Durable intent must resolve successfully BEFORE any application write. + * Consumers must be drained separately; this lease does not fence legacy + * writers that have not adopted this control protocol. */ + transition: async (rawCommand: DatabaseSchemaControlCommand) => { + // Copy and validate synchronously before transaction() reaches BEGIN. + const command = checkDatabaseSchemaControlCommand(rawCommand) + return transaction(state => transitionDatabaseSchemaControl(state, owner, command), command) + }, + /** A signed, durable handoff belongs to this one held application transaction. */ + beginLegacyHandoff: async (proof: HeldLegacyProbeOwner) => { + const command = checkDatabaseSchemaControlCommand({ type: "begin", kind: "migration", id: proof.intent.operationId, + manifestSha256: proof.intent.manifestSha256, artifactSha256: proof.intent.intentSha256 }) + return transaction(state => transitionDatabaseSchemaControl(state, owner, command), command, proof) + }, + adoptLegacyHandoff: (proof: LegacyHandoffAdoption) => { + if (proof.release) throw new DatabaseSchemaControlError("Unexpected release proof") + return transaction(undefined, undefined, undefined, proof) + }, + releaseLegacyHandoff: (proof: LegacyHandoffAdoption) => { + if (!proof.release) throw new DatabaseSchemaControlError("Exact release proof required") + return transaction(undefined, undefined, undefined, proof) + }, + read: () => transaction(), + admission: async () => databaseSchemaAdmission((await transaction()).state), + close: async () => { + if (closed) return + if (busy) throw new DatabaseSchemaControlError("Control transaction still active") + busy = true + try { + const result = await client.query(DATABASE_SCHEMA_CONTROL_SQL.unlock, [DATABASE_SCHEMA_CONTROL_LOCK]) + if (result.rows.length !== 1 || result.rows[0].unlocked !== true) throw new DatabaseSchemaControlError("Control lease release unconfirmed") + closed = true; client.release(false) + } catch (cause) { destroy(); throw cause } finally { busy = false } + }, + }) +} diff --git a/backend/src/db/database-source-activation-record.ts b/backend/src/db/database-source-activation-record.ts new file mode 100644 index 00000000..d5341cc8 --- /dev/null +++ b/backend/src/db/database-source-activation-record.ts @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict' +import { z } from 'zod' +import { sourceBindingBytes } from './database-source-binding.js' + +export const DATABASE_SOURCE_ACTIVATION_FILENAME = 'source-activation.json' +export const MAX_DATABASE_SOURCE_ACTIVATION_BYTES = 131072 + +const digest = z.string().regex(/^[a-f0-9]{64}$/) +const uuid = z.string().uuid() +const generation = z.object({ generation: uuid, headSha256: digest, bindingSha256: digest, tlsSpkiSha256: digest }).strict() + +export const databaseSourceActivationRecordSchema = z.object({ + format: z.literal('database-source-activation-v1'), + activationDirectory: z.string().min(1).max(4096), + installationId: uuid, + sourceIdentityId: uuid, + publicKeySpkiSha256: digest, + status: z.enum(['active', 'revoked']), + generations: z.array(generation).min(1).max(256), +}).strict() + +export type DatabaseSourceActivationRecord = z.infer + +export function parseDatabaseSourceActivationRecord(bytes: Buffer): DatabaseSourceActivationRecord { + assert(bytes.length <= MAX_DATABASE_SOURCE_ACTIVATION_BYTES, 'Activation record exceeds size limit') + const state = databaseSourceActivationRecordSchema.parse(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(bytes))) + assert(sourceBindingBytes(state).equals(bytes), 'Noncanonical activation record') + for (const field of ['generation', 'headSha256', 'tlsSpkiSha256'] as const) { + assert.equal(new Set(state.generations.map(item => item[field].toLowerCase())).size, state.generations.length, 'Reused activation identity') + } + return state +} diff --git a/backend/src/db/database-source-binding.test.ts b/backend/src/db/database-source-binding.test.ts new file mode 100644 index 00000000..118ab91e --- /dev/null +++ b/backend/src/db/database-source-binding.test.ts @@ -0,0 +1,180 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { generateKeyPairSync, randomUUID, sign } from "node:crypto" +import { mkdtemp, readFile, writeFile, rm, readdir } from "node:fs/promises" +import { tmpdir } from "node:os" +import path from "node:path" +import { captureDatabaseSourceObservation } from "./database-source-observation.js" +import type { ProbeStorageSql } from "../services/server-probe-journal-maintenance.js" +import { createDatabaseSourceBindingVerifier, readDatabaseSourceBindingRegistry, sourceBindingBytes, sourceBindingCanonical, sourceBindingSha256, + DATABASE_SOURCE_SIGNATURE_DOMAINS, DatabaseSourceBindingError } from "./database-source-binding.js" +import { provisionDatabaseSourceBindingOffline, databaseSourceBindingOperatorCli } from "../../scripts/database-source-binding-operator.js" + +assert.equal(process.env.MMAPP_OWN_SOURCE_BINDING_SYNTHETIC, "true") +assert.equal(process.env.DATABASE_URL, undefined) +const options = () => ({ deadlineAt: Date.now() + 10000 }) +const row = () => ({ system_identifier: "18446744073709551615", database_oid: "16384", database_name: "synthetic", + session_user: "source_operator", current_user: "source_operator", in_recovery: false, server_version_num: "180006", backend_pid: 123, + transaction_timestamp: "2026-10-31 12:34:56.123456+00", snapshot: "100:110:101,105,109", + isolation: "repeatable read", read_only: "on", explicit_transaction: true }) +function facade(change?: (count: number, value: ReturnType) => Promise) { + let count = 0 + const sql: ProbeStorageSql = { async query() { const value = row(); await change?.(++count, value); return { rows: [value] } } } + return sql +} +const refused = (error: unknown) => error instanceof DatabaseSourceBindingError +async function own(work: (root: string) => Promise) { + const root = await mkdtemp(path.join(tmpdir(), "mm-source-binding-synthetic-")) + try { return await work(root) } + finally { + const relative = path.relative(path.resolve(tmpdir()), path.resolve(root)) + assert(relative && !relative.startsWith("..") && !path.isAbsolute(relative) && path.basename(root).startsWith("mm-source-binding-synthetic-")) + await rm(root, { recursive: true, force: true }) + } +} +async function fixture(root: string) { + const key = generateKeyPairSync("ed25519"), privateKeyFile = path.join(root, "synthetic-private.pem") + await writeFile(privateKeyFile, key.privateKey.export({ type: "pkcs8", format: "pem" }), { flag: "wx" }) + const sql = facade(), observation = await captureDatabaseSourceObservation(sql, options()) + const request = { installationId: randomUUID(), sourceIdentityId: randomUUID(), generation: randomUUID(), source: { ...observation.source, inRecovery: false as const } } + const issued = await provisionDatabaseSourceBindingOffline({ request, privateKeyFile, outputDirectory: path.join(root, "registry-one") }) + const pin = issued.deploymentPin, verifier = createDatabaseSourceBindingVerifier(pin) + const headPath = path.join(pin.registryDirectory, "head-" + pin.currentHeadSha256 + ".json") + const head = JSON.parse(await readFile(headPath, "utf8")) + const bindingPath = path.join(pin.registryDirectory, "binding-" + head.body.bindingSha256 + ".json") + return { key, privateKeyFile, sql, observation, request, pin, verifier, headPath, bindingPath } +} + +test("real offline Ed25519 issuance + bounded registry files + genuine same-facade observation verify without READY/generation claims", () => own(async root => { + const f = await fixture(root), result = await f.verifier.verify(f.sql, f.observation, options()) + assert.equal(result.operatorSignatureVerified, true); assert.equal(result.currentHeadPinVerified, true); assert.equal(result.genuineSourceFactsMatched, true) + assert.equal(result.platformProtectionVerified, false); assert.equal(result.generationAuthenticated, false); assert.equal(result.allWriterDrainAuthenticated, false); assert.equal(result.readyAuthorized, false) + assert.equal(result.ordinal, 1) + for (const name of await readdir(f.pin.registryDirectory)) assert(!(await readFile(path.join(f.pin.registryDirectory, name), "utf8")).includes("PRIVATE KEY")) +})) +test("runtime registry read never enrolls absent trusted state or creates any files", () => own(async root => { + const f = await fixture(root), missing = path.join(root, "not-provisioned") + await assert.rejects(readDatabaseSourceBindingRegistry({ ...f.pin, registryDirectory: missing }), refused) + await assert.rejects(readdir(missing)); assert.deepEqual((await readdir(root)).sort(), ["registry-one", "synthetic-private.pem"]) +})) +test("runtime-supplied envelope/unknown trust configuration fields refuse", () => own(async root => { + const f = await fixture(root) + assert.throws(() => createDatabaseSourceBindingVerifier({ ...f.pin, body: f.request } as typeof f.pin)) + await assert.rejects(readDatabaseSourceBindingRegistry(f.request as unknown as typeof f.pin), refused) +})) +test("JSON/copies and other SQL facade cannot forge the existing private source observation capability", () => own(async root => { + const f = await fixture(root) + for (const observation of [structuredClone(f.observation), JSON.parse(JSON.stringify(f.observation)), { ...f.observation }]) await assert.rejects(f.verifier.verify(f.sql, observation, options()), refused) + await assert.rejects(f.verifier.verify(facade(), f.observation, options()), refused) +})) +test("logical clone/dump source with different database OID or cluster identifier refuses even with valid signed old binding", () => own(async root => { + const f = await fixture(root) + for (const changes of [{ database_oid: "16385" }, { system_identifier: "18446744073709551614" }, { database_name: "dump_clone" }]) { + const sql = facade(async (_count, value) => { Object.assign(value, changes) }), observation = await captureDatabaseSourceObservation(sql, options()) + await assert.rejects(f.verifier.verify(sql, observation, options()), refused) + } +})) +test("genuine same-database observation with role/version drift refuses signed source comparison", () => own(async root => { + const f = await fixture(root) + for (const changes of [{ current_user: "other_operator" }, { server_version_num: "180007" }]) { + const sql = facade(async (_count, value) => { Object.assign(value, changes) }), observation = await captureDatabaseSourceObservation(sql, options()) + await assert.rejects(f.verifier.verify(sql, observation, options()), refused) + } +})) +test("attacker-generated self-signed registry cannot replace independently pinned public key", () => own(async root => { + const f = await fixture(root), attacker = generateKeyPairSync("ed25519") + await writeFile(path.join(f.pin.registryDirectory, "operator-public.pem"), attacker.publicKey.export({ type: "spki", format: "pem" })) + await assert.rejects(f.verifier.verify(f.sql, f.observation, options()), refused) +})) +test("missing current head cannot fall back to older filenames or unsigned caller JSON", () => own(async root => { + const f = await fixture(root) + await rm(f.headPath) + await writeFile(path.join(f.pin.registryDirectory, "current.json"), JSON.stringify(f.request)) + await assert.rejects(f.verifier.verify(f.sql, f.observation, options()), refused) +})) +test("current head hash and strict canonical bytes reject tamper, duplicate keys and oversized files", () => own(async root => { + const f = await fixture(root), original = await readFile(f.headPath) + for (const bytes of [Buffer.from("{}\n"), Buffer.from('{"body":{},"body":{},"signature":"x"}\n'), Buffer.alloc(8193, 32), Buffer.concat([original, Buffer.from(" ")])]) { + await writeFile(f.headPath, bytes); await assert.rejects(f.verifier.verify(f.sql, f.observation, options()), refused) + } +})) +test("binding tamper refused before any generation/evidence result", () => own(async root => { + const f = await fixture(root), value = JSON.parse(await readFile(f.bindingPath, "utf8")) + value.body.generation = randomUUID(); await writeFile(f.bindingPath, sourceBindingBytes(value)) + await assert.rejects(f.verifier.verify(f.sql, f.observation, options()), refused) +})) +test("valid signature under the wrong domain refuses despite a matching externally pinned head digest", () => own(async root => { + const f = await fixture(root), value = JSON.parse(await readFile(f.headPath, "utf8")) + value.signature = sign(null, Buffer.from(DATABASE_SOURCE_SIGNATURE_DOMAINS.binding + sourceBindingCanonical(value.body)), f.key.privateKey).toString("base64url") + const bytes = sourceBindingBytes(value), digest = sourceBindingSha256(bytes) + await writeFile(path.join(f.pin.registryDirectory, "head-" + digest + ".json"), bytes) + await assert.rejects(readDatabaseSourceBindingRegistry({ ...f.pin, currentHeadSha256: digest }), refused) +})) +test("binding signature itself is verified even when a genuine signed head authorizes its exact digest", () => own(async root => { + const f = await fixture(root), signedBinding = JSON.parse(await readFile(f.bindingPath, "utf8")), head = JSON.parse(await readFile(f.headPath, "utf8")) + const attacker = generateKeyPairSync("ed25519") + signedBinding.signature = sign(null, Buffer.from(DATABASE_SOURCE_SIGNATURE_DOMAINS.binding + sourceBindingCanonical(signedBinding.body)), attacker.privateKey).toString("base64url") + const bindingBytes = sourceBindingBytes(signedBinding), bindingHash = sourceBindingSha256(bindingBytes) + head.body.bindingSha256 = bindingHash + head.signature = sign(null, Buffer.from(DATABASE_SOURCE_SIGNATURE_DOMAINS.head + sourceBindingCanonical(head.body)), f.key.privateKey).toString("base64url") + const headBytes = sourceBindingBytes(head), headHash = sourceBindingSha256(headBytes) + await writeFile(path.join(f.pin.registryDirectory, "binding-" + bindingHash + ".json"), bindingBytes) + await writeFile(path.join(f.pin.registryDirectory, "head-" + headHash + ".json"), headBytes) + await assert.rejects(readDatabaseSourceBindingRegistry({ ...f.pin, currentHeadSha256: headHash }), refused) +})) +test("even valid operator signatures cannot authorize unknown binding fields or caller READY claims", () => own(async root => { + const f = await fixture(root), signedBinding = JSON.parse(await readFile(f.bindingPath, "utf8")), head = JSON.parse(await readFile(f.headPath, "utf8")) + signedBinding.body.readyAuthorized = true + signedBinding.signature = sign(null, Buffer.from(DATABASE_SOURCE_SIGNATURE_DOMAINS.binding + sourceBindingCanonical(signedBinding.body)), f.key.privateKey).toString("base64url") + const bindingBytes = sourceBindingBytes(signedBinding), bindingHash = sourceBindingSha256(bindingBytes) + head.body.bindingSha256 = bindingHash + head.signature = sign(null, Buffer.from(DATABASE_SOURCE_SIGNATURE_DOMAINS.head + sourceBindingCanonical(head.body)), f.key.privateKey).toString("base64url") + const headBytes = sourceBindingBytes(head), headHash = sourceBindingSha256(headBytes) + await writeFile(path.join(f.pin.registryDirectory, "binding-" + bindingHash + ".json"), bindingBytes) + await writeFile(path.join(f.pin.registryDirectory, "head-" + headHash + ".json"), headBytes) + await assert.rejects(readDatabaseSourceBindingRegistry({ ...f.pin, currentHeadSha256: headHash }), refused) +})) +test("operator explicit next generation increments ordinal, binds previous pinned head and refuses same generation/reuse", () => own(async root => { + const f = await fixture(root) + const next = await provisionDatabaseSourceBindingOffline({ request: { ...f.request, generation: randomUUID() }, privateKeyFile: f.privateKeyFile, + outputDirectory: path.join(root, "registry-two"), previousPin: f.pin }) + const verified = await readDatabaseSourceBindingRegistry(next.deploymentPin) + assert.equal(verified.body.ordinal, 2); assert.equal(verified.head.previousHeadSha256, f.pin.currentHeadSha256) + await assert.rejects(provisionDatabaseSourceBindingOffline({ request: f.request, privateKeyFile: f.privateKeyFile, outputDirectory: path.join(root, "bad"), previousPin: f.pin })) + await assert.rejects(provisionDatabaseSourceBindingOffline({ request: f.request, privateKeyFile: f.privateKeyFile, outputDirectory: f.pin.registryDirectory })) +})) +test("old genuine operator-signed head or binding substituted into current registry refuses rollback under the current independent pin", () => own(async root => { + const f = await fixture(root), next = await provisionDatabaseSourceBindingOffline({ request: { ...f.request, generation: randomUUID() }, privateKeyFile: f.privateKeyFile, + outputDirectory: path.join(root, "registry-two"), previousPin: f.pin }) + const oldHead = await readFile(f.headPath), newHead = path.join(next.deploymentPin.registryDirectory, "head-" + next.deploymentPin.currentHeadSha256 + ".json") + await writeFile(newHead, oldHead) + await assert.rejects(readDatabaseSourceBindingRegistry(next.deploymentPin), refused) +})) +test("registry replacement during genuine source recheck is refused by second fixed artifact read", () => own(async root => { + const f = await fixture(root), sql = facade(async count => { if (count === 4) await writeFile(f.headPath, "{}\n") }) + const observation = await captureDatabaseSourceObservation(sql, options()) + await assert.rejects(f.verifier.verify(sql, observation, options()), refused) +})) +test("abort/expired request refuses without reading any SQL facade", () => own(async root => { + const f = await fixture(root), controller = new AbortController(); controller.abort() + await assert.rejects(f.verifier.verify(f.sql, f.observation, { deadlineAt: Date.now() - 1 }), refused) + await assert.rejects(f.verifier.verify(f.sql, f.observation, { ...options(), signal: controller.signal }), refused) +})) +test("offline CLI uses explicit bounded files, rejects unknown/duplicate args and never auto-installs deployment config", () => own(async root => { + const f = await fixture(root), requestPath = path.join(root, "request.json") + await writeFile(requestPath, JSON.stringify({ ...f.request, generation: randomUUID() })) + const previous = path.join(root, "previous.json"); await writeFile(previous, JSON.stringify(f.pin)) + const issued = await databaseSourceBindingOperatorCli(["--request=" + requestPath, "--private-key=" + f.privateKeyFile, "--output=" + path.join(root, "cli-output"), "--previous-pin=" + previous]) + assert.equal((await readDatabaseSourceBindingRegistry(issued.deploymentPin)).body.ordinal, 2) + await assert.rejects(databaseSourceBindingOperatorCli(["--runtime-self-enroll=true"])) + await assert.rejects(databaseSourceBindingOperatorCli(["--output=x", "--output=y"])) + assert.equal(issued.keyIsolationVerified, false); assert.equal(issued.readyAuthorized, false) +})) +test("private key material is never accepted as the runtime public artifact", () => own(async root => { + const f = await fixture(root); await writeFile(path.join(f.pin.registryDirectory, "operator-public.pem"), await readFile(f.privateKeyFile)) + await assert.rejects(f.verifier.verify(f.sql, f.observation, options()), refused) +})) +test("honest indistinguishability boundary: same metadata after same-database restore is signature evidence only, never generation authentication", () => own(async root => { + const f = await fixture(root), result = await f.verifier.verify(f.sql, f.observation, options()) + assert.equal(result.generationAuthenticated, false); assert.equal(result.platformProtectionVerified, false); assert.equal(result.readyAuthorized, false) +})) diff --git a/backend/src/db/database-source-binding.ts b/backend/src/db/database-source-binding.ts new file mode 100644 index 00000000..a269ffaa --- /dev/null +++ b/backend/src/db/database-source-binding.ts @@ -0,0 +1,112 @@ +import { createHash, createPublicKey, verify } from "node:crypto" +import { constants } from "node:fs" +import { lstat, open, realpath } from "node:fs/promises" +import path from "node:path" +import { z } from "zod" +import { assertDatabaseSourceObservationCurrent, type DatabaseSourceObservation, type DatabaseSourceObservationOptions } from "./database-source-observation.js" +import type { ProbeStorageSql } from "../services/server-probe-journal-maintenance.js" + +// Public verification only. No private key, signing, enrollment, SQL mutation, +// configured pool, provisioning or automatic current-head advancement here. +export class DatabaseSourceBindingError extends Error { readonly code = "DATABASE_SOURCE_BINDING_REFUSED" } +const fail = (): never => { throw new DatabaseSourceBindingError("Доверенная подпись и текущая привязка источника БД не подтверждены") } +const sha = z.string().regex(/^[a-f0-9]{64}$/) +const uuid = z.string().uuid() +const text = z.string().min(1).max(200).regex(/^[^\x00-\x1f\x7f]+$/) +const source = z.object({ systemIdentifier: z.string().regex(/^[1-9][0-9]{0,19}$/).refine(value => BigInt(value) <= 18446744073709551615n), + databaseOid: z.number().int().positive().max(4294967295), databaseName: text, sessionUser: text, currentUser: text, + inRecovery: z.literal(false), serverVersionNum: z.number().int().min(180000).max(189999) }).strict() +export const sourceBindingBodySchema = z.object({ format: z.literal("database-source-binding-v1"), installationId: uuid, + sourceIdentityId: uuid, generation: uuid, ordinal: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), source }).strict() +export const sourceRegistryHeadSchema = z.object({ format: z.literal("database-source-registry-head-v1"), installationId: uuid, + sourceIdentityId: uuid, generation: uuid, ordinal: z.number().int().positive().max(Number.MAX_SAFE_INTEGER), bindingSha256: sha, + previousHeadSha256: sha.nullable() }).strict().refine(value => (value.ordinal === 1) === (value.previousHeadSha256 === null)) +const deployment = z.object({ registryDirectory: z.string().min(1).max(4096), installationId: uuid, publicKeySpkiSha256: sha, currentHeadSha256: sha }).strict() +const envelope = z.object({ body: z.unknown(), signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }).strict() +export type DatabaseSourceBindingBody = z.infer +export type DatabaseSourceRegistryHead = z.infer +export type DatabaseSourceDeploymentPin = z.infer +export const DATABASE_SOURCE_SIGNATURE_DOMAINS = Object.freeze({ binding: "mmapp:database-source-binding:ed25519:v1\n", head: "mmapp:database-source-head:ed25519:v1\n" }) + +export function sourceBindingCanonical(value: unknown): string { + return JSON.stringify(value, (_key, item: unknown) => item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries(Object.entries(item).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : item) +} +export const sourceBindingSha256 = (bytes: Uint8Array | string) => createHash("sha256").update(bytes).digest("hex") +export const sourceBindingBytes = (value: unknown) => Buffer.from(sourceBindingCanonical(value) + "\n", "utf8") +function parseExact(bytes: Buffer) { + const value: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)) + // Exact canonical bytes also reject duplicate JSON keys, alternate numeric + // spellings, trailing content and ambiguous/noncanonical Unicode encodings. + if (!sourceBindingBytes(value).equals(bytes)) fail() + return envelope.parse(value) +} +async function boundedFile(filename: string, maximum = 8192): Promise { + const before = await lstat(filename) + if (!before.isFile() || before.isSymbolicLink() || before.size < 1 || before.size > maximum) fail() + const handle = await open(filename, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)) + try { + const stat = await handle.stat() + if (!stat.isFile() || stat.dev !== before.dev || stat.ino !== before.ino || stat.size !== before.size) fail() + const bytes = Buffer.alloc(stat.size + 1); let length = 0 + while (length < bytes.length) { const got = await handle.read(bytes, length, bytes.length - length, length); if (!got.bytesRead) break; length += got.bytesRead } + const after = await handle.stat() + if (length !== stat.size || after.size !== stat.size || after.mtimeMs !== stat.mtimeMs || after.ctimeMs !== stat.ctimeMs) fail() + return bytes.subarray(0, length) + } finally { await handle.close() } +} +function fence(options: DatabaseSourceObservationOptions) { + if (!Number.isSafeInteger(options.deadlineAt) || options.deadlineAt <= Date.now() || options.signal?.aborted) fail() +} + +/** Input is privileged deployment configuration, never request JSON. Actual + * protection/ACL and current-process invalidation are separate deployment gates. + * This deliberately grants signature/source evidence only, NOT generation/READY. */ +export async function readDatabaseSourceBindingRegistry(raw: DatabaseSourceDeploymentPin) { + try { + const cfg = deployment.parse(raw), directory = path.resolve(cfg.registryDirectory) + if (!path.isAbsolute(cfg.registryDirectory) || directory !== await realpath(directory)) fail() + const stat = await lstat(directory); if (!stat.isDirectory() || stat.isSymbolicLink()) fail() + const publicBytes = await boundedFile(path.join(directory, "operator-public.pem"), 4096) + if (publicBytes.toString("ascii").includes("PRIVATE KEY")) fail() + const publicKey = createPublicKey(publicBytes) + if (publicKey.type !== "public" || publicKey.asymmetricKeyType !== "ed25519") fail() + const spki = publicKey.export({ type: "spki", format: "der" }) + if (sourceBindingSha256(spki) !== cfg.publicKeySpkiSha256) fail() + const headBytes = await boundedFile(path.join(directory, "head-" + cfg.currentHeadSha256 + ".json")) + if (sourceBindingSha256(headBytes) !== cfg.currentHeadSha256) fail() + const signedHead = parseExact(headBytes), head = sourceRegistryHeadSchema.parse(signedHead.body) + const check = (domain: string, body: unknown, signature: string) => verify(null, Buffer.from(domain + sourceBindingCanonical(body)), publicKey, Buffer.from(signature, "base64url")) + if (head.installationId !== cfg.installationId || !check(DATABASE_SOURCE_SIGNATURE_DOMAINS.head, head, signedHead.signature)) fail() + const bindingBytes = await boundedFile(path.join(directory, "binding-" + head.bindingSha256 + ".json")) + if (sourceBindingSha256(bindingBytes) !== head.bindingSha256) fail() + const signedBinding = parseExact(bindingBytes), body = sourceBindingBodySchema.parse(signedBinding.body) + if (!check(DATABASE_SOURCE_SIGNATURE_DOMAINS.binding, body, signedBinding.signature) || body.installationId !== head.installationId || + body.sourceIdentityId !== head.sourceIdentityId || body.generation !== head.generation || body.ordinal !== head.ordinal) fail() + return Object.freeze({ body: Object.freeze({ ...body, source: Object.freeze({ ...body.source }) }), + head: Object.freeze(head), publicKeySpkiSha256: cfg.publicKeySpkiSha256, headSha256: cfg.currentHeadSha256, + platformProtectionVerified: false as const, generationAuthenticated: false as const, readyAuthorized: false as const }) + } catch { return fail() } +} + +/** Genuine same-facade observation is mandatory. Serialized source facts never + * become a capability. Registry pin comes only from future privileged assembly. */ +export function createDatabaseSourceBindingVerifier(raw: DatabaseSourceDeploymentPin) { + const cfg = Object.freeze(deployment.parse(raw)) + return Object.freeze({ async verify(sql: ProbeStorageSql, observation: DatabaseSourceObservation, options: DatabaseSourceObservationOptions) { + try { + fence(options); await assertDatabaseSourceObservationCurrent(sql, observation, options) + const before = await readDatabaseSourceBindingRegistry(cfg); fence(options) + if (sourceBindingCanonical(before.body.source) !== sourceBindingCanonical(observation.source)) fail() + await assertDatabaseSourceObservationCurrent(sql, observation, options); fence(options) + const after = await readDatabaseSourceBindingRegistry(cfg); fence(options) + if (sourceBindingCanonical(before) !== sourceBindingCanonical(after)) fail() + await assertDatabaseSourceObservationCurrent(sql, observation, options); fence(options) + return Object.freeze({ format: "database-source-binding-evidence-v1" as const, + installationId: before.body.installationId, sourceIdentityId: before.body.sourceIdentityId, generation: before.body.generation, + ordinal: before.body.ordinal, headSha256: before.headSha256, bindingSha256: before.head.bindingSha256, + operatorSignatureVerified: true as const, currentHeadPinVerified: true as const, genuineSourceFactsMatched: true as const, + platformProtectionVerified: false as const, generationAuthenticated: false as const, allWriterDrainAuthenticated: false as const, readyAuthorized: false as const }) + } catch { return fail() } + } }) +} diff --git a/backend/src/db/database-source-capture-adopted.sql-helper.ts b/backend/src/db/database-source-capture-adopted.sql-helper.ts new file mode 100644 index 00000000..27afa20a --- /dev/null +++ b/backend/src/db/database-source-capture-adopted.sql-helper.ts @@ -0,0 +1,137 @@ +/** Test-only adopted-writer capture regression fixture. Never imported by + * startup, restore, dispatch or READY. Raw/unadopted SQL is NOT fenced. */ +import { createHash } from "node:crypto" +import { resolve } from "node:path" +import { pathToFileURL } from "node:url" +if (process.env.MMAPP_OWN_SOURCE_CAPTURE !== "true" || !process.env.MMAPP_OWN_SOURCE_CAPTURE_ROOT || process.env.DATABASE_URL) throw Error("Owned capture runner required") +const source = (path: string) => pathToFileURL(resolve(process.env.MMAPP_OWN_SOURCE_CAPTURE_ROOT!, "backend/src", path)).href +const { PARTITION_WRITER_ADVISORY_KEY } = await import(source("services/partition-write-transaction.ts")) as typeof import("../services/partition-write-transaction.js") +const { captureMigrationData } = await import(source("db/migration-data-inventory.ts")) as typeof import("./migration-data-inventory.js") +export interface CaptureClient { + query(sql: string, values?: unknown[]): Promise<{ rows: Record[]; command?: string }> + release(destroy?: boolean): void +} +export interface CapturePool { connect(): Promise } +export interface CapturePolicy { databaseName: string; captureRole: string; runtimeRole: string; timeoutMs: number } +export class AdoptedSourceCaptureError extends Error { + readonly code = "ADOPTED_SOURCE_CAPTURE_REFUSED" + constructor(message: string, readonly outcomeUnknown = false, options?: ErrorOptions) { super(message, options) } +} +const fail = (message: string): never => { throw new AdoptedSourceCaptureError(message) } +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex") +export const CAPTURE_SQL = Object.freeze({ + roleClosure: `WITH RECURSIVE roles(oid) AS ( + SELECT oid FROM pg_catalog.pg_roles WHERE rolname=$1 + UNION SELECT m.roleid FROM pg_catalog.pg_auth_members m JOIN roles r ON m.member=r.oid) + SELECT r.oid::text AS oid,r.rolname,r.rolsuper,r.rolcreaterole,r.rolcreatedb,r.rolreplication,r.rolbypassrls,r.rolcanlogin + FROM pg_catalog.pg_roles r JOIN roles c ON c.oid=r.oid ORDER BY r.oid LIMIT 129`, + privilege: `SELECT + EXISTS(SELECT 1 FROM pg_catalog.pg_database d WHERE d.datname=pg_catalog.current_database() AND (d.datdba=ANY($1::oid[]) OR EXISTS(SELECT 1 FROM unnest($1::oid[]) r WHERE pg_catalog.has_database_privilege(r,d.oid,'CREATE')))) AS database_owner_or_create, + EXISTS(SELECT 1 FROM pg_catalog.pg_namespace n WHERE n.nspname='public' AND (n.nspowner=ANY($1::oid[]) OR EXISTS(SELECT 1 FROM unnest($1::oid[]) r WHERE pg_catalog.has_schema_privilege(r,n.oid,'CREATE')))) AS schema_owner_or_create, + EXISTS(SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relowner=ANY($1::oid[])) AS relation_owner, + EXISTS(SELECT 1 FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace WHERE n.nspname='public' AND p.proowner=ANY($1::oid[])) AS routine_owner, + EXISTS(SELECT 1 FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace WHERE p.prosecdef AND n.nspname NOT IN ('pg_catalog','information_schema') AND EXISTS(SELECT 1 FROM unnest($1::oid[]) r WHERE pg_catalog.has_function_privilege(r,p.oid,'EXECUTE'))) AS callable_security_definer`, + ownLock: `SELECT count(*)::int AS n FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() AND classid=0 AND objid=$1::oid AND objsubid=1 AND mode='ExclusiveLock' AND granted`, + context: `SELECT d.oid::text AS database_oid,d.datname,SESSION_USER::text AS session_user,CURRENT_USER::text AS current_user, + pg_catalog.pg_backend_pid() AS backend_pid,c.system_identifier::text AS system_identifier, + pg_catalog.pg_current_snapshot()::text AS snapshot,pg_catalog.transaction_timestamp()::text AS transaction_started, + pg_catalog.current_setting('transaction_isolation') AS isolation,pg_catalog.current_setting('transaction_read_only') AS read_only, + pg_catalog.pg_is_in_recovery() AS in_recovery + FROM pg_catalog.pg_database d CROSS JOIN pg_catalog.pg_control_system() c WHERE d.datname=pg_catalog.current_database()`, +}) +function policy(raw: CapturePolicy) { + if (!raw || Object.getPrototypeOf(raw) !== Object.prototype || Object.keys(raw).sort().join(" ") !== "captureRole databaseName runtimeRole timeoutMs" || + Object.values(Object.getOwnPropertyDescriptors(raw)).some(d => !("value" in d) || !d.enumerable) || Reflect.ownKeys(raw).length !== 4 || + ![raw.databaseName, raw.captureRole, raw.runtimeRole].every(v => typeof v === "string" && /^[a-z][a-z0-9_]{0,62}$/.test(v)) || + raw.captureRole === raw.runtimeRole || !Number.isSafeInteger(raw.timeoutMs) || raw.timeoutMs < 100 || raw.timeoutMs > 30000) fail("Exact bounded trusted capture policy required") + return Object.freeze(structuredClone(raw)) +} +/** Owns the actual exclusive lock on the SAME checked-out backend used for + * capture. It waits for existing adopted transactions and blocks later adopted + * transactions/DDL before their first relation access. No lock callback seam. */ +export async function openAdoptedSourceCapture(pool: CapturePool, raw: CapturePolicy, options: { signal?: AbortSignal } = {}) { + const cfg = policy(raw) + if (options.signal?.aborted) fail("Capture aborted before checkout") + const client = await pool.connect() + const deadlineAt = Date.now() + cfg.timeoutMs + let closed = false, busy = false, used = false, transaction = false, commitSent = false + const release = (discard: boolean) => { + if (!closed) { closed = true; clearTimeout(timer); options.signal?.removeEventListener("abort", onAbort); client.release(discard) } + } + const destroy = () => release(true) + const onAbort = () => destroy() + const timer = setTimeout(destroy, cfg.timeoutMs); timer.unref() + options.signal?.addEventListener("abort", onAbort, { once: true }) + const fence = () => { + if (closed || options.signal?.aborted || Date.now() >= deadlineAt) { destroy(); fail("Capture closed, aborted or expired") } + } + const query: CaptureClient["query"] = async (sql, values) => { + fence(); const result = await client.query(sql, values); fence(); return result + } + async function rolePolicy() { + const roles = (await query(CAPTURE_SQL.roleClosure, [cfg.runtimeRole])).rows + if (!roles.length || roles.length > 128 || !roles.some(r => r.rolname === cfg.runtimeRole && r.rolcanlogin === true) || roles.some(r => + typeof r.oid !== "string" || !/^[1-9][0-9]*$/.test(r.oid) || typeof r.rolname !== "string" || r.rolname.startsWith("pg_") || + [r.rolsuper,r.rolcreaterole,r.rolcreatedb,r.rolreplication,r.rolbypassrls].some(v => v !== false))) fail("Runtime role/membership has privileged bypass") + const rows = (await query(CAPTURE_SQL.privilege, [roles.map(r => r.oid)])).rows + if (rows.length !== 1 || Object.values(rows[0]).some(v => v !== false)) fail("Runtime role can own/create application schema or execute security-definer bypass") + return digest({ roles, privileges: rows[0] }) + } + async function ownedLock() { const rows = (await query(CAPTURE_SQL.ownLock, [PARTITION_WRITER_ADVISORY_KEY])).rows; if (rows.length !== 1 || rows[0].n !== 1) fail("Same-backend exclusive adopted-writer lease missing") } + try { + await query("SET lock_timeout='" + cfg.timeoutMs + "ms'") + await query("SET statement_timeout='" + cfg.timeoutMs + "ms'") + const identity = (await query("SELECT pg_catalog.current_database() AS database_name,CURRENT_USER::text AS current_user,SESSION_USER::text AS session_user")).rows + if (identity.length !== 1 || identity[0].database_name !== cfg.databaseName || identity[0].current_user !== cfg.captureRole || identity[0].session_user !== cfg.captureRole) fail("Capture connection is not the explicitly provisioned source/operator") + // Authenticate privileges before waiting: a superuser/owner application + // role must not be mislabeled as an enforceable limited writer. + const rolePolicySha256 = await rolePolicy() + // SESSION lease in autocommit first: its waiting SELECT must finish BEFORE + // BEGIN RR. Taking an xact lock inside RR could pin an old snapshot while + // waiting and miss the writer's COMMIT even after the lock is granted. + await query("SELECT pg_catalog.pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]); await ownedLock() + return Object.freeze({ + async capture() { + if (closed || busy || used) fail("Capture handle closed/concurrent/reused") + used = true; busy = true + try { + await ownedLock() + if (await rolePolicy() !== rolePolicySha256) fail("Role policy changed while acquiring writer drain") + await query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY"); transaction = true + for (const statement of ["SET LOCAL search_path=pg_catalog", "SET LOCAL TimeZone='UTC'", "SET LOCAL DateStyle='ISO, YMD'", "SET LOCAL standard_conforming_strings=on", "SET LOCAL statement_timeout='5s'", + "SET LOCAL row_security=off", "SET LOCAL extra_float_digits=3", "SET LOCAL bytea_output=hex", "SET LOCAL IntervalStyle=postgres", "SET LOCAL client_encoding='UTF8'", "SET LOCAL lc_monetary='C'", "SET LOCAL lc_numeric='C'"]) await query(statement) + const first = (await query(CAPTURE_SQL.context)).rows + if (first.length !== 1 || first[0].datname !== cfg.databaseName || first[0].current_user !== cfg.captureRole || first[0].session_user !== cfg.captureRole || first[0].isolation !== "repeatable read" || first[0].read_only !== "on" || first[0].in_recovery !== false) fail("Owned source snapshot context unavailable") + const captured = await captureMigrationData({ query }, { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["mmapp_control"] }, deadlineAt, maxRows: 100000, maxBytes: 16777216 }) + const last = (await query(CAPTURE_SQL.context)).rows + if (JSON.stringify(first) !== JSON.stringify(last)) fail("Source/backend/snapshot changed across schema/data capture") + await ownedLock(); commitSent = true + const committed = await query("COMMIT") + if (committed.command !== "COMMIT") fail("Owned capture COMMIT acknowledgement unavailable") + transaction = false + // Role catalogs are re-read OUTSIDE the old RR snapshot so membership + // changes during capture are not hidden by snapshot consistency. + if (await rolePolicy() !== rolePolicySha256) fail("Role policy changed during capture") + await ownedLock() + return Object.freeze({ format: "adopted-source-capture-v1" as const, source: Object.freeze(structuredClone(first[0])), + rolePolicySha256, schemaSha256: captured.schemaSha256, dataSha256: captured.sha256, rowCount: captured.rowCount, sequenceCount: captured.sequences.length, + adoptedWriterDrainAuthenticated: true as const, ownedRepeatableReadCaptureAuthenticated: true as const, + allWriterDrainAuthenticated: false as const, sourceGenerationAuthenticated: false as const, readyAuthorized: false as const }) + } catch (cause) { + destroy() + throw new AdoptedSourceCaptureError("Owned adopted-writer source capture refused; no replay", transaction || commitSent, { cause }) + } finally { busy = false } + }, + async close() { + if (closed) return + if (busy) fail("Capture still active") + try { + const released = (await query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY])).rows + if (released.length !== 1 || released[0].unlocked !== true) fail("Adopted-writer lease release unknown") + await query("RESET lock_timeout"); await query("RESET statement_timeout") + release(false) + } catch (cause) { destroy(); throw new AdoptedSourceCaptureError("Capture owner release unconfirmed", true, { cause }) } + }, + }) + } catch (cause) { destroy(); throw cause } +} diff --git a/backend/src/db/database-source-capture-adopted.sql.test.ts b/backend/src/db/database-source-capture-adopted.sql.test.ts new file mode 100644 index 00000000..0c39b3ce --- /dev/null +++ b/backend/src/db/database-source-capture-adopted.sql.test.ts @@ -0,0 +1,180 @@ +import assert from "node:assert/strict" +import { randomBytes, randomUUID } from "node:crypto" +import type { Pool as PgPool } from "pg" +import { basename, resolve, relative, isAbsolute } from "node:path" +import { tmpdir } from "node:os" +import { readFile } from "node:fs/promises" +import { createRequire } from "node:module" +import { pathToFileURL } from "node:url" +import { openAdoptedSourceCapture, CAPTURE_SQL, AdoptedSourceCaptureError, type CapturePool } from "./database-source-capture-adopted.sql-helper.js" +assert.equal(process.env.MMAPP_OWN_SOURCE_CAPTURE, "true") +assert(process.env.MMAPP_OWN_SOURCE_CAPTURE_DIRECTORY && basename(process.env.MMAPP_OWN_SOURCE_CAPTURE_DIRECTORY).startsWith("mm-source-capture-")) +assert(process.env.MMAPP_OWN_SOURCE_CAPTURE_ROOT) +assert.equal(process.env.DATABASE_URL, undefined) +const sourceRoot = process.env.MMAPP_OWN_SOURCE_CAPTURE_ROOT! +const require = createRequire(resolve(sourceRoot, "backend/package.json")) +const { Pool } = require("pg") as typeof import("pg") +const source = (path: string) => pathToFileURL(resolve(sourceRoot, "backend/src", path)).href +const { withPartitionWrite, PARTITION_WRITER_ADVISORY_KEY } = await import(source("services/partition-write-transaction.ts")) as typeof import("../services/partition-write-transaction.js") +const url = new URL(process.env.MMAPP_OWN_SOURCE_CAPTURE_ADMIN_URL ?? "") +assert.equal(url.protocol, "postgresql:"); assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") +assert.equal(url.username, "source_operator"); assert(url.password); assert(url.port); assert.equal(url.search, "") +const directory = resolve(process.env.MMAPP_OWN_SOURCE_CAPTURE_DIRECTORY!) +const relativeDirectory = relative(resolve(tmpdir()), directory) +assert(relativeDirectory && !relativeDirectory.startsWith("..") && !isAbsolute(relativeDirectory)) +assert.equal(resolve(process.cwd()), directory) +const owned = JSON.parse(await readFile(resolve(directory, "owned-source-capture.json"), "utf8")) +assert.equal(owned.ownerToken, process.env.MMAPP_OWN_SOURCE_CAPTURE_TOKEN) +assert.equal(owned.port, Number(url.port)); assert.equal(owned.username, url.username) +const admin = new Pool({ connectionString: url.toString(), max: 4 }), pools: PgPool[] = [], names: string[] = [], roles: string[] = [], cases: string[] = [] +const handles: Awaited>[] = [] +async function fixture() { + const suffix = randomUUID().replaceAll("-", ""), name = "source_" + suffix, role = "runtime_" + suffix, password = randomBytes(24).toString("hex") + assert(/^source_[a-f0-9]{32}$/.test(name) && /^runtime_[a-f0-9]{32}$/.test(role)) + roles.push(role); names.push(name) + await admin.query('CREATE ROLE "' + role + '" LOGIN PASSWORD \'' + password + "'") + await admin.query('CREATE DATABASE "' + name + '" TEMPLATE template0') + const target = new URL(url); target.pathname = "/" + name + const operator = new Pool({ connectionString: target.toString(), max: 6, connectionTimeoutMillis: 3000 }); pools.push(operator) + target.username = role; target.password = password + const runtime = new Pool({ connectionString: target.toString(), max: 6, connectionTimeoutMillis: 3000 }); pools.push(runtime) + await operator.query("CREATE SCHEMA mmapp_control; CREATE TABLE public.sample(id integer PRIMARY KEY,payload text NOT NULL); INSERT INTO public.sample VALUES(1,'owned-seed'); CREATE SEQUENCE public.sample_seq") + await operator.query('GRANT SELECT,INSERT,UPDATE,DELETE ON public.sample TO "' + role + '"; GRANT USAGE,SELECT,UPDATE ON SEQUENCE public.sample_seq TO "' + role + '"') + const policy = { databaseName: name, captureRole: "source_operator", runtimeRole: role, timeoutMs: 3000 } + async function open(pool: CapturePool = operator, options: { signal?: AbortSignal } = {}) { const h = await openAdoptedSourceCapture(pool, policy, options); handles.push(h); return h } + return { name, role, operator, runtime, policy, open } +} +async function until(check: () => Promise) { + const deadline = performance.now() + 4000 + while (!await check()) { if (performance.now() >= deadline) throw Error("Owned SQL condition did not become observable"); await new Promise(r => setTimeout(r, 10)) } +} +const lockWaiter = (pool: PgPool) => pool.query("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND objid=$1::oid AND NOT granted) AS waiting", [PARTITION_WRITER_ADVISORY_KEY]).then(r => r.rows[0].waiting === true) +function wrapped(pool: PgPool, hook: (sql: string, after: () => Promise<{ rows: Record[]; command?: string }>) => Promise<{ rows: Record[]; command?: string }>): CapturePool { + return { async connect() { const c = await pool.connect(); return { release(destroy?: boolean) { c.release(destroy) }, query(sql: string, values?: unknown[]) { return hook(sql, async () => { const r = await c.query(sql, values); return { rows: r.rows, command: r.command } }) } } } } +} +try { + const good = await fixture(), baseline = await good.open(), evidence = await baseline.capture() + assert.equal(evidence.rowCount, 1); assert.equal(evidence.sequenceCount, 1); assert.equal(evidence.source.datname, good.name) + assert.equal(evidence.source.current_user, "source_operator"); assert.equal(evidence.ownedRepeatableReadCaptureAuthenticated, true) + assert.equal(evidence.adoptedWriterDrainAuthenticated, true); assert.equal(evidence.allWriterDrainAuthenticated, false); assert.equal(evidence.sourceGenerationAuthenticated, false); assert.equal(evidence.readyAuthorized, false) + assert.match(evidence.schemaSha256, /^[0-9a-f]{64}$/); assert.match(evidence.dataSha256, /^[0-9a-f]{64}$/) + await assert.rejects(baseline.capture()); await baseline.close(); cases.push("actual fixed source/schema/data capture owns one RR READ ONLY backend and never asserts whole-writer/generation/READY") + + let releaseWriter!: () => void, writerEntered!: () => void + const entered = new Promise(r => { writerEntered = r }), release = new Promise(r => { releaseWriter = r }) + const firstWriter = withPartitionWrite(async ({ client }) => { await client.query("UPDATE public.sample SET payload='committed-before-capture' WHERE id=1"); await client.query("INSERT INTO public.sample VALUES(3,'newly-committed-before-snapshot')"); writerEntered(); await release }, { connect: () => good.runtime.connect() }) + let captureTransactionBegan = false + const tracedPool = wrapped(good.operator, async (sql, run) => { if (sql.startsWith("BEGIN ISOLATION LEVEL REPEATABLE READ")) captureTransactionBegan = true; return run() }) + await entered; const opening = good.open(tracedPool); await until(() => lockWaiter(good.operator)) + assert.equal(captureTransactionBegan, false, "RR snapshot must not start while exclusive session lock is waiting") + releaseWriter(); await firstWriter + const drained = await opening; assert.equal(captureTransactionBegan, false) + const afterWriter = await drained.capture(); assert.equal(captureTransactionBegan, true) + assert.equal(afterWriter.rowCount, 2, "capture includes the INSERT committed by the writer that held the shared lock") + assert.equal(afterWriter.schemaSha256, evidence.schemaSha256); assert.notEqual(afterWriter.dataSha256, evidence.dataSha256); await drained.close() + cases.push("exclusive SESSION drain precedes BEGIN RR: real pending INSERT/UPDATE commits before fresh snapshot and newly committed row is included") + + const blocking = await good.open(); let lateDmlEntered = false + const lateDml = withPartitionWrite(async ({ client }) => { lateDmlEntered = true; await client.query("INSERT INTO public.sample VALUES(2,'after-release')") }, { connect: () => good.runtime.connect() }) + const ddl = await good.operator.connect(); const ddlPid = (await ddl.query("SELECT pg_backend_pid() AS pid")).rows[0].pid + const lateDdl = (async () => { await ddl.query("SELECT pg_advisory_lock_shared($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]); await ddl.query("CREATE TABLE public.admitted_ddl(id integer)"); await ddl.query("SELECT pg_advisory_unlock_shared($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]); ddl.release() })() + await until(async () => (await good.operator.query("SELECT count(*)::int AS n FROM pg_locks WHERE locktype='advisory' AND objid=$1::oid AND NOT granted", [PARTITION_WRITER_ADVISORY_KEY])).rows[0].n === 2) + assert.equal(lateDmlEntered, false); const held = await blocking.capture(); assert.equal(held.rowCount, 2) + assert.equal((await good.operator.query("SELECT EXISTS(SELECT 1 FROM pg_stat_activity WHERE pid=$1 AND wait_event_type='Lock') AS waiting", [ddlPid])).rows[0].waiting, true) + await blocking.close(); await Promise.all([lateDml, lateDdl]); assert.equal(lateDmlEntered, true) + cases.push("same real exclusive lease blocks new admitted DML and migration-style DDL before target SQL until capture releases") + + const bypass = await fixture(), bypassHandle = await bypass.open() + await bypass.runtime.query("INSERT INTO public.sample VALUES(2,'raw-unadopted-bypass')") + assert.equal((await bypassHandle.capture()).rowCount, 2); await bypassHandle.close() + cases.push("actual raw runtime SQL bypasses advisory drain, proving existing key cannot certify ALL writers") + + const rr = await fixture(); let inserted = false + const rrPool = wrapped(rr.operator, async (sql, run) => { + const result = await run() + if (sql === CAPTURE_SQL.context && !inserted) { inserted = true; await rr.runtime.query("INSERT INTO public.sample VALUES(2,'concurrent-raw')") } + return result + }) + const rrHandle = await rr.open(rrPool), rrEvidence = await rrHandle.capture(); assert.equal(rrEvidence.rowCount, 1) + assert.equal((await rr.operator.query("SELECT count(*)::int AS n FROM public.sample")).rows[0].n, 2); await rrHandle.close() + cases.push("actual RR snapshot stays coherent while unadopted concurrent DML commits; consistency is not physical quiescence") + + const seq = await fixture(); let sequenceMoved = false + const seqPool = wrapped(seq.operator, async (sql, run) => { + const result = await run() + if (sql.startsWith("/* migration-data:sequence */") && !sequenceMoved) { sequenceMoved = true; await seq.runtime.query("SELECT nextval('public.sample_seq')") } + return result + }) + const seqHandle = await seq.open(seqPool); await assert.rejects(seqHandle.capture()); await assert.rejects(seqHandle.capture()) + cases.push("actual nextval bypass during RR capture is detected by existing before/after sequence inventory and poisons capture") + + const unsafe = await fixture() + await admin.query('ALTER ROLE "' + unsafe.role + '" SUPERUSER'); await assert.rejects(unsafe.open()); await admin.query('ALTER ROLE "' + unsafe.role + '" NOSUPERUSER') + cases.push("actual superuser runtime refuses before lock/capture") + await admin.query('GRANT pg_write_all_data TO "' + unsafe.role + '"'); await assert.rejects(unsafe.open()); await admin.query('REVOKE pg_write_all_data FROM "' + unsafe.role + '"') + cases.push("actual indirect builtin write-all role membership refuses") + await unsafe.operator.query('GRANT CREATE ON SCHEMA public TO "' + unsafe.role + '"'); await assert.rejects(unsafe.open()); await unsafe.operator.query('REVOKE CREATE ON SCHEMA public FROM "' + unsafe.role + '"') + cases.push("actual runtime CREATE privilege on application schema refuses") + await unsafe.operator.query('ALTER TABLE public.sample OWNER TO "' + unsafe.role + '"'); await assert.rejects(unsafe.open()); await unsafe.operator.query('ALTER TABLE public.sample OWNER TO source_operator') + cases.push("actual application-relation ownership refuses even without superuser flags") + await unsafe.operator.query("CREATE FUNCTION public.unsafe_bypass() RETURNS integer LANGUAGE sql SECURITY DEFINER AS 'SELECT 1'") + await assert.rejects(unsafe.open()); await unsafe.operator.query("DROP FUNCTION public.unsafe_bypass()") + cases.push("actual callable SECURITY DEFINER routine refuses privileged bypass surface") + + const changed = await fixture(); let roleChanged = false + const changedPool = wrapped(changed.operator, async (sql, run) => { const r = await run(); if (sql === "COMMIT" && !roleChanged) { roleChanged = true; await admin.query('GRANT pg_write_all_data TO "' + changed.role + '"') }; return r }) + const changedHandle = await changed.open(changedPool); await assert.rejects(changedHandle.capture()); await admin.query('REVOKE pg_write_all_data FROM "' + changed.role + '"') + cases.push("real role drift after RR COMMIT is detected by fresh outside-snapshot privilege read") + + const lost = await fixture() + const lostPool = wrapped(lost.operator, async (sql, run) => { const r = await run(); if (sql === "COMMIT") throw Error("owned fixture loses actual read-only COMMIT ACK"); return r }) + const lostHandle = await lost.open(lostPool) + await assert.rejects(lostHandle.capture(), e => e instanceof AdoptedSourceCaptureError && e.outcomeUnknown) + await assert.rejects(lostHandle.capture()); const fresh = await lost.open(); await fresh.close() + cases.push("lost actual RR COMMIT acknowledgement refuses evidence, destroys owner, and does not replay capture") + + const aborted = await fixture(), abortController = new AbortController() + const abortedHandle = await aborted.open(aborted.operator, { signal: abortController.signal }) + abortController.abort(); await assert.rejects(abortedHandle.capture()); await abortedHandle.close() + const afterAbort = await aborted.open(); await afterAbort.close() + const beforeAbort = new AbortController(); beforeAbort.abort() + await assert.rejects(aborted.open(aborted.operator, { signal: beforeAbort.signal })) + cases.push("actual abort destroys owned session, rejects capture/replay and releases exclusive lease; pre-aborted checkout refuses") + + const expired = await fixture() + const expiredHandle = await openAdoptedSourceCapture(expired.operator, { ...expired.policy, timeoutMs: 100 }) + handles.push(expiredHandle) + await new Promise(resolve => setTimeout(resolve, 150)) + await assert.rejects(expiredHandle.capture()); await expiredHandle.close() + const afterExpiry = await expired.open(); await afterExpiry.close() + cases.push("bounded whole capture lease expires, refuses evidence/replay and releases real exclusive session lock") + + const rolled = await fixture(); let commits = 0, rolledTag: string | undefined + // A successful COMMIT cannot be relabeled as ROLLBACK. This separate wrapper + // issues division-by-zero on the exact backend immediately before COMMIT. + const abortedCommitPool: CapturePool = { async connect() { const c = await rolled.operator.connect(); return { + release(discard?: boolean) { c.release(discard) }, async query(sql: string, values?: unknown[]) { + if (sql === "COMMIT") { commits++; await assert.rejects(c.query("SELECT 1/0")) } + const result = await c.query(sql, values); if (sql === "COMMIT") rolledTag = result.command + return { rows: result.rows, command: result.command } + }, + } } } + const rolledHandle = await rolled.open(abortedCommitPool) + await assert.rejects(rolledHandle.capture(), e => e instanceof AdoptedSourceCaptureError && e.outcomeUnknown) + assert.equal(commits, 1); assert.equal(rolledTag, "ROLLBACK"); await assert.rejects(rolledHandle.capture()) + cases.push("actual aborted read-only COMMIT resolves to ROLLBACK, produces no evidence and never replays capture") + + const wrong = await fixture(); await assert.rejects(openAdoptedSourceCapture(wrong.operator, { ...wrong.policy, captureRole: "other_operator" })) + cases.push("source checkout must match explicitly expected database and capture-role/session identity") + console.log(JSON.stringify({ cases: cases.length, names: cases, databases: names.length, actualPostgresRoles: roles.length, actualSql: true, + actualCurrentProductionPartitionWriterModule: true, importerControlCoveredBySeparatePermanentProfile: true, + permanentTestOnlySourceCaptureFixture: true, actualExistingSchemaDataInventory: true, rawUnadoptedBypassDemonstrated: true, + ownedRepeatableReadCapture: true, allWriterDrainAuthenticated: false, sourceGenerationAuthenticated: false, readyAuthorized: false, + productionDatabaseTouched: false, applicationConfigOrDotenvLoaded: false, noHostUsersGroupsServicesCreated: true }, null, 2)) +} finally { + for (const h of handles) await h.close().catch(() => {}) + await Promise.all(pools.map(p => p.end())) + for (const name of names) await admin.query('DROP DATABASE "' + name + '" WITH (FORCE)') + for (const role of roles) await admin.query('DROP ROLE "' + role + '"') + await admin.end() +} diff --git a/backend/src/db/database-source-observation.sql-helper.ts b/backend/src/db/database-source-observation.sql-helper.ts new file mode 100644 index 00000000..6f81080b --- /dev/null +++ b/backend/src/db/database-source-observation.sql-helper.ts @@ -0,0 +1,51 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { basename, dirname, resolve } from "node:path" + +/** Catalog observation on two disposable databases, not source enrollment. */ +export async function acceptDatabaseSourceObservationSql() { + assert.equal(process.env.MMAPP_OWN_PROBE_STORAGE_JOURNAL, "true") + const url = new URL(process.env.DATABASE_URL ?? "") + assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") + assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) + const { pool } = await import("./index.js") + const { Pool } = await import("pg") + const { captureDatabaseSourceObservation, assertDatabaseSourceObservationCurrent } = await import("./database-source-observation.js") + const name = "owned_probe_source_" + randomUUID().replaceAll("-", "") + await pool.query(`CREATE DATABASE "${name}" TEMPLATE template0`) + const otherUrl = new URL(url); otherUrl.pathname = "/" + name + const other = new Pool({ connectionString: otherUrl.toString(), max: 1, connectionTimeoutMillis: 3000, statement_timeout: 10000 }) + const observe = async (selected: typeof pool) => { + const client = await selected.connect() + const sql = { query: async (text: string, values?: unknown[], options?: { signal?: AbortSignal; timeoutMs: number }) => { + const config = { text, values, query_timeout: options?.timeoutMs ?? 10000 } + return client.query>(config) + } } + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + await client.query("SET LOCAL statement_timeout='10s'") + await client.query("SET LOCAL search_path=public,pg_catalog") + const options = { deadlineAt: Date.now() + 10000 } + const source = await captureDatabaseSourceObservation(sql, options) + await assertDatabaseSourceObservationCurrent(sql, source, options) + assert.equal(source.ready, false); assert.equal(source.generationAuthenticated, false) + assert(Object.isFrozen(source) && Object.isFrozen(source.source) && Object.isFrozen(source.snapshot)) + return source + } finally { try { await client.query("ROLLBACK") } finally { client.release(true) } } + } + try { + // Deliberately shadow names only inside the second disposable database. + // Qualified source SQL must not resolve this rejecting type/operator. + await other.query("CREATE DOMAIN public.text AS pg_catalog.text CHECK(false)") + await other.query("CREATE FUNCTION public.source_wrong_eq(pg_catalog.name,pg_catalog.name) RETURNS pg_catalog.bool LANGUAGE sql IMMUTABLE AS 'SELECT false'") + await other.query("CREATE OPERATOR public.= (LEFTARG=pg_catalog.name,RIGHTARG=pg_catalog.name,FUNCTION=public.source_wrong_eq)") + const original = await observe(pool), another = await observe(other) + assert.equal(original.source.systemIdentifier, another.source.systemIdentifier) + assert.notEqual(original.source.databaseOid, another.source.databaseOid) + assert.notEqual(original.source.databaseName, another.source.databaseName) + assert.notEqual(original.snapshot.backendPid, another.snapshot.backendPid) + return { actualSql: true, sameClusterTwoDatabases: true, distinctDatabaseOidAndName: true, + actualRepeatableReadSourcePin: true, reorderedSearchPathAndShadowTypeOperatorSafe: true, noGenerationAuthorityClaimed: true, + applicationDataWrites: 0, productionDatabaseTouched: false } + } finally { await other.end(); await pool.query(`DROP DATABASE "${name}"`) } +} diff --git a/backend/src/db/database-source-observation.test.ts b/backend/src/db/database-source-observation.test.ts new file mode 100644 index 00000000..2a8f1f92 --- /dev/null +++ b/backend/src/db/database-source-observation.test.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { McpError } from "../modules/mcp/security.js" +import type { ProbeStorageSql } from "../services/server-probe-journal-maintenance.js" +import { captureDatabaseSourceObservation, assertDatabaseSourceObservationCurrent } from "./database-source-observation.js" + +const options = () => ({ deadlineAt: Date.now() + 2000 }) +const row = (): Record => ({ system_identifier: "18446744073709551615", database_oid: "16384", database_name: "fixture", + session_user: "operator", current_user: "operator", in_recovery: false, server_version_num: "180006", backend_pid: 1234, + transaction_timestamp: "2026-10-25 12:34:56.123456+00", snapshot: "100:110:101,105,109", + isolation: "repeatable read", read_only: "on", explicit_transaction: true }) +function fixture(rows: Record[][] = [[row()]]) { + const calls: { text: string; timeout: number; signal?: AbortSignal }[] = [] + const sql: ProbeStorageSql = { async query(text, values, queryOptions) { + assert.deepEqual(values, []); assert(queryOptions) + calls.push({ text, timeout: queryOptions.timeoutMs, signal: queryOptions.signal }) + return { rows: rows[Math.min(calls.length - 1, rows.length - 1)] } + } } + return { sql, calls } +} +const safe = (error: unknown) => error instanceof McpError && error.message === "Текущий источник и снимок БД не подтверждены" + +test("fixed genuine facts are immutable, exact and explicitly lack generation authority", async () => { + const f = fixture(), observation = await captureDatabaseSourceObservation(f.sql, options()) + assert.equal(observation.source.systemIdentifier, "18446744073709551615") + assert.equal(observation.source.databaseOid, 16384) + assert.equal(observation.snapshot.transactionTimestamp, "2026-10-25 12:34:56.123456+00") + assert.equal(observation.ready, false); assert.equal(observation.generationAuthenticated, false); assert.equal(observation.requiresExternalSourceBinding, true) + assert(Object.isFrozen(observation)); assert(Object.isFrozen(observation.source)); assert(Object.isFrozen(observation.snapshot)) + assert.equal(f.calls.length, 2) + assert(f.calls.every(call => call.timeout >= 1 && call.timeout <= 10000)) + assert(f.calls[0].text.includes("pg_catalog.pg_control_system()")); assert(f.calls[0].text.includes("pg_catalog.pg_current_snapshot()")) + assert(!/\b(BEGIN|COMMIT|ROLLBACK|INSERT|UPDATE|DELETE|CREATE|ALTER|DROP|pg_export_snapshot)\b/i.test(f.calls[0].text)) + await assertDatabaseSourceObservationCurrent(f.sql, observation, options()); assert.equal(f.calls.length, 3) +}) + +test("every source or transaction dimension drifting between reads is refused", async t => { + const changes: Record = { system_identifier: "9223372036854775808", database_oid: "16385", database_name: "other", + session_user: "another", current_user: "another", in_recovery: true, server_version_num: "180007", backend_pid: 1235, + transaction_timestamp: "2026-10-25 12:34:56.123457+00", snapshot: "100:111:101,105,109" } + for (const [key, value] of Object.entries(changes)) await t.test(key, async () => { + const f = fixture([[row()], [{ ...row(), [key]: value }]]) + await assert.rejects(captureDatabaseSourceObservation(f.sql, options()), safe) + }) +}) + +test("same-lease verifier rejects changed transaction after fixed data collection", async () => { + const f = fixture([[row()], [row()], [{ ...row(), backend_pid: 44 }]]) + const observation = await captureDatabaseSourceObservation(f.sql, options()) + await assert.rejects(assertDatabaseSourceObservationCurrent(f.sql, observation, options()), safe) +}) + +test("JSON/copies and cross-client source records never establish source binding", async () => { + const first = fixture(), second = fixture(), observation = await captureDatabaseSourceObservation(first.sql, options()) + await assert.rejects(assertDatabaseSourceObservationCurrent(first.sql, structuredClone(observation), options()), safe) + await assert.rejects(assertDatabaseSourceObservationCurrent(first.sql, { ...observation }, options()), safe) + await assert.rejects(assertDatabaseSourceObservationCurrent(second.sql, observation, options()), safe) + assert.equal(first.calls.length, 2); assert.equal(second.calls.length, 0) +}) + +test("expired or aborted requests perform no SQL", async () => { + const f = fixture(), controller = new AbortController(); controller.abort() + for (const value of [{ deadlineAt: Date.now() - 1 }, { deadlineAt: NaN }, { ...options(), signal: controller.signal }]) { + await assert.rejects(captureDatabaseSourceObservation(f.sql, value), safe) + } + assert.equal(f.calls.length, 0) +}) + +test("original deadline and signal remain authoritative during later pairing", async () => { + const f = fixture(), controller = new AbortController(), original = { deadlineAt: Date.now() + 40, signal: controller.signal } + const observation = await captureDatabaseSourceObservation(f.sql, original) + controller.abort() + await assert.rejects(assertDatabaseSourceObservationCurrent(f.sql, observation, options()), safe) + const other = fixture(), deadline = { deadlineAt: Date.now() + 40 } + const timed = await captureDatabaseSourceObservation(other.sql, deadline) + deadline.deadlineAt += 100000 + await new Promise(resolve => setTimeout(resolve, 50)) + await assert.rejects(assertDatabaseSourceObservationCurrent(other.sql, timed, options()), safe) + assert.equal(f.calls.length, 2); assert.equal(other.calls.length, 2) +}) + +test("strict singleton/schema and explicit RR read-only transaction required", async () => { + const missing = row(); delete missing.current_user + const variants = [[], [row(), row()], [missing], [{ ...row(), extra: "private" }], [{ ...row(), read_only: "off" }], + [{ ...row(), isolation: "read committed" }], [{ ...row(), explicit_transaction: false }], [{ ...row(), backend_pid: 1.5 }], + [{ ...row(), in_recovery: "false" }], [{ ...row(), transaction_timestamp: "not a timestamp" }], + [{ ...row(), transaction_timestamp: "2026-99-99 12:34:56.123456+00" }], [{ ...row(), server_version_num: "120000" }]] + for (const rows of variants) await assert.rejects(captureDatabaseSourceObservation(fixture([rows]).sql, options()), safe) +}) + +test("uint64, OID and snapshot range/order/size validation is fail closed", async () => { + const variants = [{ system_identifier: "18446744073709551616" }, { system_identifier: "0" }, { system_identifier: "01" }, + { database_oid: "4294967296" }, { database_oid: "0" }, { database_oid: "01" }, { database_oid: "1.1" }, + { snapshot: "110:100:" }, { snapshot: "100:110:105,105" }, { snapshot: "100:110:109,105" }, + { snapshot: "100:110:99" }, { snapshot: "100:110:110" }, { snapshot: "100:110:0105" }, + { snapshot: "100:18446744073709551616:" }, { snapshot: "100:110" }, { snapshot: "100:110::" }, + { snapshot: "0:5000:" + Array.from({ length: 4097 }, (_, index) => index).join(",") }, + { snapshot: "0:1:" + "1".repeat(65536) }] + for (const change of variants) await assert.rejects(captureDatabaseSourceObservation(fixture([[{ ...row(), ...change }]]).sql, options()), safe) + const empty = fixture([[{ ...row(), snapshot: "100:100:" }]]) + assert.equal((await captureDatabaseSourceObservation(empty.sql, options())).snapshot.snapshot, "100:100:") +}) + +test("provider errors never leak", async () => { + const privateSql: ProbeStorageSql = { query: async () => { throw new Error("private-provider password db-url") } } + await assert.rejects(captureDatabaseSourceObservation(privateSql, options()), safe) +}) + +test("abort fences after caller-owned SQL settles and prevents a second read", async () => { + let resolve!: (value: { rows: Record[] }) => void, queries = 0 + const controller = new AbortController() + const sql: ProbeStorageSql = { query: () => { queries++; return new Promise(done => { resolve = done }) } } + const pending = captureDatabaseSourceObservation(sql, { ...options(), signal: controller.signal }) + controller.abort() + // Foreign SQL remains caller-owned until it settles; this helper must not + // race it and allow a caller to reuse its backend while work is in flight. + resolve({ rows: [row()] }); await assert.rejects(pending, safe); assert.equal(queries, 1) +}) diff --git a/backend/src/db/database-source-observation.ts b/backend/src/db/database-source-observation.ts new file mode 100644 index 00000000..539ee72d --- /dev/null +++ b/backend/src/db/database-source-observation.ts @@ -0,0 +1,120 @@ +/** Internal source facts from an authenticated caller-owned RR READ ONLY lease. + * Trusted SQL must honor timeout/signal; its caller owns whole-callback bounds, + * cancellation, destruction and cleanup. This helper never abandons foreign SQL. + * Cluster system identifier + database OID do not distinguish physical clones, + * restores, or database generations. These facts never grant READY/conversion. */ +import { z } from "zod" +import { McpError } from "../modules/mcp/security.js" +import type { ProbeStorageSql } from "../services/server-probe-journal-maintenance.js" + +export interface DatabaseSourceObservationOptions { deadlineAt: number; signal?: AbortSignal } +export interface DatabaseSourceObservation { + readonly version: 1 + readonly source: Readonly<{ systemIdentifier: string; databaseOid: number; databaseName: string; sessionUser: string; + currentUser: string; inRecovery: boolean; serverVersionNum: number }> + readonly snapshot: Readonly<{ backendPid: number; transactionTimestamp: string; snapshot: string }> + readonly ready: false + readonly generationAuthenticated: false + readonly requiresExternalSourceBinding: true +} +const refused = () => new McpError("DATABASE_SOURCE_UNAVAILABLE", "Текущий источник и снимок БД не подтверждены", 503) +const MAX_UINT64 = 18446744073709551615n +function uint64(value: string, positive = false) { + if (!/^(0|[1-9][0-9]{0,19})$/.test(value)) throw refused() + const number = BigInt(value) + if (number > MAX_UINT64 || positive && number === 0n) throw refused() + return number +} +function snapshotText(value: string) { + if (Buffer.byteLength(value, "utf8") > 65536) throw refused() + const parts = value.split(":") + if (parts.length !== 3) throw refused() + const xmin = uint64(parts[0]), xmax = uint64(parts[1]) + if (xmin > xmax) throw refused() + const active = parts[2] ? parts[2].split(",") : [] + if (active.length > 4096) throw refused() + let previous: bigint | undefined + for (const item of active) { + const xid = uint64(item) + if (xid < xmin || xid >= xmax || previous !== undefined && xid <= previous) throw refused() + previous = xid + } + return value +} +const text = z.string().min(1).max(200) +const rowSchema = z.object({ + system_identifier: z.string().max(20), database_oid: z.string().max(10), database_name: text, + session_user: text, current_user: text, in_recovery: z.boolean(), server_version_num: z.string().regex(/^[1-9][0-9]{0,7}$/), + backend_pid: z.number().int().positive().max(2147483647), + transaction_timestamp: z.string().max(128).regex(/^[0-9]{4,6}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,6})?[+-][0-9]{2}(?::[0-9]{2}(?::[0-9]{2})?)?$/), + snapshot: z.string().max(65536), isolation: z.literal("repeatable read"), read_only: z.literal("on"), explicit_transaction: z.literal(true), +}).strict() +// All executable functions and catalogs are qualified. SESSION_USER/CURRENT_USER +// are SQL keywords. No search_path-dependent user function is invoked. +const SOURCE_SQL = `/* database-source-observation:current */ SELECT + control.system_identifier::pg_catalog.text AS system_identifier,d.oid::pg_catalog.text AS database_oid,d.datname::pg_catalog.text AS database_name, + SESSION_USER::pg_catalog.text AS session_user,CURRENT_USER::pg_catalog.text AS current_user, + pg_catalog.pg_is_in_recovery() AS in_recovery,pg_catalog.current_setting('server_version_num') AS server_version_num, + pg_catalog.pg_backend_pid() AS backend_pid,pg_catalog.transaction_timestamp()::pg_catalog.text AS transaction_timestamp, + pg_catalog.pg_current_snapshot()::pg_catalog.text AS snapshot, + pg_catalog.current_setting('transaction_isolation') AS isolation,pg_catalog.current_setting('transaction_read_only') AS read_only, + pg_catalog.transaction_timestamp() OPERATOR(pg_catalog.<) pg_catalog.statement_timestamp() AS explicit_transaction + FROM pg_catalog.pg_database d CROSS JOIN pg_catalog.pg_control_system() control + WHERE d.datname OPERATOR(pg_catalog.=) pg_catalog.current_database() LIMIT 2` + +const origins = new WeakMap() +function fence(options: DatabaseSourceObservationOptions) { + if (!Number.isSafeInteger(options.deadlineAt) || Date.now() >= options.deadlineAt || options.signal?.aborted) throw refused() +} +async function read(sql: ProbeStorageSql, options: DatabaseSourceObservationOptions) { + fence(options) + const timeoutMs = Math.max(1, Math.min(10000, options.deadlineAt - Date.now())) + try { + const result = await sql.query(SOURCE_SQL, [], { timeoutMs, signal: options.signal }) + fence(options) + if (!result || !Array.isArray(result.rows) || result.rows.length !== 1) throw refused() + const row = rowSchema.parse(result.rows[0]) + uint64(row.system_identifier, true) + if (!/^[1-9][0-9]*$/.test(row.database_oid)) throw refused() + const databaseOid = Number(row.database_oid), serverVersionNum = Number(row.server_version_num) + if (!Number.isSafeInteger(databaseOid) || databaseOid > 4294967295 || !Number.isSafeInteger(serverVersionNum) + || serverVersionNum < 130000 || !Number.isFinite(Date.parse(row.transaction_timestamp))) throw refused() + snapshotText(row.snapshot) + return { source: { systemIdentifier: row.system_identifier, databaseOid, databaseName: row.database_name, + sessionUser: row.session_user, currentUser: row.current_user, inRecovery: row.in_recovery, serverVersionNum }, + snapshot: { backendPid: row.backend_pid, transactionTimestamp: row.transaction_timestamp, snapshot: row.snapshot } } + } catch { throw refused() } +} +function equal(left: Awaited>, right: Awaited>) { + if (JSON.stringify(left) !== JSON.stringify(right)) throw refused() +} + +/** Repeated genuine source metadata plus exact backend/transaction/snapshot pin. + * Caller owns authorization, checkout, transaction, cleanup and writer/DDL drain. */ +export async function captureDatabaseSourceObservation(sql: ProbeStorageSql, rawOptions: DatabaseSourceObservationOptions): Promise { + const options = { ...rawOptions } + try { + const first = await read(sql, options), second = await read(sql, options) + equal(first, second); fence(options) + const observation: DatabaseSourceObservation = Object.freeze({ version: 1, + source: Object.freeze(first.source), snapshot: Object.freeze(first.snapshot), + ready: false, generationAuthenticated: false, requiresExternalSourceBinding: true }) + origins.set(observation, { sql, ...options }) + return observation + } catch { throw refused() } +} + +/** In-process same-lease verifier for surrounding a fixed data/archive read. + * JSON, copies and observations from another facade cannot establish a binding. */ +export async function assertDatabaseSourceObservationCurrent(sql: ProbeStorageSql, expected: DatabaseSourceObservation, options: DatabaseSourceObservationOptions): Promise { + try { + const origin = origins.get(expected) + if (!origin || origin.sql !== sql) throw refused() + const signal = origin.signal && options.signal ? AbortSignal.any([origin.signal, options.signal]) : origin.signal ?? options.signal + const bounded = { deadlineAt: Math.min(origin.deadlineAt, options.deadlineAt), signal } + fence(bounded) + const current = await read(sql, bounded) + equal({ source: expected.source, snapshot: expected.snapshot }, current) + fence(bounded) + } catch { throw refused() } +} diff --git a/backend/src/db/index.ts b/backend/src/db/index.ts index f383ca19..7f01649d 100644 --- a/backend/src/db/index.ts +++ b/backend/src/db/index.ts @@ -4,6 +4,7 @@ import { drizzle } from "drizzle-orm/node-postgres" import { env } from "../config.js" import * as schema from "./schema.js" import { bindSql } from "./sql-bind.js" +import { DatabaseRuntimeSourceRefused, readRuntimeDatabasePoolOptions, type DatabaseArchivePeer } from "./database-runtime-source.js" const INT8_OID = 20 const DATE_OID = 1082 @@ -14,18 +15,83 @@ pg.types.setTypeParser(INT8_OID, (val) => { }) pg.types.setTypeParser(DATE_OID, (val) => val) -export function createPool(max = 16): pg.Pool { - return new pg.Pool({ - connectionString: env.DATABASE_URL, +class RuntimeCheckedPool extends pg.Pool { + constructor( + options: pg.PoolConfig, + private readonly checkCheckout: (client: pg.PoolClient) => Promise, + readonly checkActivation?: () => void, + readonly archivePeer?: (client: pg.ClientBase) => DatabaseArchivePeer, + readonly readWriterHead?: (client: pg.ClientBase) => Promise<{ epoch: string; sha256: string; issuedAt: number }>, + ) { super(options) } + + connect(): Promise + connect(callback: (error: Error | undefined, client: pg.PoolClient | undefined, done: (release?: Error | boolean) => void) => void): void + connect(callback?: (error: Error | undefined, client: pg.PoolClient | undefined, done: (release?: Error | boolean) => void) => void): Promise | void { + const check = async (client: pg.PoolClient) => { + let connectionFailed = false + const onError = () => { connectionFailed = true } + // pg-pool has removed its idle listener and the caller does not own the client yet. + client.on("error", onError) + try { + await this.checkCheckout(client) + if (connectionFailed) throw new DatabaseRuntimeSourceRefused() + return client + } catch { + client.release(true) + throw new DatabaseRuntimeSourceRefused() + } finally { + client.removeListener("error", onError) + } + } + if (!callback) return super.connect().then(check) + return super.connect((error, client, done) => { + if (error || !client) return callback(error ?? new DatabaseRuntimeSourceRefused(), undefined, done) + void check(client).then( + () => callback(undefined, client, done), + (error: Error) => callback(error, undefined, () => {}), + ) + }) + } +} + +export function createPool(max = 16, allowExitOnIdle = process.env.NODE_ENV === "test"): pg.Pool { + const { archivePeer, checkActivation, checkCheckout, readWriterHead, ...connection } = readRuntimeDatabasePoolOptions({ + databaseUrl: env.DATABASE_URL, + requestFile: process.env.DATABASE_RUNTIME_REQUEST_FILE, + requestSha256: process.env.DATABASE_RUNTIME_REQUEST_SHA256, + caFile: process.env.DATABASE_RUNTIME_CA_FILE, + activationDirectory: process.env.DATABASE_RUNTIME_ACTIVATION_DIRECTORY, + activationSha256: process.env.DATABASE_RUNTIME_ACTIVATION_SHA256, + writerApprovalFile: process.env.DATABASE_RUNTIME_WRITER_APPROVAL_FILE, + writerApprovalSha256: process.env.DATABASE_RUNTIME_WRITER_APPROVAL_SHA256, + }) + if (connection.onConnect && !process.env.DATABASE_URL) throw new DatabaseRuntimeSourceRefused() + const options = { + ...connection, max, idleTimeoutMillis: 30_000, - allowExitOnIdle: process.env.NODE_ENV === "test", + allowExitOnIdle, connectionTimeoutMillis: 8_000, statement_timeout: 120_000, - }) + } + const pool = checkCheckout ? new RuntimeCheckedPool(options, checkCheckout, checkActivation, archivePeer, readWriterHead) : new pg.Pool(options) + if (connection.onConnect) pool.on("error", () => console.error("Потеряно проверяемое соединение приложения с PostgreSQL")) + return pool } export const pool = createPool(isMainThread ? 16 : 4) +export function assertRuntimeDatabaseActivationCurrent(): void { + if (pool instanceof RuntimeCheckedPool) pool.checkActivation?.() +} +export async function readRuntimeDatabaseWriterHead(client: unknown) { + if (!(pool instanceof RuntimeCheckedPool) || !pool.readWriterHead) return undefined + if (!(client instanceof pg.Client)) throw new DatabaseRuntimeSourceRefused() + return pool.readWriterHead(client) +} +export function readRuntimeDatabaseArchivePeer(client: unknown): DatabaseArchivePeer { + if (!(pool instanceof RuntimeCheckedPool) || !pool.archivePeer || !(client instanceof pg.Client)) throw new DatabaseRuntimeSourceRefused() + return pool.archivePeer(client) +} export const db = drizzle(pool, { schema }) export async function dbQuery( diff --git a/backend/src/db/legacy-poll-storage-fixture.ts b/backend/src/db/legacy-poll-storage-fixture.ts new file mode 100644 index 00000000..65fb8db1 --- /dev/null +++ b/backend/src/db/legacy-poll-storage-fixture.ts @@ -0,0 +1,26 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { basename,dirname,resolve } from "node:path" +import type { Pool } from "pg" +import { analyzeMigration,MIGRATION_MANIFEST } from "./migration-manifest.js" + +/** Fixed owned historical prefix, never called by application startup. No + * caller-selected ceiling or production schema flag exists. */ +export async function applyLegacyPollStorageFixtureMigrations(pool:Pool){ + assert(["MMAPP_OWN_SERVER_POLL_STORAGE","MMAPP_OWN_SERVER_POLL_STORAGE_FRESH"].some(key=>process.env[key]==="true")) + const source=new URL(process.env.DATABASE_URL??"");assert.equal(source.hostname,"127.0.0.1");assert.equal(source.pathname,"/postgres") + assert(process.env.MCP_STATE_DIR&&basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) + const pins=MIGRATION_MANIFEST.filter(pin=>pin.file<"0039_") + assert.equal(pins.length,38);assert.equal(pins.at(-1)?.file,"0038_server_poll_journal_and_writer_guards.sql") + const files=await Promise.all(pins.map(async pin=>{const bytes=await readFile(new URL("../../drizzle/"+pin.file,import.meta.url));analyzeMigration(pin.file,bytes,pin);return {id:pin.file.slice(0,-4),sql:bytes.toString("utf8")}})) + const {PARTITION_WRITER_ADVISORY_KEY}=await import("../services/partition-write-transaction.js") + const client=await pool.connect() + try{ + await client.query("SET statement_timeout='120s'") + await client.query("SELECT pg_advisory_lock_shared($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY]) + await client.query("CREATE TABLE IF NOT EXISTS schema_migrations(id text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now())") + const applied=new Set((await client.query("SELECT id FROM schema_migrations")).rows.map(row=>row.id)) + assert([...applied].every(id=>files.some(file=>file.id===id)),"Historical poll fixture cannot open a newer schema") + for(const file of files)if(!applied.has(file.id)){await client.query(file.sql);await client.query("INSERT INTO schema_migrations(id) VALUES($1) ON CONFLICT DO NOTHING",[file.id])} + }finally{client.release(true)} +} diff --git a/backend/src/db/legacy-probe-0037-body.ts b/backend/src/db/legacy-probe-0037-body.ts new file mode 100644 index 00000000..a540e9eb --- /dev/null +++ b/backend/src/db/legacy-probe-0037-body.ts @@ -0,0 +1,17 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { analyzeMigration, MIGRATION_MANIFEST } from "./migration-manifest.js" + +/** Only the explicitly reviewed 0037 handoff substitutes owner transaction boundaries. */ +export async function readReviewedLegacyProbe0037() { + const pin = MIGRATION_MANIFEST[36] + assert.equal(pin.file, "0037_server_probe_storage_v2.sql") + const bytes = await readFile(new URL("../../drizzle/0037_server_probe_storage_v2.sql", import.meta.url)) + const review = analyzeMigration(pin.file, bytes, pin) + const [begin, commit] = review.topLevelTransactionControls + assert(begin.kind === "BEGIN" && commit.kind === "COMMIT" && review.selfMarker?.atomicWithAllFileEffects) + const lines = bytes.toString("utf8").split(/(?<=\n)/) + assert.equal(lines[begin.line - 1].trim(), "BEGIN;") + assert.equal(lines[commit.line - 1].trim(), "COMMIT;") + return { body: lines.slice(begin.line, commit.line - 1).join(""), pin } +} diff --git a/backend/src/db/legacy-probe-fixture.ts b/backend/src/db/legacy-probe-fixture.ts new file mode 100644 index 00000000..670472e8 --- /dev/null +++ b/backend/src/db/legacy-probe-fixture.ts @@ -0,0 +1,34 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { basename, dirname, resolve } from "node:path" +import type { Pool } from "pg" +import { analyzeMigration, MIGRATION_MANIFEST } from "./migration-manifest.js" + +/** Fixed historical test schema only. Never imported by application startup; + * no caller-selected migration ceiling or production environment switch. */ +export async function applyLegacyProbeFixtureMigrations(pool: Pool) { + assert(["MMAPP_OWN_PROBE_JOURNAL", "MMAPP_OWN_PROBE_STORAGE_JOURNAL", "MMAPP_OWN_PROBE_STORAGE_HISTORY", "MMAPP_OWN_PROBE_LEGACY_ORPHANS"].some(key => process.env[key] === "true")) + const url = new URL(process.env.DATABASE_URL ?? "") + assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") + assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) + const pins = MIGRATION_MANIFEST.filter(pin => pin.file < "0037_") + assert.equal(pins.length, 36); assert.equal(pins.at(-1)?.file, "0036_diagnostic_cancellations.sql") + const files = await Promise.all(pins.map(async pin => { + const bytes = await readFile(new URL("../../drizzle/" + pin.file, import.meta.url)) + analyzeMigration(pin.file, bytes, pin) + return { id: pin.file.slice(0, -4), sql: bytes.toString("utf8") } + })) + const { PARTITION_WRITER_ADVISORY_KEY } = await import("../services/partition-write-transaction.js") + const client = await pool.connect() + try { + await client.query("SET statement_timeout='120s'") + await client.query("SELECT pg_advisory_lock_shared($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + await client.query("CREATE TABLE IF NOT EXISTS schema_migrations(id text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now())") + const ids = new Set((await client.query("SELECT id FROM schema_migrations")).rows.map(row => row.id)) + assert([...ids].every(id => files.some(file => file.id === id)), "Historical fixture cannot open a newer schema") + for (const file of files) if (!ids.has(file.id)) { + await client.query(file.sql) + await client.query("INSERT INTO schema_migrations(id) VALUES($1) ON CONFLICT DO NOTHING", [file.id]) + } + } finally { client.release(true) } +} diff --git a/backend/src/db/legacy-source58-transition.ts b/backend/src/db/legacy-source58-transition.ts new file mode 100644 index 00000000..92942d58 --- /dev/null +++ b/backend/src/db/legacy-source58-transition.ts @@ -0,0 +1,95 @@ +import { createPublicKey, verify } from "node:crypto" +import { z } from "zod" +import { readControlledSchemaReference } from "./controlled-schema-reference-core.js" +import { exactHistory } from "./controlled-migration-history.js" +import { checkDatabaseSchemaControlState, type DatabaseSchemaTerminal } from "./database-schema-control-model.js" +import { RESTORE_WRITER_PROFILE_DEFINITION_SHA256 } from "./database-restore-quarantine-profile.js" +import { sourceBindingBytes, sourceBindingCanonical, sourceBindingSha256 } from "./database-source-binding.js" +import { MIGRATION_MANIFEST, migrationManifestDigest } from "./migration-manifest.js" +import { captureMigrationSchema, compareMigrationSchemas, type SchemaInventoryClient } from "./migration-schema-inventory.js" + +export const LEGACY_SOURCE58_PREFLIGHT_DOMAIN = "mmapp:legacy-source58-preflight:ed25519:v1\n" +const digest = z.string().regex(/^[a-f0-9]{64}$/) +const role = z.object({ name: z.string().regex(/^(?!pg_)[a-z_][a-z0-9_]{0,62}$/), + signedOid: z.string().regex(/^[1-9][0-9]{0,9}$/).refine(value => Number(value) <= 4294967295) }).strict() +const roles = z.object({ publisher: role, control: role, operator: role, runtime: role, capture: role }).strict() +const approval = z.object({ body: z.object({ format: z.literal("legacy-source58-preflight-approval-v1"), manifestSha256: digest, + sourceHeadSha256: digest, sourceTlsSpkiSha256: digest, currentSchemaSha256: digest, operationId: z.uuidv4(), + intentSha256: digest, releaseApprovalSha256: digest, targetDefinitionSha256: z.literal(RESTORE_WRITER_PROFILE_DEFINITION_SHA256), + targetCatalogueSha256: digest, roles, expiresAt: z.number().int().positive() }).strict(), + signature: z.string().regex(/^[A-Za-z0-9_-]{86}$/) }).strict() +const verified = new WeakSet() +const refuse = (reason: string): never => { throw new Error("Legacy Source58 predecessor refused: " + reason) } + +export function verifyLegacySource58PreflightApproval(raw: Buffer, pem: Buffer, pins: { + coldSpkiSha256: string; sourceSpkiSha256: string; sourceHeadSha256: string; sourceTlsSpkiSha256: string; roles: z.infer +}) { + const parsed: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(raw)) + const { body, signature } = approval.parse(parsed), key = createPublicKey(pem), sig = Buffer.from(signature, "base64url") + const bindings = Object.values(body.roles) + if (!sourceBindingBytes(parsed).equals(raw) || sig.toString("base64url") !== signature || pem.includes("PRIVATE KEY") || + key.asymmetricKeyType !== "ed25519" || pins.coldSpkiSha256 === pins.sourceSpkiSha256 || + sourceBindingSha256(key.export({ type: "spki", format: "der" })) !== pins.coldSpkiSha256 || + body.sourceHeadSha256 !== pins.sourceHeadSha256 || body.sourceTlsSpkiSha256 !== pins.sourceTlsSpkiSha256 || + body.manifestSha256 !== migrationManifestDigest() || body.expiresAt <= Date.now() || body.expiresAt - Date.now() > 900000 || + new Set(bindings.map(binding => binding.name)).size !== bindings.length || new Set(bindings.map(binding => binding.signedOid)).size !== bindings.length || + sourceBindingCanonical(body.roles) !== sourceBindingCanonical(pins.roles) || + !verify(null, Buffer.from(LEGACY_SOURCE58_PREFLIGHT_DOMAIN + sourceBindingCanonical(body)), key, sig)) return refuse("independent target approval") + for (const binding of bindings) Object.freeze(binding) + const result = Object.freeze({ ...body, roles: Object.freeze(body.roles), approvalSha256: sourceBindingSha256(raw) }) + verified.add(result) + return result +} + +/** Verify the exact pre-transition state through one retained snapshot. The + * independently approved target is a pin for future enrollment and is never + * inferred from the source or interpreted as an applied catalogue. */ +export async function inspectApprovedLegacySource58Predecessor(sql: SchemaInventoryClient, + pin: ReturnType, deadlineAt: number) { + if (!verified.has(pin) || Date.now() >= pin.expiresAt) return refuse("verified approval required") + const reference = await readControlledSchemaReference() + if (!reference || reference.sha256 !== pin.currentSchemaSha256) return refuse("independent predecessor reference") + const actual = await captureMigrationSchema(sql, { scope: reference.scope, deadlineAt }) + compareMigrationSchemas(reference, actual) + const { rows: [gate] } = await sql.query(`SELECT q.operation_id AS quarantine_operation,s.state + FROM mmapp_control.database_restore_quarantine q CROSS JOIN mmapp_control.database_schema_state s WHERE q.id=1 AND s.id=1`) + const state = checkDatabaseSchemaControlState(gate?.state) + const { rows: history } = await sql.query(`SELECT id::text AS id,intent,terminal FROM mmapp_control.database_schema_intents + ORDER BY id LIMIT ${MIGRATION_MANIFEST.length + 1}`) + const completedFiles = await exactHistory(history, state, pin.manifestSha256, true) + const terminal = history.find(row => row.id === pin.operationId)?.terminal as DatabaseSchemaTerminal | undefined + if (gate.quarantine_operation !== null || completedFiles !== MIGRATION_MANIFEST.length || + terminal?.intent.artifactSha256 !== pin.intentSha256 || terminal.legacyRelease?.approvalSha256 !== pin.releaseApprovalSha256) { + return refuse("exact authenticated release and open quarantine") + } + const { rows: markers } = await sql.query('SELECT id FROM public.schema_migrations ORDER BY id COLLATE "C" LIMIT $1', [MIGRATION_MANIFEST.length + 1]) + if (markers.length !== MIGRATION_MANIFEST.length || + !markers.every((row, index) => row.id === MIGRATION_MANIFEST[index].file.slice(0, -4))) return refuse("exact current markers") + const { rows: liveRoles } = await sql.query(`SELECT rolname AS name,oid::text AS oid,rolcanlogin AS login, + (rolsuper OR rolcreatedb OR rolcreaterole OR rolreplication OR rolbypassrls OR EXISTS(SELECT FROM pg_catalog.pg_database d WHERE d.datdba=r.oid) + OR EXISTS(SELECT FROM pg_catalog.pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid)) AS unsafe + FROM pg_catalog.pg_roles r WHERE rolname=ANY($1::text[])`, [Object.values(pin.roles).map(binding => binding.name)]) + if (liveRoles.length !== 5 || !Object.values(pin.roles).every(binding => liveRoles.some(row => row.name === binding.name && row.oid === binding.signedOid))) { + return refuse("exact live role OIDs") + } + const { rows: [layout] } = await sql.query(`SELECT q.relowner::text AS quarantine_owner,n.nspowner::text AS namespace_owner, + pg_catalog.to_regclass('mmapp_control.application_writer_profile_history') IS NOT NULL AS history, + pg_catalog.to_regclass('mmapp_control.application_writer_profile_current') IS NOT NULL AS current, + (SELECT count(*)::int FROM pg_catalog.pg_trigger WHERE tgrelid='public.schema_migrations'::pg_catalog.regclass AND NOT tgisinternal) AS marker_triggers + FROM pg_catalog.pg_class q JOIN pg_catalog.pg_namespace n ON n.oid=q.relnamespace WHERE q.oid='mmapp_control.database_restore_quarantine'::pg_catalog.regclass`) + if (!layout || (layout.history !== layout.current)) return refuse("partial writer registry") + const blockers = [ + ...(layout.namespace_owner !== pin.roles.control.signedOid ? ["CONTROL_NAMESPACE_OWNER"] : []), + ...(layout.quarantine_owner !== pin.roles.control.signedOid ? ["QUARANTINE_OWNER"] : []), + ...(!layout.history ? ["WRITER_REGISTRY_MISSING"] : []), + ...(layout.marker_triggers !== 3 ? ["REVIEWED_MARKER_FENCE_MISSING"] : []), + ...(["operator", "publisher", "runtime", "capture"] as const).filter(scope => { + const actor = liveRoles.find(row => row.oid === pin.roles[scope].signedOid) + return actor?.unsafe !== false || actor.login !== true + }).map(scope => "ROLE_PROFILE_" + scope.toUpperCase()), + ] + if (Date.now() >= deadlineAt || Date.now() >= pin.expiresAt) return refuse("attestation lifetime") + return Object.freeze({ approvalSha256: pin.approvalSha256, sourceHeadSha256: pin.sourceHeadSha256, releaseApprovalSha256: pin.releaseApprovalSha256, + currentSchemaSha256: actual.sha256, approvedTargetCatalogueSha256: pin.targetCatalogueSha256, authenticatedFiles: completedFiles, + blockers, transitionAuthorized: false as const, readyAuthorized: false as const }) +} diff --git a/backend/src/db/migrate-partition-admission.test.ts b/backend/src/db/migrate-partition-admission.test.ts new file mode 100644 index 00000000..d20a692f --- /dev/null +++ b/backend/src/db/migrate-partition-admission.test.ts @@ -0,0 +1,59 @@ +import assert from "node:assert/strict" +import { test } from "node:test" +import { readdirSync } from "node:fs" +import type { Pool } from "pg" +import { applySqlMigrations } from "./migrate.js" +import { PartitionWriteError } from "../services/partition-write-transaction.js" + +const files = readdirSync(new URL("../../drizzle/", import.meta.url)).filter(file => /^\d{4}_.+\.sql$/i.test(file)) +function fixture(pending?: string) { + const trace: Array<{ text: string; values: unknown[] }> = [], releases: boolean[] = [] + let fail: string | undefined, unlock = true, connections = 0 + const client = { + async query(text: string, values: unknown[] = []) { + trace.push({ text, values }) + if (fail && text.includes(fail)) throw Error("Synthetic SQL ACK loss") + if (text === "SELECT id FROM schema_migrations") return { rows: files.filter(file => file !== pending).map(file => ({ id: file.replace(/\.sql$/, "") })) } + if (text.includes("pg_advisory_unlock_shared")) return { rows: [{ unlocked: unlock }] } + return { rows: [] } + }, + release(destroy: boolean) { releases.push(destroy) }, + } + const pool = { connect: async () => { connections++; return client }, query: () => { assert.fail("Migration SQL escaped onto global pool") } } as unknown as Pool + return { pool, trace, releases, connects: () => connections, fail: (text: string) => { fail = text }, badUnlock: () => { unlock = false } } +} +const f = fixture("0007_flow_buckets_nat.sql") +await applySqlMigrations(f.pool) +assert.equal(f.connects(), 1); assert.deepEqual(f.releases, [false]) +assert.equal(f.trace[0].text, "SET lock_timeout='10s'"); assert.equal(f.trace[1].text, "SET statement_timeout='120s'") +assert.equal(f.trace[2].text, "SELECT pg_catalog.pg_advisory_lock_shared($1::pg_catalog.int8)"); assert.deepEqual(f.trace[2].values, [8723110]) +assert(f.trace[3].text.includes("CREATE TABLE IF NOT EXISTS schema_migrations")) +const migration = f.trace.findIndex(row => row.text.includes("ADD COLUMN IF NOT EXISTS nat_src")) +const marker = f.trace.findIndex(row => row.values[0] === "0007_flow_buckets_nat") +assert(migration > 2 && marker > migration) +assert(f.trace.at(-3)?.text.includes("pg_advisory_unlock_shared")); assert.equal(f.trace.at(-2)?.text, "RESET lock_timeout"); assert.equal(f.trace.at(-1)?.text, "RESET statement_timeout") + +for (const failure of ["pg_advisory_lock_shared", "ADD COLUMN IF NOT EXISTS nat_src", "INSERT INTO schema_migrations"]) { + const f = fixture("0007_flow_buckets_nat.sql"); f.fail(failure) + await assert.rejects(applySqlMigrations(f.pool), error => error instanceof PartitionWriteError && error.outcomeUnknown === (failure !== "pg_advisory_lock_shared")) + assert.deepEqual(f.releases, [true]); assert.equal(f.trace.filter(row => row.text.includes(failure)).length, 1) + assert(!f.trace.some(row => row.text === "ROLLBACK" || row.text.includes("pg_advisory_unlock_shared"))) +} +const badUnlock = fixture(); badUnlock.badUnlock() +await assert.rejects(applySqlMigrations(badUnlock.pool), (error: unknown) => error instanceof PartitionWriteError && error.outcomeUnknown) +assert.deepEqual(badUnlock.releases, [true]) +console.log("Migration partition admission: 5 inert exact-backend/session and no-replay scenarios passed") + +test("post-restore migration rechecks the bound owner session and refuses before DDL on mismatch", async () => { + const proof = "SELECT 'owned target proof'" + const accepted = fixture("0007_flow_buckets_nat.sql") + await applySqlMigrations(accepted.pool, proof) + const statements = accepted.trace.map(row => row.text) + assert.deepEqual(statements.slice(2, 5), ["SET search_path = public, pg_temp", "SELECT pg_catalog.pg_advisory_lock_shared($1::pg_catalog.int8)", proof]) + assert(statements.lastIndexOf(proof) > statements.findIndex(text => text.includes("ADD COLUMN IF NOT EXISTS nat_src"))) + const wrongTarget = fixture() + wrongTarget.fail(proof) + await assert.rejects(applySqlMigrations(wrongTarget.pool, proof), error => error instanceof PartitionWriteError && !error.outcomeUnknown) + assert.deepEqual(wrongTarget.releases, [true]) + assert(!wrongTarget.trace.some(row => row.text.includes("CREATE TABLE"))) +}) diff --git a/backend/src/db/migrate.ts b/backend/src/db/migrate.ts index bf858b8f..280bf93c 100644 --- a/backend/src/db/migrate.ts +++ b/backend/src/db/migrate.ts @@ -2,6 +2,9 @@ import { readdirSync, readFileSync } from "node:fs" import { dirname, join } from "node:path" import { fileURLToPath } from "node:url" import type { Pool } from "pg" +import { withSystemDatabaseOperationLock, type DatabaseOperationLockClient } from "./operation-lock.js" +import { assertDatabaseRestoreQuarantineOpen, databaseRestoreQuarantineEnabled } from "./database-restore-quarantine.js" +import { PARTITION_WRITER_ADVISORY_KEY, PartitionWriteError } from "../services/partition-write-transaction.js" const FIRST_MIGRATION = "0000_postgresql.sql" @@ -27,28 +30,84 @@ function migrationId(file: string): string { return file.replace(/\.sql$/i, "") } -export async function applySqlMigrations(pool: Pool): Promise { - await pool.query(` +function migrationFiles(directory: string): string[] { + return readdirSync(directory) + .filter(file => /^\d{4}_.+\.sql$/i.test(file)) + .sort((left, right) => left.localeCompare(right)) +} + +/** Runtime startup checks that the offline operator prepared this release. */ +export async function assertSqlMigrationsApplied(client: Pick): Promise { + const { validateMigrationFileSet } = await import("./migration-manifest.js") + const files = migrationFiles(migrationsDir()) + validateMigrationFileSet(files) + const expected = new Set(files.map(migrationId)) + const { rows } = await client.query( + "SELECT id FROM public.schema_migrations LIMIT $1", [expected.size + 1], + ) + if (rows.length !== expected.size || rows.some(row => typeof row.id !== "string" || !expected.delete(row.id))) { + throw new Error("Версия схемы PostgreSQL не совпадает с приложением. Требуется подготовка базы оператором") + } +} + +export function applyStandaloneSqlMigrations(pool: Pool): Promise { + return withSystemDatabaseOperationLock(() => pool.connect(), async client => { + if (databaseRestoreQuarantineEnabled()) await assertDatabaseRestoreQuarantineOpen(client) + return applySqlMigrations(pool) + }) +} + +/** Restore calls this while its outer database operation already owns the global lock. */ +export async function applySqlMigrations(pool: Pool, restoreTargetSql?: string): Promise { + // Migration files can own BEGIN/COMMIT themselves. A session shared lease + // therefore spans every file on THIS checked-out backend, including schema + // discovery and marker writes; a lock on another pool backend is insufficient. + const client = await pool.connect() + let released = false, targetSqlSent = false + try { + await client.query("SET lock_timeout='10s'") + await client.query("SET statement_timeout='120s'") + // Explicitly place temporary relations after public in reused restore sessions. + if (restoreTargetSql) await client.query("SET search_path = public, pg_temp") + await client.query("SELECT pg_catalog.pg_advisory_lock_shared($1::pg_catalog.int8)", [PARTITION_WRITER_ADVISORY_KEY]) + if (restoreTargetSql) await client.query(restoreTargetSql) + targetSqlSent = true + await client.query(` CREATE TABLE IF NOT EXISTS schema_migrations ( id TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now() ) `) const dir = migrationsDir() - const files = readdirSync(dir) - .filter((f) => /^\d{4}_.+\.sql$/i.test(f)) - .sort((a, b) => a.localeCompare(b)) + const files = migrationFiles(dir) const applied = new Set( - (await pool.query<{ id: string }>(`SELECT id FROM schema_migrations`)).rows.map((r) => r.id), + (await client.query<{ id: string }>(`SELECT id FROM schema_migrations`)).rows.map((r) => r.id), ) for (const file of files) { const id = migrationId(file) if (applied.has(id)) continue const sql = readFileSync(join(dir, file), "utf8") - await pool.query(sql) - await pool.query( + await client.query(sql) + await client.query( `INSERT INTO schema_migrations (id) VALUES ($1) ON CONFLICT (id) DO NOTHING`, [id], ) } + if (restoreTargetSql) { + await client.query(restoreTargetSql) + await client.query("RESET search_path") + } + const unlocked = await client.query<{ unlocked: boolean }>("SELECT pg_catalog.pg_advisory_unlock_shared($1::pg_catalog.int8) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY]) + if (unlocked.rows[0]?.unlocked !== true) throw new PartitionWriteError(targetSqlSent) + await client.query("RESET lock_timeout") + await client.query("RESET statement_timeout") + released = true; client.release(false) + } catch { + // A whole migration query may include several committed statements. Never + // send ROLLBACK or automatically repeat files after an uncertain boundary. + if (!released) { released = true; client.release(true) } + throw new PartitionWriteError(targetSqlSent) + } finally { + if (!released) client.release(true) + } } diff --git a/backend/src/db/migration-data-inventory.sql.test.ts b/backend/src/db/migration-data-inventory.sql.test.ts new file mode 100644 index 00000000..c5cbf919 --- /dev/null +++ b/backend/src/db/migration-data-inventory.sql.test.ts @@ -0,0 +1,155 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { basename, dirname, join, resolve } from "node:path" +import { writeFile, unlink, mkdir } from "node:fs/promises" +import type { SchemaInventoryClient } from "./migration-schema-inventory.js" + +assert.equal(process.env.MMAPP_OWN_MIGRATION_DATA_INVENTORY, "true") +const databaseUrl = new URL(process.env.DATABASE_URL ?? "") +assert.equal(databaseUrl.hostname, "127.0.0.1"); assert.equal(databaseUrl.pathname, "/postgres") +assert.equal(databaseUrl.username,"mcp_test"); assert(databaseUrl.port&&databaseUrl.port!=="5432"); assert.equal(process.env.NODE_ENV,"test") +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const { pool, closePool } = await import("./index.js") +const { Pool } = await import("pg") +const { applySqlMigrations } = await import("./migrate.js") +const { captureMigrationData, compareMigrationData } = await import("./migration-data-inventory.js") +const execute = promisify(execFile), cases: string[] = [] +let restored: InstanceType | undefined +const restoredName = "mm_data_restore_" + randomUUID().replaceAll("-", "") +assert(/^mm_data_restore_[a-f0-9]{32}$/.test(restoredName)) +const archive = join(resolve(process.env.MCP_STATE_DIR!), "owned-migration-data.dump") +async function capture(connection = pool, options: Parameters[1] = {}, intercept?: (client: SchemaInventoryClient) => SchemaInventoryClient) { + const client = await connection.connect() + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + for (const query of ["SET LOCAL search_path=pg_catalog", "SET LOCAL TimeZone='UTC'", "SET LOCAL DateStyle='ISO, YMD'", "SET LOCAL standard_conforming_strings=on", "SET LOCAL statement_timeout='5s'", "SET LOCAL row_security=off", "SET LOCAL extra_float_digits=3", "SET LOCAL bytea_output=hex", "SET LOCAL IntervalStyle=postgres", "SET LOCAL lc_monetary='C'", "SET LOCAL lc_numeric='C'"]) await client.query(query) + return await captureMigrationData(intercept ? intercept(client) : client, options) + } finally { await client.query("ROLLBACK"); client.release(true) } +} +try { + await applySqlMigrations(pool) + // Plain pg_dump/pg_restore does not copy the source database's own comment. + // Match the independently-created target profile before comparing schemas. + await pool.query("COMMENT ON DATABASE postgres IS NULL") + await pool.query("UPDATE traffic_settings SET enabled=false; UPDATE uptime_settings SET enabled=false,resources_enabled=false,ping_enabled=false,speed_enabled=false") + await pool.query("CREATE TABLE mm_data_probe(id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,value text NOT NULL,payload jsonb,amount numeric,at timestamptz)") + await pool.query("INSERT INTO mm_data_probe(value,payload,amount,at) VALUES('OWNED_RAW_VALUE_MUST_NOT_ESCAPE','{\"a\":2,\"b\":[1,true]}',123.4500,'2026-10-25T12:34:56+07:00')") + await pool.query("ALTER TABLE mm_data_probe ADD COLUMN flag integer NOT NULL DEFAULT 7") + await pool.query("CREATE TABLE mm_data_duplicates(value text); INSERT INTO mm_data_duplicates VALUES('same'),('same'),('different')") + await pool.query("CREATE TABLE mm_data_arrays(value text[]); INSERT INTO mm_data_arrays VALUES('[0:1]={left,right}'::text[])") + await pool.query("CREATE TABLE mm_data_partitioned(k integer,value text) PARTITION BY RANGE(k); CREATE TABLE mm_data_partition_a PARTITION OF mm_data_partitioned FOR VALUES FROM(0) TO(10); CREATE TABLE mm_data_partition_b PARTITION OF mm_data_partitioned FOR VALUES FROM(10) TO(20); INSERT INTO mm_data_partitioned VALUES(1,'a'),(11,'b')") + await pool.query("CREATE MATERIALIZED VIEW mm_data_cached AS SELECT value FROM mm_data_duplicates") + await pool.query("CREATE VIEW mm_data_never_execute AS SELECT 1/0 AS forbidden") + await assert.rejects(pool.query("SELECT * FROM mm_data_never_execute")) + const baseline = await capture() + assert.equal(baseline.tables.find(row => row.name === "mm_data_partitioned")!.rowCount, 0) + assert.equal(baseline.tables.find(row => row.name === "mm_data_partition_a")!.rowCount, 1) + assert.equal(baseline.tables.find(row => row.name === "mm_data_partition_b")!.rowCount, 1) + assert.equal(baseline.tables.find(row => row.name === "mm_data_cached")!.rowCount, 3) + assert(!baseline.tables.some(row => row.name === "mm_data_never_execute")) + assert(!JSON.stringify(baseline).includes("OWNED_RAW_VALUE_MUST_NOT_ESCAPE")) + assert.equal(baseline.ready, false); assert.equal(baseline.allReplicaQuiescence, false) + cases.push("actual PG18 captures private row digests, ONLY physical partition rows and materialized data without executing views or exposing values") + await pool.query("DELETE FROM mm_data_duplicates; INSERT INTO mm_data_duplicates VALUES('different'),('same'),('same')") + assert.equal(compareMigrationData(baseline, await capture()).equal, true) + cases.push("actual row insertion order does not change multiset evidence; duplicate multiplicity is retained") + await pool.query("DELETE FROM mm_data_duplicates WHERE ctid=(SELECT ctid FROM mm_data_duplicates WHERE value='same' LIMIT 1)") + await assert.rejects(async () => compareMigrationData(baseline, await capture())) + await pool.query("INSERT INTO mm_data_duplicates VALUES('same')") + cases.push("actual loss of one duplicate changes content evidence despite unchanged logical schema") + await pool.query("UPDATE mm_data_arrays SET value=ARRAY['left','right']") + assert.equal((await pool.query("SELECT to_jsonb('[0:1]={left,right}'::text[])=to_jsonb(ARRAY['left','right']) AS same")).rows[0].same, true) + await assert.rejects(async () => compareMigrationData(baseline, await capture())) + await pool.query("UPDATE mm_data_arrays SET value='[0:1]={left,right}'::text[]") + cases.push("native array lower bounds remain distinct even when JSONB projections are equal") + await assert.rejects(capture(pool, { maxRows: 1 })) + await assert.rejects(capture(pool, { maxBytes: 1 })) + cases.push("actual row and byte overflow refuse a complete snapshot instead of truncating") + await pool.query("INSERT INTO mm_data_duplicates VALUES(repeat('Z',1048577))") + await assert.rejects(capture(), error => error instanceof Error && !error.message.includes("ZZZZZ")) + await pool.query("DELETE FROM mm_data_duplicates WHERE length(value)>1048576") + cases.push("actual oversized row refuses without reflecting its contents") + const sequence = baseline.sequences.find(row => row.name === "mm_data_probe_id_seq")! + await pool.query("SELECT setval('mm_data_probe_id_seq',$1,true)", [String(BigInt(sequence.lastValue) + 1n)]) + await assert.rejects(async () => compareMigrationData(baseline, await capture())) + await pool.query("SELECT setval('mm_data_probe_id_seq',$1,$2)", [sequence.lastValue, sequence.isCalled]) + cases.push("actual sequence state participates in equivalence independently of table contents") + let wrote = false + const during = await capture(pool, {}, client => ({ async query(text, values) { + const result = await client.query(text, values) + if (!wrote && text.includes("migration-data:declare")) { wrote = true; await pool.query("INSERT INTO mm_data_duplicates VALUES('concurrent-outside-snapshot')") } + return result + } })) + assert(wrote); assert.equal(compareMigrationData(baseline, during).equal, true) + await assert.rejects(async () => compareMigrationData(baseline, await capture())) + await pool.query("DELETE FROM mm_data_duplicates WHERE value='concurrent-outside-snapshot'") + cases.push("actual repeatable-read rows exclude a concurrent committed insert while the next capture observes it; no quiescence grant") + let advanced = false + await assert.rejects(capture(pool, {}, client => ({ async query(text, values) { + const result = await client.query(text, values) + if (!advanced && text.includes("migration-data:sequence") && text.includes('"mm_data_probe_id_seq"')) { advanced = true; await pool.query("SELECT setval('mm_data_probe_id_seq',$1,true)", [String(BigInt(sequence.lastValue) + 10n)]) } + return result + } }))) + assert(advanced) + await pool.query("SELECT setval('mm_data_probe_id_seq',$1,$2)", [sequence.lastValue, sequence.isCalled]) + cases.push("actual nontransactional sequence change between stability reads refuses complete evidence") + await assert.rejects(capture(pool, {}, client => ({ async query(text, values) { + if (text.includes("migration-data:context")) await client.query("SET LOCAL row_security=on") + return client.query(text, values) + } }))) + cases.push("actual unsafe row-security GUC is refused before private row capture") + const beforeRestore = await capture() + const pgEnv = { ...process.env, PGPASSWORD: decodeURIComponent(databaseUrl.password) } + const args = ["--host", databaseUrl.hostname, "--port", databaseUrl.port, "--username", decodeURIComponent(databaseUrl.username), "--no-password"] + const dumped = await execute("pg_dump", [...args, "--format=custom", "--dbname=postgres"], { env: pgEnv, encoding: "buffer", maxBuffer: 16 * 1048576 }) + await mkdir(dirname(archive), { recursive: true }); await writeFile(archive, dumped.stdout) + await pool.query('CREATE DATABASE "' + restoredName + '" TEMPLATE template0') + await execute("pg_restore", [...args, "--single-transaction", "--exit-on-error", "--dbname=" + restoredName, archive], { env: pgEnv, maxBuffer: 1048576 }) + const cloneUrl = new URL(databaseUrl); cloneUrl.pathname = "/" + restoredName + restored = new Pool({ connectionString: cloneUrl.toString(), max: 2 }) + const afterRestore = await capture(restored) + assert.equal(compareMigrationData(beforeRestore, afterRestore).equal, true) + assert.notEqual(JSON.stringify(beforeRestore.schema.observedPhysical), JSON.stringify(afterRestore.schema.observedPhysical)) + cases.push("actual pg_dump/pg_restore preserves full logical data and sequence evidence despite fast-default physical-cache differences") + await restored.query("CREATE SCHEMA mmapp_control; CREATE TABLE mmapp_control.owned_data_probe(value text); INSERT INTO mmapp_control.owned_data_probe VALUES('OWNED_CONTROL_VALUE_NOT_APPLICATION_DATA')") + const scope = { applicationNamespaces: ["public"], excludedControlNamespaces: ["mmapp_control"] } + const controlBefore = await capture(restored, { scope }) + await restored.query("UPDATE mmapp_control.owned_data_probe SET value='CONTROL_CHANGED_OUTSIDE_APPLICATION'") + const controlAfter = await capture(restored, { scope }) + assert.equal(compareMigrationData(controlBefore, controlAfter).equal, true) + assert(!controlAfter.tables.some(row => row.namespace === "mmapp_control")) + assert(!JSON.stringify(controlBefore).includes("OWNED_CONTROL_VALUE_NOT_APPLICATION_DATA")) + cases.push("actual external control table values remain excluded from application data evidence") + await restored.query(` + CREATE SEQUENCE mmapp_control.owned_data_probe_seq; + CREATE FUNCTION mmapp_control.owned_data_guard() RETURNS event_trigger LANGUAGE plpgsql AS $$BEGIN NULL; END$$; + CREATE EVENT TRIGGER owned_data_guard ON ddl_command_start EXECUTE FUNCTION mmapp_control.owned_data_guard(); + `) + const scoped = { scope: { applicationNamespaces: ["public", "mmapp_control"] }, dataNamespaces: ["public"], includeControlEventTriggers: true } + const protectedBefore = await capture(restored, scoped) + assert.equal(protectedBefore.schema.eventTriggers?.[0].name, "owned_data_guard") + assert(protectedBefore.schema.sections.sequences.some(row => row.namespace === "mmapp_control")) + assert(protectedBefore.tables.every(row => row.namespace === "public")) + assert(protectedBefore.sequences.every(row => row.namespace === "public")) + await restored.query("UPDATE mmapp_control.owned_data_probe SET value='CONTROL_CHANGED_AGAIN'; SELECT pg_catalog.setval('mmapp_control.owned_data_probe_seq', 42)") + assert.equal(compareMigrationData(protectedBefore, await capture(restored, scoped)).equal, true) + await restored.query("ALTER EVENT TRIGGER owned_data_guard DISABLE") + const protectedAfter = await capture(restored, scoped) + assert.equal(protectedAfter.schema.eventTriggers?.[0].enabled, "D") + assert.throws(() => compareMigrationData(protectedBefore, protectedAfter)) + cases.push("actual scoped capture binds control schema and event trigger mode while excluding mutable control rows and sequences") + const altered = structuredClone(afterRestore); altered.tables[0].rowCount++ + assert.throws(() => compareMigrationData(beforeRestore, altered)) + cases.push("altered evidence vector is refused; unsigned content equality never grants enrollment or READY") + const layout = await (await import("./migration-schema-restore-layout.sql-helper.js")).acceptMigrationSchemaRestoreLayoutSql() + cases.push(...layout.names) + console.log("Owned migration data inventory SQL:", JSON.stringify({ cases: cases.length, names: cases, actualSql: true, actualDumpRestoreDataEquivalent: true, nativeCalls: 0, readyAuthorized: false, allReplicaQuiescence: false, productionDatabaseTouched: false })) +} finally { + if (restored) { await restored.end(); restored = undefined } + const owned = (await pool.query("SELECT datname,pg_get_userbyid(datdba)=current_user AS owned FROM pg_database WHERE datname=$1", [restoredName])).rows + if (owned.length) { assert.equal(owned.length, 1); assert.equal(owned[0].owned, true); assert.equal(owned[0].datname, restoredName); await pool.query('DROP DATABASE "' + restoredName + '"') } + await unlink(archive).catch(error => { if ((error as { code?: string }).code !== "ENOENT") throw error }) + await closePool() +} diff --git a/backend/src/db/migration-data-inventory.test.ts b/backend/src/db/migration-data-inventory.test.ts new file mode 100644 index 00000000..85875588 --- /dev/null +++ b/backend/src/db/migration-data-inventory.test.ts @@ -0,0 +1,241 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { readFileSync } from "node:fs" +import { captureMigrationData, compareMigrationData, MIGRATION_DATA_CONTEXT_QUERY, MigrationDataInventoryError, type CapturedMigrationData } from "./migration-data-inventory.js" +import { MIGRATION_SCHEMA_QUERIES, type SchemaSection } from "./migration-schema-inventory.js" + +type Row = Record +const schemaMetadata = { backend_pid: "71", transaction_started: "2026-10-25 10:00:00+00", read_only: "on", isolation: "repeatable read", search_path: "pg_catalog", timezone: "UTC", date_style: "ISO, YMD", standard_strings: "on", server_version: "180000", statement_timeout: "5s" } +const metadata = { ...schemaMetadata, row_security: "off", extra_float_digits: "3", bytea_output: "hex", interval_style: "postgres", client_encoding: "UTF8", lc_monetary: "C", lc_numeric: "C" } +const relation = (name: string, kind = "r", extra: Row = {}) => ({ namespace: "public", name, kind, persistence: "p", owner: "postgres", acl: null, options: null, comment: null, access_method: ["r", "p", "m"].includes(kind) ? "heap" : null, tablespace: null, toast_options: null, of_type: null, row_security: false, force_row_security: false, replica_identity: "d", is_partition: false, populated: true, partition_key: kind === "p" ? "RANGE (id)" : null, partition_bound: null, ...extra }) +function schemaFixture() { + const data = Object.fromEntries(Object.keys(MIGRATION_SCHEMA_QUERIES).map(key => [key, []])) as unknown as Record + data.database = [{ encoding: "UTF8", collate: "C", ctype: "C", locale_provider: "c", locale: null, locale_rules: null, collation_version: null, default_tablespace: "pg_default", owner: "postgres", acl: null, comment: null }] + data.namespaces = [{ namespace: "public", owner: "postgres", acl: null, comment: null }] + data.relations = [relation("servers"), relation("parent", "p"), relation("child", "r", { is_partition: true, partition_bound: "FOR VALUES FROM (0) TO (10)" }), relation("cached", "m"), relation("never_execute", "v"), relation("ids", "S")] + data.views = ["cached", "never_execute"].map(name => ({ namespace: "public", name, kind: name === "cached" ? "m" : "v", definition: "SELECT 1" })) + data.inheritance = [{ namespace: "public", child: "child", parent_namespace: "public", parent: "parent", position: 1, detach_pending: false }] + data.sequences = [{ namespace: "public", name: "ids", type: "bigint", start: "1", increment: "1", maximum: "9223372036854775807", minimum: "1", cache: "1", cycle: false, owner_namespace: null, owner_relation: null, owner_column: null, owner_dependency: null }] + return data +} +interface FakeOptions { + rows?: Record + metadata?: Row + mutate?: (row: Row, kind: string, index: number) => void + sequence?: (read: number) => { last_value: string; is_called: boolean } + queryError?: string + preserveOrder?: boolean + includeControl?: boolean +} +function fake(options: FakeOptions = {}) { + const calls: string[] = [], data = schemaFixture() + if (options.includeControl) { + data.namespaces.push({ ...data.namespaces[0], namespace: "mmapp_control" }) + data.relations.push(relation("control_log", "r", { namespace: "mmapp_control" }), relation("control_ids", "S", { namespace: "mmapp_control" })) + data.sequences.push({ ...data.sequences[0], namespace: "mmapp_control", name: "control_ids" }) + } + let cursor: string | undefined, offset = 0, reads = 0, rowIndex = 0 + const envelope = (kind: string, fields: Row = {}) => { + const row = { ...metadata, ...options.metadata, ...fields } + options.mutate?.(row, kind, rowIndex++) + return row + } + return { calls, data, query: async (sql: string, values: unknown[] = []) => { + calls.push(sql) + const section = (Object.keys(MIGRATION_SCHEMA_QUERIES) as SchemaSection[]).find(section => MIGRATION_SCHEMA_QUERIES[section] === sql) + if (section) { + assert.deepEqual(values, ["database", "namespaces"].includes(section) ? [] : [options.includeControl ? ["mmapp_control", "public"] : ["public"]]) + return { rows: [{ ...schemaMetadata, objects: structuredClone(data[section]) }] } + } + assert.deepEqual(values, []) + if (sql === MIGRATION_DATA_CONTEXT_QUERY) return { rows: [envelope("context")] } + if (sql.startsWith("/* migration-data:sequence */")) { + assert(sql.endsWith('FROM "public"."ids"')) + return { rows: [envelope("sequence", options.sequence?.(++reads) ?? { last_value: "42", is_called: true })] } + } + if (sql.startsWith("/* migration-data:declare */")) { + assert.equal(cursor, undefined) + const match = /FROM ONLY "public"\."([^"]+)" AS t/.exec(sql) + assert(match); assert.notEqual(match[1], "never_execute") + assert(sql.includes('ORDER BY row_text COLLATE pg_catalog."C"')) + assert(sql.includes("octet_length(row_text)<=1048576")) + assert(sql.includes("pg_catalog.jsonb_build_array(pg_catalog.to_jsonb(t),pg_catalog.record_out(t)::text)::text AS row_text")) + cursor = match[1]; offset = 0 + return { rows: [] } + } + if (sql === "/* migration-data:fetch */ FETCH FORWARD 16 FROM mm_migration_data_inventory") { + assert(cursor) + if (options.queryError) throw new Error(options.queryError) + const raw = [...(options.rows?.[cursor] ?? [])] + if (!options.preserveOrder) raw.sort((a, b) => Buffer.compare(Buffer.from(a), Buffer.from(b))) + const batch = raw.slice(offset, offset + 16); offset += batch.length + return { rows: batch.map(text => envelope("row", { row_text: Buffer.byteLength(text) <= 1048576 ? text : null, row_bytes: String(Buffer.byteLength(text)) })) } + } + assert.equal(sql, "/* migration-data:close */ CLOSE mm_migration_data_inventory") + assert(cursor); cursor = undefined + return { rows: [] } + } } +} +let cases = 0 +async function test(name: string, work: () => unknown | Promise) { await work(); cases++; console.log("ok", name) } +const captured = await captureMigrationData(fake({ rows: { servers: ['{"id": 1}', '{"id": 1}', '{"id": 2}'], child: ['{"id": 3}'], cached: ['{"id": 4}'] } })) +const clone = () => structuredClone(captured) +await test("actual catalog relations drive bounded physical-only cursor capture", () => { + assert.equal(captured.rowCount, 5) + assert.deepEqual(captured.tables.map(table => table.name), ["cached", "child", "parent", "servers"]) + assert.equal(captured.tables.find(table => table.name === "parent")?.rowCount, 0) + assert.deepEqual(captured.sequences, [{ namespace: "public", name: "ids", lastValue: "42", isCalled: true }]) + assert(!captured.canonical.includes('"id": 1')) + assert(Object.isFrozen(captured)); assert(Object.isFrozen(captured.tables)); assert(Object.isFrozen(captured.schema.sections.relations[0])) + assert.equal(captured.ready, false); assert.equal(captured.allReplicaQuiescence, false) + assert.deepEqual(compareMigrationData(captured, clone()), { equal: true, evidenceOnly: true, enrollmentAuthorized: false, allReplicaQuiescence: false, ready: false, sha256: captured.sha256 }) +}) +await test("ordering is stable and duplicate rows affect the multiset digest", async () => { + const original = await captureMigrationData(fake({ rows: { servers: ['{"id": 1}', '{"id": 1}', '{"id": 2}'] } })) + const reordered = await captureMigrationData(fake({ rows: { servers: ['{"id": 2}', '{"id": 1}', '{"id": 1}'] } })) + assert.equal(original.sha256, reordered.sha256) + for (const rows of [['{"id": 1}', '{"id": 2}'], ['{"id": 1}', '{"id": 2}', '{"id": 2}']]) { + const different = await captureMigrationData(fake({ rows: { servers: rows } })) + assert.throws(() => compareMigrationData(original, different)) + } +}) +await test("scoped data binds control schema definitions and refuses extra, incomplete or altered scope vectors", async () => { + const scope = { applicationNamespaces: ["public", "mmapp_control"] } + const capture = (client = fake({ includeControl: true })) => captureMigrationData(client, { scope, dataNamespaces: ["public"] }) + const reference = await capture() + assert.equal(reference.format, "pg18-application-data-v3") + assert.deepEqual(JSON.parse(reference.canonical).dataNamespaces, ["public"]) + assert.equal(reference.tables.length, 4) + assert.equal(reference.sequences.length, 1) + assert(reference.schema.sections.relations.some(row => row.name === "control_log")) + assert.equal(compareMigrationData(reference, await capture()).equal, true) + const changedDefinition = fake({ includeControl: true }) + changedDefinition.data.relations.find(row => row.name === "control_log")!.comment = "changed" + const changedSchema = await capture(changedDefinition) + assert.throws(() => compareMigrationData(reference, changedSchema)) + const changedData = await capture(fake({ includeControl: true, rows: { servers: ['{"id": 1}'] } })) + assert.throws(() => compareMigrationData(reference, changedData)) + for (const dataNamespaces of [[], ["public", "public"], ["public", "absent"]]) { + await assert.rejects(captureMigrationData(fake({ includeControl: true }), { scope, dataNamespaces }), MigrationDataInventoryError) + } + const expanded = { ...reference, dataNamespaces: ["mmapp_control", "public"] } + assert.throws(() => compareMigrationData(expanded, expanded), /Incomplete data evidence vector/) + const incomplete = { ...reference, sequences: [] } + assert.throws(() => compareMigrationData(incomplete, incomplete), /Incomplete data evidence vector/) + const extra = { ...reference, sequences: [...reference.sequences, { ...reference.sequences[0], namespace: "mmapp_control", name: "control_ids" }] } + assert.throws(() => compareMigrationData(extra, extra), /Incomplete data evidence vector/) +}) +await test("v2 native-record supplement distinguishes identical JSONB with different array bounds", async () => { + // PostgreSQL renders these native arrays differently while to_jsonb yields + // the same ["a","b"]. The SQL acceptance owner verifies real PG output. + const logical = { values: ["a", "b"] } + const zeroBased = JSON.stringify([logical, '("[0:1]={a,b}")']) + const oneBased = JSON.stringify([logical, '("{a,b}")']) + assert.deepEqual(JSON.parse(zeroBased)[0], JSON.parse(oneBased)[0]) + const zero = await captureMigrationData(fake({ rows: { servers: [zeroBased] } })) + const one = await captureMigrationData(fake({ rows: { servers: [oneBased] } })) + assert.equal(zero.format, "pg18-application-data-v2") + assert.equal(Object.hasOwn(zero, "dataNamespaces"), false) + assert.equal(Object.hasOwn(JSON.parse(zero.canonical), "dataNamespaces"), false) + assert.notEqual(zero.tables.find(table => table.name === "servers")!.sha256, one.tables.find(table => table.name === "servers")!.sha256) + assert.throws(() => compareMigrationData(zero, one)) +}) +await test("batches fetch to exhaustion beyond sixteen rows without truncation", async () => { + const client = fake({ rows: { servers: Array.from({ length: 33 }, (_, id) => JSON.stringify({ id })) } }) + const result = await captureMigrationData(client) + assert.equal(result.rowCount, 33) + assert.equal(client.calls.filter(sql => sql.includes("migration-data:fetch")).length, 7) + assert.equal(client.calls.filter(sql => sql.includes("migration-data:declare")).length, 4) + assert.equal(client.calls.filter(sql => sql.includes("migration-data:close")).length, 4) +}) +await test("bounds are fail closed and cleanup closes only the owned cursor", async () => { + for (const options of [{ maxRows: 1 }, { maxBytes: 3 }]) { + const client = fake({ rows: { servers: ['{"id": 1}', '{"id": 2}'] } }) + await assert.rejects(() => captureMigrationData(client, options), MigrationDataInventoryError) + assert.equal(client.calls.filter(sql => sql.includes("migration-data:close")).length, 4) + } + const client = fake({ rows: { servers: ['"' + "x".repeat(1048576) + '"'] } }) + await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError) + assert(client.calls.at(-1)?.includes("migration-data:close")) +}) +await test("invalid trusted bounds are rejected before any client query", async () => { + for (const options of [{ maxRows: 0 }, { maxRows: 10000001 }, { maxBytes: 1073741825 }, { maxBytes: 0 }, { deadlineAt: Date.now() - 1 }, { deadlineAt: Date.now() + 121000 }]) { + const client = fake(); await assert.rejects(() => captureMigrationData(client, options), MigrationDataInventoryError); assert.equal(client.calls.length, 0) + } +}) +await test("unsafe GUCs, backend/transaction drift and per-row drift are refused", async () => { + for (const [key, value] of Object.entries({ row_security: "on", extra_float_digits: "0", bytea_output: "escape", interval_style: "iso_8601", client_encoding: "LATIN1", lc_monetary: "en_US.UTF-8", lc_numeric: "de_DE.UTF-8", read_only: "off", isolation: "read committed", timezone: "Europe/Moscow" })) { + await assert.rejects(() => captureMigrationData(fake({ metadata: { [key]: value } })), MigrationDataInventoryError) + } + for (const [key, value] of Object.entries({ backend_pid: "72", transaction_started: "2026-10-25 10:01:00+00", extra_float_digits: "2" })) { + await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (kind === "row") row[key] = value } })), MigrationDataInventoryError) + } +}) +await test("non-MVCC sequence instability and untrusted raw-error leakage are refused", async () => { + await assert.rejects(() => captureMigrationData(fake({ sequence: read => ({ last_value: String(read), is_called: true }) })), MigrationDataInventoryError) + for (const last_value of ["9223372036854775808", "-9223372036854775809", "-0", "00"]) await assert.rejects(() => captureMigrationData(fake({ sequence: () => ({ last_value, is_called: true }) })), MigrationDataInventoryError) + const privateValue = "PRIVATE CUSTOMER RECORD" + try { await captureMigrationData(fake({ queryError: privateValue })); assert.fail("must fail") } + catch (error) { assert(error instanceof MigrationDataInventoryError); assert(!String(error).includes(privateValue)); assert.equal(error.cause, undefined) } +}) +await test("catalog identity ambiguity and sequence definitions without relations are refused", async () => { + const duplicate = fake(); duplicate.data.relations.push({ ...duplicate.data.relations[0], owner: "another" }) + await assert.rejects(() => captureMigrationData(duplicate), MigrationDataInventoryError) + const extra = fake(); extra.data.sequences.push({ ...extra.data.sequences[0], name: "orphan" }) + await assert.rejects(() => captureMigrationData(extra), MigrationDataInventoryError) +}) +await test("cursor owner is not closed after connection ownership changes", async () => { + let declared = false + const base = fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (declared && (kind === "row" || kind === "context")) row.backend_pid = "999" } }) + const client = { query: async (sql: string, values?: unknown[]) => { + const result = await base.query(sql, values) + if (sql.includes("migration-data:declare")) declared = true + return result + } } + await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError) + assert.equal(base.calls.filter(sql => sql.includes("migration-data:close")).length, 0) +}) +await test("concurrent capture cannot take over another capture cursor", async () => { + let release!: () => void, entered!: () => void + const waiting = new Promise(resolve => { release = resolve }), arrived = new Promise(resolve => { entered = resolve }) + const base = fake(); let first = true + const client = { query: async (sql: string, values?: unknown[]) => { + if (first) { first = false; entered(); await waiting } + return base.query(sql, values) + } } + const capture = captureMigrationData(client); await arrived + await assert.rejects(() => captureMigrationData(client), MigrationDataInventoryError) + release(); await capture + await captureMigrationData(client) +}) +await test("out-of-order rows, mismatched byte envelopes and extra fields fail closed", async () => { + await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 2}', '{"id": 1}'] }, preserveOrder: true })), MigrationDataInventoryError) + for (const mutate of [(row: Row) => { row.row_bytes = "0" }, (row: Row) => { row.surprise = true }]) { + await assert.rejects(() => captureMigrationData(fake({ rows: { servers: ['{"id": 1}'] }, mutate: (row, kind) => { if (kind === "row") mutate(row) } })), MigrationDataInventoryError) + } +}) +await test("comparison rejects missing/extra/duplicate tables, sequences and tampered hashes/counts/schema", () => { + const mutations: ((value: CapturedMigrationData) => void)[] = [ + value => { value.tables.pop() }, value => { value.tables.push(value.tables[0]) }, value => { value.tables[1] = value.tables[0] }, + value => { value.sequences = [] }, value => { value.sequences.push(value.sequences[0]) }, value => { value.sequences[0].lastValue = "43" }, + value => { value.tables[0].sha256 = "a".repeat(64) }, value => { value.tables[0].rowCount++ }, value => { value.rowCount++ }, + value => { value.tables[0].byteCount++ }, value => { value.sha256 = "a".repeat(64) }, value => { value.schemaSha256 = "a".repeat(64) }, + value => { value.schema.sections.relations[0].owner = "another" }, value => { value.schema.observedPhysical.sha256 = "a".repeat(64) }, + value => { Reflect.set(value, "authority", true) }, value => { Reflect.set(value, "ready", true) }, + ] + for (const mutate of mutations) { const value = clone(); mutate(value); assert.throws(() => compareMigrationData(captured, value)) } +}) +await test("comparison recomputes an envelope rather than trusting a supplied matching hash", () => { + const value = clone(); value.tables[0].sha256 = "a".repeat(64); value.canonical = captured.canonical; value.sha256 = captured.sha256 + assert.throws(() => compareMigrationData(value, value)) + const empty = clone(); const table = empty.tables.find(table => table.rowCount === 0)! + table.sha256 = createHash("sha256").update("").digest("hex") + assert.throws(() => compareMigrationData(empty, empty)) +}) +await test("no configured pool, signing, transaction control, mutation or enrollment imports", () => { + const source = readFileSync(new URL("./migration-data-inventory.ts", import.meta.url), "utf8") + const imports = [...source.matchAll(/from "([^"]+)"/g)].map(match => match[1]) + assert.deepEqual(imports, ["node:crypto", "./migration-schema-inventory.js"]) + assert(!/\b(?:INSERT INTO|UPDATE public|DELETE FROM|nextval\(|setval\(|set_config\(|BEGIN TRANSACTION|COMMIT;|ROLLBACK;|createSign\()/.test(source)) +}) +console.log(`${cases} migration data inventory tests passed`) diff --git a/backend/src/db/migration-data-inventory.ts b/backend/src/db/migration-data-inventory.ts new file mode 100644 index 00000000..65ae4f1c --- /dev/null +++ b/backend/src/db/migration-data-inventory.ts @@ -0,0 +1,266 @@ +import { createHash, type Hash } from "node:crypto" +import { captureMigrationSchema, compareMigrationSchemas, type CapturedMigrationSchema, type SchemaInventoryClient, type SchemaInventoryScope } from "./migration-schema-inventory.js" + +/** Independent read-only, unsigned evidence. The caller owns the checked-out + * PG18 transaction and its rollback; this module never configures a pool, + * changes GUCs, mutates application data, signs or authorizes enrollment. */ +export class MigrationDataInventoryError extends Error { readonly code = "MIGRATION_DATA_REFUSED" } +type Row = Record +type Table = { namespace: string; name: string; kind: "r" | "p" | "m"; rowCount: number; byteCount: number; sha256: string } +type Sequence = { namespace: string; name: string; lastValue: string; isCalled: boolean } +export interface CapturedMigrationData { + format: "pg18-application-data-v2" | "pg18-application-data-v3" + schema: CapturedMigrationSchema + schemaSha256: string + dataNamespaces?: readonly string[] + tables: Table[] + sequences: Sequence[] + rowCount: number + byteCount: number + allReplicaQuiescence: false + ready: false + canonical: string + sha256: string +} +export interface MigrationDataInventoryOptions { + scope?: SchemaInventoryScope + dataNamespaces?: readonly string[] + includeControlEventTriggers?: boolean + deadlineAt?: number + maxRows?: number + maxBytes?: number +} +const MAX_ROW_BYTES = 1048576 +const CURSOR = "mm_migration_data_inventory" +const activeClients = new WeakSet() +const hash = (text: string) => createHash("sha256").update(text).digest("hex") +function fail(message: string): never { throw new MigrationDataInventoryError(message) } +function object(value: unknown): value is Row { return !!value && typeof value === "object" && Object.getPrototypeOf(value) === Object.prototype } +function keys(value: unknown, expected: string): asserts value is Row { + if (!object(value) || JSON.stringify(Object.keys(value).sort()) !== JSON.stringify(expected.split(" ").sort())) fail("Unexpected data evidence shape") +} +function canonical(value: unknown): string { + if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value) + if (typeof value === "number" && Number.isSafeInteger(value)) return JSON.stringify(value) + if (Array.isArray(value)) return "[" + value.map(canonical).join(",") + "]" + if (object(value)) return "{" + Object.keys(value).sort().map(key => JSON.stringify(key) + ":" + canonical(value[key])).join(",") + "}" + fail("Invalid data evidence value") +} +function freeze(value: T): T { + if (value && typeof value === "object") { for (const child of Object.values(value)) freeze(child); Object.freeze(value) } + return value +} +const quote = (name: string) => '"' + name.replaceAll('"', '""') + '"' +const identity = (value: { namespace: string; name: string }) => canonical([value.namespace, value.name]) +function sequenceValue(value: unknown): value is string { + return typeof value === "string" && /^(0|-?[1-9]\d{0,18})$/.test(value) && BigInt(value) >= -9223372036854775808n && BigInt(value) <= 9223372036854775807n +} +function frame(digest: Hash, text: string) { + const bytes = Buffer.from(text, "utf8"), length = Buffer.alloc(8) + length.writeBigUInt64BE(BigInt(bytes.length)); digest.update(length); digest.update(bytes) +} +const metadata = `pg_catalog.pg_backend_pid()::text AS backend_pid, + pg_catalog.transaction_timestamp()::text AS transaction_started, + pg_catalog.current_setting('transaction_read_only') AS read_only, + pg_catalog.current_setting('transaction_isolation') AS isolation, + pg_catalog.current_setting('search_path') AS search_path, + pg_catalog.current_setting('TimeZone') AS timezone, + pg_catalog.current_setting('DateStyle') AS date_style, + pg_catalog.current_setting('standard_conforming_strings') AS standard_strings, + pg_catalog.current_setting('server_version_num') AS server_version, + pg_catalog.current_setting('statement_timeout') AS statement_timeout, + pg_catalog.current_setting('row_security') AS row_security, + pg_catalog.current_setting('extra_float_digits') AS extra_float_digits, + pg_catalog.current_setting('bytea_output') AS bytea_output, + pg_catalog.current_setting('IntervalStyle') AS interval_style, + pg_catalog.current_setting('client_encoding') AS client_encoding, + pg_catalog.current_setting('lc_monetary') AS lc_monetary, + pg_catalog.current_setting('lc_numeric') AS lc_numeric` +export const MIGRATION_DATA_CONTEXT_QUERY = `/* migration-data:context */ SELECT ${metadata}` +const contextKeys = "backend_pid transaction_started read_only isolation search_path timezone date_style standard_strings server_version statement_timeout row_security extra_float_digits bytea_output interval_style client_encoding lc_monetary lc_numeric" +const schemaContextKeys = "backend_pid transaction_started read_only isolation search_path timezone date_style standard_strings server_version statement_timeout".split(" ") +function context(row: Row, extra = ""): string { + keys(row, contextKeys + (extra ? " " + extra : "")) + if (typeof row.backend_pid !== "string" || !/^\d+$/.test(row.backend_pid) || typeof row.transaction_started !== "string" || !Number.isFinite(Date.parse(row.transaction_started)) || row.read_only !== "on" || !["repeatable read", "serializable"].includes(String(row.isolation)) || row.search_path !== "pg_catalog" || row.timezone !== "UTC" || row.date_style !== "ISO, YMD" || row.standard_strings !== "on" || typeof row.server_version !== "string" || !/^18\d{4}$/.test(row.server_version) || row.row_security !== "off" || row.extra_float_digits !== "3" || row.bytea_output !== "hex" || row.interval_style !== "postgres" || row.client_encoding !== "UTF8" || row.lc_monetary !== "C" || row.lc_numeric !== "C") fail("PG18 fixed data context and row_security=off required") + const timeout = /^(\d+(?:\.\d+)?)(ms|s|min)?$/.exec(String(row.statement_timeout)) + const milliseconds = timeout ? Number(timeout[1]) * ({ ms: 1, s: 1000, min: 60000 }[timeout[2] ?? "ms"] ?? 0) : 0 + if (milliseconds < 1 || milliseconds > 10000) fail("Bounded data statement timeout required") + return canonical(Object.fromEntries(contextKeys.split(" ").map(key => [key, row[key]]))) +} +function dataScope(namespaces: readonly string[] | undefined, schema: CapturedMigrationSchema): string[] { + const allowed = schema.scope.applicationNamespaces + if (!Array.isArray(namespaces) || !namespaces.length || namespaces.length > allowed.length) fail("Invalid data namespace scope") + const sorted = [...namespaces].sort() + if (new Set(sorted).size !== sorted.length || sorted.some(name => !allowed.includes(name))) fail("Invalid data namespace scope") + return sorted +} +function dataRelations(schema: CapturedMigrationSchema, namespaces: readonly string[]) { + // Schema validation already refuses foreign/external and unsupported kinds. + // Views are definitions only and are never executed for data capture. + for (const row of schema.sections.relations) if (!["r", "p", "m", "i", "I", "S", "v", "c"].includes(String(row.kind))) fail("Unsupported data-bearing relation kind") + const relations = schema.sections.relations.map(row => ({ namespace: String(row.namespace), name: String(row.name), kind: String(row.kind) })) + if (new Set(relations.map(identity)).size !== relations.length) fail("Duplicate captured relation identity") + return relations.filter(row => namespaces.includes(row.namespace) && ["r", "p", "m"].includes(row.kind)).map(row => ({ ...row, kind: row.kind as Table["kind"] })).sort((a, b) => identity(a) < identity(b) ? -1 : identity(a) > identity(b) ? 1 : 0) +} +function sequenceRelations(schema: CapturedMigrationSchema, namespaces: readonly string[]) { + const sequences = schema.sections.sequences.map(row => ({ namespace: String(row.namespace), name: String(row.name) })).sort((a, b) => identity(a) < identity(b) ? -1 : identity(a) > identity(b) ? 1 : 0) + if (new Set(sequences.map(identity)).size !== sequences.length || sequences.some(sequence => !schema.sections.relations.some(row => row.kind === "S" && row.namespace === sequence.namespace && row.name === sequence.name))) fail("Invalid captured sequence identity") + return sequences.filter(row => namespaces.includes(row.namespace)) +} +function envelope(value: Omit): string { + // Logical schema is content-checked independently. Fast-default physical + // caches may differ despite identical logical values; keep them in schema. + return canonical({ format: value.format, schemaSha256: value.schemaSha256, ...(value.dataNamespaces ? { dataNamespaces: value.dataNamespaces } : {}), tables: value.tables, sequences: value.sequences, rowCount: value.rowCount, byteCount: value.byteCount, allReplicaQuiescence: false, ready: false }) +} + +/** Row text is jsonb_build_array(to_jsonb(t),record_out(t)::text)::text. + * The native record supplement preserves array dimensions/lower bounds and + * native distinctions that a standalone JSONB conversion can discard. Fixed + * numeric/monetary locales also bind native output formatting. + * ONLY avoids counting inherited/partition rows twice; C ordering and framed + * hashes preserve multiplicity without storing or exposing raw row values. + * Sequences are non-MVCC: two stable reads detect some activity, never prove + * writer/replica quiescence (including change-and-change-back races). */ +export async function captureMigrationData(client: SchemaInventoryClient, options: MigrationDataInventoryOptions = {}): Promise { + const start = Date.now(), deadline = options.deadlineAt ?? start + 60000 + const maxRows = options.maxRows ?? 1000000, maxBytes = options.maxBytes ?? 134217728 + if (!Number.isSafeInteger(deadline) || deadline <= start || deadline - start > 120000 || !Number.isSafeInteger(maxRows) || maxRows < 1 || maxRows > 10000000 || !Number.isSafeInteger(maxBytes) || maxBytes < 1 || maxBytes > 1073741824) fail("Invalid data capture bounds") + if (activeClients.has(client)) fail("Concurrent data capture on one client refused") + activeClients.add(client) + let owner = "", baseline: Row = {}, cursorOwned = false + const checkDeadline = () => { if (Date.now() >= deadline) fail("Data capture deadline exceeded") } + const query = async (sql: string, values: unknown[] = []) => { + checkDeadline() + let result: Awaited> + try { result = await client.query(sql, values) } catch { fail("Data capture query failed; caller must roll back transaction") } + checkDeadline() + if (!result || !Array.isArray(result.rows)) fail("Data query result unavailable") + return result.rows + } + const checkContext = async () => { + const rows = await query(MIGRATION_DATA_CONTEXT_QUERY) + if (rows.length !== 1) fail("Data context unavailable") + const current = context(rows[0]) + if (owner && owner !== current) fail("Data capture changed backend, transaction or context") + owner = current; baseline = rows[0] + } + try { + await checkContext() + const schema = await captureMigrationSchema({ query: async (sql, values) => { + await checkContext() + const rows = await query(sql, values) + for (const row of rows) for (const key of schemaContextKeys) if (row[key] !== baseline[key]) fail("Schema/data capture context changed") + await checkContext() + return { rows } + } }, { scope: options.scope, deadlineAt: deadline, includeControlEventTriggers: options.includeControlEventTriggers }) + compareMigrationSchemas(schema, schema) + const scoped = options.dataNamespaces !== undefined || options.includeControlEventTriggers === true + const namespaces = options.dataNamespaces === undefined ? schema.scope.applicationNamespaces : dataScope(options.dataNamespaces, schema) + await checkContext() + const readSequence = async (sequence: { namespace: string; name: string }): Promise => { + const rows = await query(`/* migration-data:sequence */ SELECT ${metadata}, last_value::text AS last_value, is_called FROM ${quote(sequence.namespace)}.${quote(sequence.name)}`) + if (rows.length !== 1 || context(rows[0], "last_value is_called") !== owner) fail("Sequence context changed") + if (!sequenceValue(rows[0].last_value) || typeof rows[0].is_called !== "boolean") fail("Invalid sequence state") + return { ...sequence, lastValue: rows[0].last_value, isCalled: rows[0].is_called } + } + const sequences: Sequence[] = [] + for (const sequence of sequenceRelations(schema, namespaces)) sequences.push(await readSequence(sequence)) + const tables: Table[] = [] + let rowCount = 0, byteCount = 0 + for (const relation of dataRelations(schema, namespaces)) { + await checkContext() + const digest = createHash("sha256") + frame(digest, "pg18-application-data-table-v2-jsonb-record"); frame(digest, schema.sha256); frame(digest, canonical(relation)) + const table: Table = { ...relation, rowCount: 0, byteCount: 0, sha256: "" } + await query(`/* migration-data:declare */ DECLARE ${CURSOR} NO SCROLL CURSOR WITHOUT HOLD FOR + WITH raw AS MATERIALIZED (SELECT pg_catalog.jsonb_build_array(pg_catalog.to_jsonb(t),pg_catalog.record_out(t)::text)::text AS row_text FROM ONLY ${quote(relation.namespace)}.${quote(relation.name)} AS t) + SELECT ${metadata}, pg_catalog.octet_length(row_text)::text AS row_bytes, + CASE WHEN pg_catalog.octet_length(row_text)<=${MAX_ROW_BYTES} THEN row_text ELSE NULL END AS row_text + FROM raw ORDER BY row_text COLLATE pg_catalog."C"`) + cursorOwned = true + let previous: Buffer | undefined + for (;;) { + const rows = await query(`/* migration-data:fetch */ FETCH FORWARD 16 FROM ${CURSOR}`) + if (rows.length > 16) fail("Data cursor batch exceeds bound") + for (const row of rows) { + if (context(row, "row_bytes row_text") !== owner) fail("Data row context changed") + if (typeof row.row_bytes !== "string" || !/^(0|[1-9]\d{0,9})$/.test(row.row_bytes) || typeof row.row_text !== "string") fail("Data row overflow or invalid shape") + const size = Buffer.byteLength(row.row_text), bytes = Buffer.from(row.row_text, "utf8") + if (size > MAX_ROW_BYTES || size !== Number(row.row_bytes)) fail("Data row byte bound exceeded") + if (previous && Buffer.compare(previous, bytes) > 0) fail("Data cursor order changed") + previous = bytes + if (++rowCount > maxRows || (byteCount += size) > maxBytes) fail("Data capture row/byte bound exceeded") + table.rowCount++; table.byteCount += size; frame(digest, row.row_text) + } + if (!rows.length) break + } + await checkContext() + await query(`/* migration-data:close */ CLOSE ${CURSOR}`); cursorOwned = false + table.sha256 = digest.digest("hex"); tables.push(table) + } + for (const sequence of sequences) if (canonical(sequence) !== canonical(await readSequence(sequence))) fail("Sequence state changed during data capture") + await checkContext() + const value: Omit = { + format: scoped ? "pg18-application-data-v3" : "pg18-application-data-v2", + schema, schemaSha256: schema.sha256, ...(scoped ? { dataNamespaces: namespaces } : {}), + tables, sequences, rowCount, byteCount, allReplicaQuiescence: false, ready: false, + } + const text = envelope(value) + return freeze({ ...value, canonical: text, sha256: hash(text) }) + } catch { + // Database/serialization errors can contain row values. Never retain causes + // or interpolate driver errors into evidence or user-visible errors. + throw new MigrationDataInventoryError("Data capture refused; no complete evidence was produced") + } finally { + if (cursorOwned) { + try { + // Never close a cursor belonging to a different backend/transaction. + const result = await client.query(MIGRATION_DATA_CONTEXT_QUERY) + if (result.rows.length === 1 && context(result.rows[0]) === owner) await client.query(`/* migration-data:close */ CLOSE ${CURSOR}`) + } catch { /* An aborted transaction requires caller rollback. */ } + } + activeClients.delete(client) + } +} + +/** Recomputes envelope/schema hashes, validates exact table/sequence vectors, + * and compares evidence. Row hashes cannot be recomputed without private raw + * rows: this detects content tampering, not a forged issuer/reference. */ +export function compareMigrationData(expected: CapturedMigrationData, observed: CapturedMigrationData) { + const checked = (value: CapturedMigrationData) => { + const scoped = value?.format === "pg18-application-data-v3" + keys(value, "format schema schemaSha256 tables sequences rowCount byteCount allReplicaQuiescence ready canonical sha256" + (scoped ? " dataNamespaces" : "")) + if ((!scoped && value.format !== "pg18-application-data-v2") || value.allReplicaQuiescence !== false || value.ready !== false) fail("Unsupported data evidence format/authority") + compareMigrationSchemas(value.schema, value.schema) + if (value.schemaSha256 !== value.schema.sha256 || !Array.isArray(value.tables) || !Array.isArray(value.sequences)) fail("Invalid data schema binding/vector") + const namespaces = scoped ? dataScope(value.dataNamespaces, value.schema) : value.schema.scope.applicationNamespaces + if (scoped && canonical(value.dataNamespaces) !== canonical(namespaces)) fail("Noncanonical data namespace scope") + const relations = dataRelations(value.schema, namespaces), sequenceNames = sequenceRelations(value.schema, namespaces) + if (value.tables.length !== relations.length || value.sequences.length !== sequenceNames.length) fail("Incomplete data evidence vector") + let count = 0, bytes = 0 + for (let index = 0; index < value.tables.length; index++) { + const table = value.tables[index] + keys(table, "namespace name kind rowCount byteCount sha256") + if (canonical({ namespace: table.namespace, name: table.name, kind: table.kind }) !== canonical(relations[index]) || !Number.isSafeInteger(table.rowCount) || table.rowCount < 0 || table.rowCount > 10000000 || !Number.isSafeInteger(table.byteCount) || table.byteCount < 0 || table.byteCount > 1073741824 || table.rowCount === 0 && table.byteCount !== 0 || table.rowCount > 0 && (table.byteCount < table.rowCount * 2 || table.byteCount > table.rowCount * MAX_ROW_BYTES) || typeof table.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(table.sha256)) fail("Invalid table evidence identity/count/digest") + if (table.rowCount === 0) { + const digest = createHash("sha256") + frame(digest, "pg18-application-data-table-v2-jsonb-record"); frame(digest, value.schemaSha256); frame(digest, canonical(relations[index])) + if (digest.digest("hex") !== table.sha256) fail("Invalid empty-table digest") + } + count += table.rowCount; bytes += table.byteCount + } + for (let index = 0; index < value.sequences.length; index++) { + const sequence = value.sequences[index] + keys(sequence, "namespace name lastValue isCalled") + if (identity(sequence) !== identity(sequenceNames[index]) || !sequenceValue(sequence.lastValue) || typeof sequence.isCalled !== "boolean") fail("Invalid sequence evidence") + } + if (count > 10000000 || bytes > 1073741824 || value.rowCount !== count || value.byteCount !== bytes) fail("Invalid data evidence totals") + const text = envelope(value) + if (text !== value.canonical || hash(text) !== value.sha256) fail("Data evidence envelope content/digest altered") + return text + } + const reference = checked(expected), actual = checked(observed) + compareMigrationSchemas(expected.schema, observed.schema) + if (reference !== actual) fail("Observed data differs from captured reference") + return freeze({ equal: true as const, evidenceOnly: true as const, enrollmentAuthorized: false as const, allReplicaQuiescence: false as const, ready: false as const, sha256: hash(actual) }) +} diff --git a/backend/src/db/migration-manifest.test.ts b/backend/src/db/migration-manifest.test.ts new file mode 100644 index 00000000..f0252cd2 --- /dev/null +++ b/backend/src/db/migration-manifest.test.ts @@ -0,0 +1,247 @@ +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { readFile } from "node:fs/promises" +import { fileURLToPath } from "node:url" +import { MIGRATION_MANIFEST, CURRENT_REQUIRED_TABLES, analyzeMigration, assessLegacyMigrationBaseline, inventoryMigrationDirectory, migrationManifestDigest, MigrationManifestError, parseMigrationStatements, validateMigrationFileSet, type LegacyBaselineEvidence, type MigrationReview } from "./migration-manifest.js" + +let cases = 0 +async function test(name: string, work: () => Promise | void) { await work(); cases++; console.log("ok", name) } +const directory = fileURLToPath(new URL("../../drizzle/", import.meta.url)) +const inventory = await inventoryMigrationDirectory(directory) +const reviewed = (source: string, extra: Partial = {}) => { + const bytes = Buffer.from(source), file = "9000_fixture.sql" + return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra }) +} +await test("all 57 exact file pins and every top-level schema operation are inventoried", () => { + assert.equal(inventory.length, 57); assert.equal(MIGRATION_MANIFEST.length, 57) + assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17) + assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 40) + assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0)) + let tables = new Set() + for (const file of inventory) { + if (file.publicTableReset) tables = new Set(["schema_migrations"]) + for (const operation of file.operations) if (operation.command === "CREATE" && operation.objectKind === "TABLE" && operation.target) tables.add(operation.target.replace(/^public\./, "")) + } + assert.deepEqual([...tables].sort(), [...CURRENT_REQUIRED_TABLES]); assert(CURRENT_REQUIRED_TABLES.length > 80) +}) +await test("DO BEGIN and stored function BEGIN are never transaction boundaries", () => { + const compact = inventory.find(file => file.id === "0002_compact_schema")! + assert.equal(compact.transaction, "owner-transaction"); assert.equal(compact.topLevelTransactionControls.length, 0); assert.equal(compact.opaqueBodies[0].kind, "executed-do-body"); assert.equal(compact.publicTableReset, true) + assert(compact.effects.some(effect => effect.includes("DESTRUCTIVE"))) + const endpoint = inventory.find(file => file.id === "0029_mcp_routing_sources")! + assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2) + assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"]) +}) +await test("0035 through 0057 have exact atomic file-owned terminal markers", () => { + assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs", "0054_mcp_geoip_retirement", "0055_mcp_backup_schedule_retirement", "0056_mcp_manual_inverse_recovery_v8", "0057_mcp_routing_charge_audit_index"]) + for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) } + assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34) +}) +await test("0041 preserves nonempty full journals and defines only fixed atomic lossless terminal archive transitions", async () => { + const source = await readFile(new URL("../../drizzle/0041_server_poll_terminal_archives.sql", import.meta.url), "utf8") + const migration = inventory.find(file => file.id === "0041_server_poll_terminal_archives")! + assert.equal(migration.opaqueBodies.length, 13) + assert.equal(migration.operations[1].command, "DO"); assert.equal(migration.operations[2].command, "LOCK") + assert(source.includes("existing poll accounting requires explicit reconciliation")) + assert(source.includes("pg_get_constraintdef(oid)='CHECK ((reserved_rows = (admissions * 4019)))'")) + assert(source.includes("DROP CONSTRAINT server_poll_storage_usage_check,")); assert(source.includes("DROP CONSTRAINT server_poll_storage_usage_check1,")) + assert(source.includes("pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'")); assert(source.includes("FOR UPDATE NOWAIT")) + assert(source.includes("pg_try_advisory_xact_lock(hashtextextended('server-poll-execution-v1:mcp:'")) + assert(source.includes("pg_try_advisory_xact_lock(hashtextextended('server-poll-receipt-v1:'")) + assert(source.includes("o.error_code IS NOT NULL")); assert(source.includes("s.server_id IS NOT NULL")) + assert(source.includes("p.created_at>p.approved_at OR p.approved_at>o.created_at OR o.created_at>=LEAST(p.expires_at,a.authority_expires_at)")) + assert(source.includes("r.created_at=a.authority_expires_at")) + assert(!source.includes("o.finished_at>=a.authority_expires_at")); assert(!source.includes("s.finished_at>=a.authority_expires_at")) + assert(source.includes("s.state<>'verified'")); assert(source.includes("o.state<>'applied'")); assert(source.includes("p.approved_by IS NULL")) + assert(source.includes("outcome<>'confirmed'")); assert(source.includes("generate_series(0,pubs-1)")) + assert(source.includes("NEW.source_sha256<>encode(sha256(convert_to(payload::text,'UTF8')),'hex')")) + assert(source.includes("NEW.decoded_bytes<>octet_length(payload::text)")); assert(source.includes("compressed_bytes BETWEEN 1 AND 98048")) + assert(source.includes("compaction_xid=pg_current_xact_id()")); assert(!source.includes("mm.poll_storage_compact")) + assert(source.includes("ADD COLUMN compaction_claim uuid")); assert(source.includes("ADD COLUMN compaction_claimed_at timestamptz")) + assert(source.includes("ADD COLUMN compaction_claim_xid xid8")) + assert(source.includes("OLD.compaction_claim IS NULL AND OLD.compaction_claimed_at IS NULL")) + assert(source.includes("THEN NEW.compaction_claimed_at:=clock_timestamp(); NEW.compaction_claim_xid:=pg_current_xact_id(); RETURN NEW; END IF")) + assert(source.includes("a.compaction_claim IS NULL OR a.compaction_claimed_at IS NULL")) + assert(source.includes("a.compaction_claim_xid IS NULL OR a.compaction_claim_xid=pg_current_xact_id()")) + assert.equal((source.match(/DEFERRABLE INITIALLY DEFERRED/g) ?? []).length, 3) + assert(source.includes("reserved_rows=admissions*4019+compacted_admissions*2")) + assert(source.includes("reserved_bytes=admissions*8421376+compacted_admissions*32768+archive_bytes")) + assert(source.includes("GREATEST(a.retention_days,retention)")); assert(source.includes("activity:=GREATEST(a.original_activity_at")) + assert(CURRENT_REQUIRED_TABLES.includes("server_poll_archives")) + assert.equal(migration.operations.filter(operation => operation.command === "INSERT").length, 1) + assert(!migration.operations.some(operation => operation.command === "UPDATE" || operation.command === "DELETE"), "No compaction or source rewrite runs during DDL") +}) +await test("0040 additive routing enrollment refuses legacy reserved/proposal collisions and never backfills authority", async () => { + const source = await readFile(new URL("../../drizzle/0040_mcp_routing_reservations.sql", import.meta.url), "utf8") + const migration = inventory.find(file => file.id === "0040_mcp_routing_reservations")! + assert.equal(migration.opaqueBodies.length, 1) + assert.equal(migration.operations[1].command, "LOCK"); assert.equal(migration.operations[2].command, "DO") + const guard = parseMigrationStatements(source)[2].tokens.find(token => token.kind === "dollar")!.value + assert(guard.includes("WHERE mode='reserved'")); assert(guard.includes("pg_catalog.pg_attribute")); assert(guard.includes("pg_catalog.pg_constraint")) + assert(source.includes("source_configuration_proof IS NOT NULL")); assert(source.includes("inverse_configuration_sealed IS NULL OR inverse_intent_sealed IS NOT NULL")) + assert.equal(migration.operations.filter(operation => operation.command === "INSERT").length, 1, "Only the atomic file marker is inserted") + assert(!migration.operations.some(operation => operation.command === "UPDATE" || operation.command === "DELETE")) + assert(CURRENT_REQUIRED_TABLES.includes("mcp_routing_reservations")); assert(CURRENT_REQUIRED_TABLES.includes("mcp_routing_reservation_slots")) +}) +await test("fresh storage enrollment locks and refuses history before DDL without backfill", async () => { + const migration = inventory.find(file => file.id === "0037_server_probe_storage_v2")! + const source = await readFile(new URL("../../drizzle/0037_server_probe_storage_v2.sql", import.meta.url), "utf8") + const statements = parseMigrationStatements(source) + assert.equal(migration.statementCount, 27) + assert.equal(migration.operations[1].command, "LOCK"); assert.equal(migration.operations[2].command, "DO") + const lock = statements[1].tokens.map(token => token.value).join(" ") + assert(lock.includes("ACCESS EXCLUSIVE MODE")) + const guard = statements[2].tokens.find(token => token.kind === "dollar")!.value + for (const table of ["server_probe_executions", "server_probe_execution_stages", "server_probe_execution_quarantines", "mcp_diagnostic_cancellations"]) { + assert(lock.includes(table.toUpperCase())); assert(guard.includes(`EXISTS(SELECT 1 FROM ${table})`)) + } + for (const table of ["server_probe_storage_usage", "server_probe_storage_admissions", "server_probe_execution_payloads"]) { + assert(guard.includes("'" + table + "'")); assert(CURRENT_REQUIRED_TABLES.includes(table)) + } + assert.equal(migration.opaqueBodies.filter(body => body.kind === "executed-do-body").length, 1) + assert.equal(migration.opaqueBodies.filter(body => body.kind === "stored-function-body").length, 10) + assert.deepEqual(migration.operations.filter(operation => operation.command === "INSERT").map(operation => operation.target), ["server_probe_storage_usage", "schema_migrations"]) + assert.deepEqual(migration.operations.filter(operation => operation.command === "UPDATE"), []) + const seed = statements.find(statement => statement.tokens[0].value === "INSERT" && statement.tokens[2].value === "SERVER_PROBE_STORAGE_USAGE")! + assert(!seed.tokens.some(token => token.kind === "word" && token.value === "SELECT")) + assert.deepEqual(seed.tokens.filter(token => token.kind === "number").map(token => token.value), ["0", "0", "0"]) +}) +await test("poll enrollment fences writers before inspecting unmarked schema and installs exact fixed guards", async () => { + const source = await readFile(new URL("../../drizzle/0038_server_poll_journal_and_writer_guards.sql", import.meta.url), "utf8") + const statements = parseMigrationStatements(source) + const migration = inventory.find(file => file.id === "0038_server_poll_journal_and_writer_guards")! + assert.equal(migration.operations[1].command, "DO") + const guard = statements[1].tokens.find(token => token.kind === "dollar")!.value + const lockAt = guard.indexOf("PERFORM pg_catalog.pg_advisory_xact_lock(8723110::bigint)") + assert(lockAt >= 0 && lockAt < guard.indexOf("IF EXISTS")) + for (const name of ["server_poll_executions", "server_poll_intents", "server_poll_acks", "server_poll_receipts", "server_poll_quarantines", "server_poll_journal_immutable", "server_partition_writer_guard"]) assert(guard.includes("'" + name + "'")) + const { SERVER_POLL_JOURNAL_DDL } = await import("../services/server-poll-journal.js") + const { SERVER_PARTITION_WRITER_DDL } = await import("../services/server-partition-maintenance-lease.js") + assert(source.includes(SERVER_POLL_JOURNAL_DDL.trim())) + assert(source.includes(SERVER_PARTITION_WRITER_DDL.trim())) + assert.equal((source.match(/ENABLE ALWAYS TRIGGER server_partition_writer_guard/g) ?? []).length, 12) + assert.deepEqual(migration.operations.filter(operation => operation.command === "INSERT").map(operation => operation.target), ["schema_migrations"]) + assert.equal(migration.opaqueBodies.length, 3) +}) +await test("poll storage enrollment refuses history before any change and preserves reviewed quota/GC bodies", async () => { + const source=await readFile(new URL("../../drizzle/0039_server_poll_storage.sql",import.meta.url),"utf8") + const proposal=await readFile(new URL("./proposals/server-poll-storage.sql",import.meta.url),"utf8") + const migration=inventory.find(file=>file.id==="0039_server_poll_storage")! + const statements=parseMigrationStatements(source),historical=parseMigrationStatements(proposal) + assert.equal(migration.statementCount,25);assert.equal(migration.opaqueBodies.length,6) + assert.equal(migration.operations[1].command,"LOCK");assert.equal(migration.operations[2].command,"DO") + const guard=statements[2].tokens.find(token=>token.kind==="dollar")!.value + for(const name of ["server_poll_executions","server_poll_intents","server_poll_acks","server_poll_receipts","server_poll_quarantines"])assert(guard.includes(`EXISTS(SELECT 1 FROM ${name})`)) + for(const name of ["server_poll_storage_admissions","server_poll_storage_usage"])assert(guard.includes("'"+name+"'")&&CURRENT_REQUIRED_TABLES.includes(name)) + const bodies=(list:ReturnType)=>list.filter(statement=>statement.tokens[0].value==="CREATE").flatMap(statement=>statement.tokens.filter(token=>token.kind==="dollar").map(token=>token.value)) + const comparableBodies=(list:ReturnType)=>bodies(list).map(body=>body.replace(/\r\n/g,"\n")) + assert.deepEqual(comparableBodies(statements),comparableBodies(historical));assert.equal(bodies(statements).length,5) + assert(bodies(statements)[0].includes("mcp_settings.id=1 FOR SHARE"),"Predicate itself fences current retention, including direct guarded deletes") + assert.deepEqual(migration.operations.filter(operation=>operation.command==="INSERT").map(operation=>operation.target),["server_poll_storage_usage","schema_migrations"]) + assert.equal(migration.operations.some(operation=>["UPDATE","DELETE","DROP"].includes(operation.command)),false) +}) +await test("comment-only CONCURRENTLY is not an executed nontransactional effect", () => { + const index = inventory.find(file => file.id === "0013_scheduler_retention_index")! + assert.equal(index.statementCount, 1); assert.equal(index.transaction, "owner-transaction") +}) +await test("nested comments, quoted identifiers, escaped strings and tagged bodies split correctly", () => { + const source = `/* BEGIN; /* COMMIT; */ still comment */ DO $tag$ BEGIN PERFORM ';COMMIT'; END; $tag$; +CREATE TABLE "semi;COMMIT"(value text DEFAULT 'it''s; BEGIN'); +INSERT INTO "semi;COMMIT" VALUES(E'back\\slash; END', $$literal;COMMIT;$$); -- BEGIN;` + const statements = parseMigrationStatements(source) + assert.equal(statements.length, 3); assert.equal(statements[0].tokens[1].kind, "dollar") + assert.equal(statements[1].tokens.find(token => token.kind === "identifier")?.value, "semi;COMMIT") + assert.equal(statements[2].tokens.filter(token => token.kind === "dollar").length, 1) +}) +await test("unterminated/unsupported source refuses rather than guessing", () => { + for (const source of ["DO $$ BEGIN;", "CREATE TABLE x(v text DEFAULT 'oops);", "/* never closes", 'CREATE TABLE "missing(v int);', "CREATE TABLE x(v int)", "\\i arbitrary.sql", "SELECT U&'unsupported';", "SELECT 1;\0"]) assert.throws(() => parseMigrationStatements(source), MigrationManifestError) +}) +await test("split commits, statements after COMMIT and unsupported controls refuse", () => { + for (const source of ["BEGIN; CREATE TABLE x(v int); COMMIT; CREATE TABLE y(v int);", "BEGIN; COMMIT; BEGIN; COMMIT;", "SAVEPOINT x; CREATE TABLE x(v int);", "START TRANSACTION; COMMIT;", "BEGIN; ROLLBACK;", "BEGIN WORK; COMMIT;"]) assert.throws(reviewed(source, { transaction: "self-managed-single-transaction" }), MigrationManifestError) +}) +await test("exact synthetic transaction contract and terminal selfmarker are checked", () => { + const source = "BEGIN; CREATE TABLE fixture(v int); INSERT INTO schema_migrations(id) VALUES('9000_fixture') ON CONFLICT(id) DO NOTHING; COMMIT;" + assert.equal(reviewed(source, { transaction: "self-managed-single-transaction", selfMarker: true })().selfMarker?.atomicWithAllFileEffects, true) + assert.throws(reviewed(source.replace("9000_fixture", "9000_foreign"), { transaction: "self-managed-single-transaction", selfMarker: true }), MigrationManifestError) + assert.throws(reviewed(source.replace("COMMIT;", "CREATE TABLE later(v int); COMMIT;"), { transaction: "self-managed-single-transaction", selfMarker: true }), MigrationManifestError) +}) +await test("opaque body requires its exact declared semantic review", () => { + const source = "DO $$ BEGIN EXECUTE 'DROP TABLE x'; END $$;" + assert.throws(reviewed(source), MigrationManifestError) + assert.equal(reviewed(source, { opaqueBodies: 1, bodyReview: ["Reviewed inert dynamic destructive fixture"] })().opaqueBodies.length, 1) + assert.throws(reviewed("DO 'BEGIN NULL; END';"), MigrationManifestError) +}) +await test("executed nontransactional/unclassified commands fail closed", () => { + for (const source of ["CREATE INDEX CONCURRENTLY ix ON t(x);", "SELECT setval('s',1);", "VACUUM x;", "ALTER SYSTEM SET x=1;", "CALL arbitrary();", "COPY x FROM PROGRAM 'x';"]) assert.throws(reviewed(source), MigrationManifestError) +}) +await test("raw byte drift including newline changes refuses pinned inventory", async () => { + const pin = MIGRATION_MANIFEST[0], bytes = await readFile(new URL("../../drizzle/" + pin.file, import.meta.url)) + assert.throws(() => analyzeMigration(pin.file, Buffer.concat([bytes, Buffer.from("\n")]), pin), MigrationManifestError) + assert.throws(() => analyzeMigration(pin.file, bytes, { ...pin, transaction: "self-managed-single-transaction" }), MigrationManifestError) +}) +await test("new/missing/duplicate SQL files require an explicit classified pin", () => { + const files = MIGRATION_MANIFEST.map(pin => pin.file) + validateMigrationFileSet([...files].reverse()) + for (const changed of [[...files, "0040_unclassified.sql"], files.slice(1), [...files, files[0]], [...files, "manual.sql"]]) assert.throws(() => validateMigrationFileSet(changed), MigrationManifestError) + assert(Object.isFrozen(MIGRATION_MANIFEST)); assert(Object.isFrozen(MIGRATION_MANIFEST[0].effects)) +}) +const empty: LegacyBaselineEvidence = { inventoryComplete: true, databaseGeneration: "owned-generation", unresolvedAttemptIds: [], markerTable: "absent", markerIds: [], applicationObjects: [] } +await test("absent markers never alone authorize fresh execution", () => { + assert.equal(assessLegacyMigrationBaseline(empty).state, "clean-database-candidate") + assert.equal(assessLegacyMigrationBaseline(empty).dispatchAuthorized, false) + assert.equal(assessLegacyMigrationBaseline({ ...empty, inventoryComplete: false }).state, "blocked") + assert.equal(assessLegacyMigrationBaseline({ ...empty, applicationObjects: [{ schema: "public", name: "servers", kind: "table" }] }).state, "blocked") + assert.equal(assessLegacyMigrationBaseline({ ...empty, unresolvedAttemptIds: ["uncertain0000"] }).state, "blocked") +}) +const complete: LegacyBaselineEvidence = { ...empty, markerTable: "present", markerIds: MIGRATION_MANIFEST.map(pin => pin.file.slice(0, -4)), applicationObjects: CURRENT_REQUIRED_TABLES.map(name => ({ schema: "public", name, kind: "table" })), verifiedSchema: { manifestDigest: migrationManifestDigest(), databaseGeneration: "owned-generation", exactCurrentSchema: true, observedSchemaDigest: "a".repeat(64) } } +await test("existing current schema needs exact separately supplied comparator evidence", () => { + assert.equal(assessLegacyMigrationBaseline(complete).state, "legacy-current-enrollment-candidate") + assert.equal(assessLegacyMigrationBaseline(complete).dispatchAuthorized, false) + for (const changed of [{ ...complete, verifiedSchema: undefined }, { ...complete, markerIds: complete.markerIds.slice(0, 1) }, { ...complete, applicationObjects: complete.applicationObjects.slice(1) }, { ...complete, markerIds: [...complete.markerIds, "0001_unknown"] }, { ...complete, verifiedSchema: { ...complete.verifiedSchema!, databaseGeneration: "restored-foreign" } }, { ...complete, verifiedSchema: { ...complete.verifiedSchema!, manifestDigest: "0".repeat(64) } }]) assert.equal(assessLegacyMigrationBaseline(changed).state, "blocked") +}) + + +await test("0043 adds a separate one-unit immutable forward ledger with an atomic marker", async () => { + const pin = inventory.find(file => file.id === "0043_mcp_routing_forward_charges")! + assert.equal(pin.opaqueBodies.length, 3); assert.equal(pin.selfMarker?.atomicWithAllFileEffects, true) + const source = await readFile(new URL("../../drizzle/0043_mcp_routing_forward_charges.sql", import.meta.url), "utf8") + assert(source.includes("CHECK(charge_units=1)")); assert(source.includes("source_operation_id uuid PRIMARY KEY")) + assert(!/REFERENCES|CASCADE|SECURITY DEFINER/.test(source.replace(/--[^\n]*/g, ""))) + assert(source.includes("BEFORE UPDATE OR DELETE")); assert(source.includes("BEFORE TRUNCATE")) +}) + +await test("0044 adds only a fixed manual inverse ledger and current-version intent fence in one atomic file", async () => { + const pin = inventory.find(file => file.id === "0044_mcp_routing_manual_inverse_charges")! + assert.equal(pin.transaction, "self-managed-single-transaction") + assert.equal(pin.opaqueBodies.length, 4); assert.equal(pin.selfMarker?.atomicWithAllFileEffects, true) + const source = await readFile(new URL("../../drizzle/0044_mcp_routing_manual_inverse_charges.sql", import.meta.url), "utf8") + const executable = source.replace(/--[^\n]*/g, "") + assert(!/REFERENCES|CASCADE|SECURITY DEFINER/.test(executable), "Logical charge identity must not acquire family lifetime FKs or extra authority") + assert(source.includes("CHECK(charge_units=1)")); assert(source.includes("UNIQUE(source_operation_id,source_step_index)")) + assert(source.includes("BEFORE UPDATE OR DELETE")); assert(source.includes("BEFORE TRUNCATE")) + assert(source.includes("pg_try_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))")) + assert(source.includes("p.operation_id='operations.recover' AND p.operation_version=7")) + assert(source.includes("sp.operation_id='routing.community.switch' AND sp.operation_version=7")) + assert(source.includes("manual inverse requires current exact running recovery")) + assert(source.includes("WHERE mode='manual'")); assert(CURRENT_REQUIRED_TABLES.includes("mcp_routing_manual_inverse_charges")) + assert.equal(pin.operations.filter(operation => operation.command === "INSERT").length, 1, "Only atomic file marker is inserted; no historical charge backfill") + assert(!pin.operations.some(operation => operation.command === "UPDATE" || operation.command === "DELETE")) +}) + +await test("0056 changes only live manual inverse recovery version guards", async () => { + const migration = inventory.find(file => file.id === "0056_mcp_manual_inverse_recovery_v8")! + assert.equal(migration.statementCount, 5) + assert.equal(migration.opaqueBodies.length, 2) + assert.equal(migration.selfMarker?.atomicWithAllFileEffects, true) + const oldSql = await readFile(new URL("../../drizzle/0044_mcp_routing_manual_inverse_charges.sql", import.meta.url), "utf8") + const newSql = await readFile(new URL("../../drizzle/0056_mcp_manual_inverse_recovery_v8.sql", import.meta.url), "utf8") + for (const name of ["mcp_routing_manual_inverse_charge_source", "mcp_scoped_recovery_manual_version"]) { + const definition = new RegExp(`CREATE(?: OR REPLACE)? FUNCTION public\\.${name}\\(\\)[\\s\\S]*?END \\$\\$;`) + const original = oldSql.match(definition)?.[0]?.replaceAll("\r\n", "\n") + assert.ok(original) + assert.equal(newSql.match(definition)?.[0]?.replaceAll("\r\n", "\n"), original.replace("CREATE FUNCTION", "CREATE OR REPLACE FUNCTION") + .replace("p.operation_id='operations.recover' AND p.operation_version=7", "p.operation_id='operations.recover' AND p.operation_version=8")) + } +}) + +console.log(`migration-manifest: ${cases} inert parser/inventory/baseline scenarios passed; ${inventory.length} exact files`) diff --git a/backend/src/db/migration-manifest.ts b/backend/src/db/migration-manifest.ts new file mode 100644 index 00000000..5b7c6884 --- /dev/null +++ b/backend/src/db/migration-manifest.ts @@ -0,0 +1,927 @@ +import { createHash } from "node:crypto" +import { readdir, readFile } from "node:fs/promises" +import { join } from "node:path" + +/** Inventory only: no database imports, SQL execution, rewrite or dispatcher. */ +export type MigrationTransaction = "owner-transaction" | "self-managed-single-transaction" +export interface MigrationReview { + file: string; sha256: string; transaction: MigrationTransaction; selfMarker: boolean + opaqueBodies: number; bodyReview: readonly string[]; effects: readonly string[] + publicTableReset?: true +} +export interface SqlToken { kind: "word" | "identifier" | "string" | "dollar" | "symbol" | "number"; value: string; line: number } +export interface MigrationStatement { line: number; endLine: number; tokens: SqlToken[] } +export class MigrationManifestError extends Error { readonly code = "MIGRATION_MANIFEST_REFUSED" } +const refuse = (message: string): never => { throw new MigrationManifestError(message) } +const hash = (value: Uint8Array | string) => createHash("sha256").update(value).digest("hex") + +/** PostgreSQL lexical subset needed by the pinned inventory. Dollar-quoted + * bodies/strings/quoted identifiers are opaque: semicolons and BEGIN inside + * them never become top-level statements. Unsupported/unterminated input + * refuses. This is not a general PL/pgSQL semantic evaluator. */ +export function parseMigrationStatements(source: string): MigrationStatement[] { + if (Buffer.byteLength(source) > 2097152 || source.includes("\0")) refuse("Migration source exceeds lexical bounds") + let index = 0, line = 1, count = 0, tokens: SqlToken[] = [] + const statements: MigrationStatement[] = [] + const advance = (length: number) => { line += (source.slice(index, index + length).match(/\n/g) ?? []).length; index += length } + const push = (kind: SqlToken["kind"], value: string, at: number) => { if (++count > 200000) refuse("Migration token bound exceeded"); tokens.push({ kind, value, line: at }) } + while (index < source.length) { + const tail = source.slice(index), char = source[index], at = line + if (/\s/.test(char)) { advance(1); continue } + if (tail.startsWith("--")) { const end = source.indexOf("\n", index); advance((end < 0 ? source.length : end) - index); continue } + if (tail.startsWith("/*")) { + let depth = 1; advance(2) + while (depth && index < source.length) { + if (source.startsWith("/*", index)) { depth++; advance(2) } + else if (source.startsWith("*/", index)) { depth--; advance(2) } + else advance(1) + } + if (depth) refuse("Unterminated migration block comment") + continue + } + if (char === ";") { + if (tokens.length) { statements.push({ line: tokens[0].line, endLine: line, tokens }); tokens = []; if (statements.length > 5000) refuse("Migration statement bound exceeded") } + advance(1); continue + } + const escapeString = /^[eE]'/.test(tail) + if (char === "'" || char === '"' || escapeString) { + const quote = char === '"' ? '"' : "'"; advance(escapeString ? 2 : 1) + let value = "", closed = false + while (index < source.length) { + if (escapeString && source[index] === "\\") { if (index + 1 >= source.length) refuse("Unterminated escape string"); value += source.slice(index, index + 2); advance(2) } + else if (source[index] === quote) { + if (source[index + 1] === quote) { value += quote; advance(2) } + else { advance(1); closed = true; break } + } else { value += source[index]; advance(1) } + } + if (!closed) refuse("Unterminated migration quote") + push(quote === '"' ? "identifier" : "string", value, at); continue + } + const dollar = /^\$(?:[A-Za-z_][A-Za-z_0-9]*)?\$/.exec(tail)?.[0] + if (dollar) { + const end = source.indexOf(dollar, index + dollar.length) + if (end < 0) refuse("Unterminated migration dollar body") + const body = source.slice(index + dollar.length, end) + if (Buffer.byteLength(body) > 1048576) refuse("Migration body bound exceeded") + push("dollar", body, at); advance(end + dollar.length - index); continue + } + if (/^[uU]&['"]/.test(tail) || char === "\\") refuse("Unsupported migration lexical construct") + const word = /^[A-Za-z_][A-Za-z_0-9$]*/.exec(tail)?.[0] + if (word) { push("word", word.toUpperCase(), at); advance(word.length); continue } + const number = /^\d+(?:\.\d+)?/.exec(tail)?.[0] + if (number) { push("number", number, at); advance(number.length); continue } + if (/[(),.=<>:+\-/%\[\]|!?^~*]/.test(char)) { push("symbol", char, at); advance(1); continue } + refuse("Unsupported migration token at line " + line) + } + if (tokens.length) refuse("Migration has an unterminated top-level statement") + return statements +} +const words = (statement: MigrationStatement) => statement.tokens.filter(token => token.kind === "word").map(token => token.value) +function target(statement: MigrationStatement) { + const tokens = statement.tokens + let position = tokens.findIndex(token => token.kind === "word" && ["TABLE", "INDEX", "FUNCTION", "TRIGGER", "INTO"].includes(token.value)) + 1 + if (!position) return undefined + while (tokens[position]?.kind === "word" && ["IF", "NOT", "EXISTS"].includes(tokens[position].value)) position++ + const first = tokens[position] + if (!first || !["word", "identifier"].includes(first.kind)) return undefined + const name = first.kind === "word" ? first.value.toLowerCase() : first.value + if (tokens[position + 1]?.value === ".") { + const second = tokens[position + 2] + if (!second || !["word", "identifier"].includes(second.kind)) refuse("Invalid qualified migration target") + return name + "." + (second.kind === "word" ? second.value.toLowerCase() : second.value) + } + return name +} +export function analyzeMigration(file: string, bytes: Uint8Array, review: MigrationReview) { + if (!/^\d{4}_[a-z_0-9]+\.sql$/.test(file) || review.file !== file || !/^[a-f0-9]{64}$/.test(review.sha256) || hash(bytes) !== review.sha256) refuse("Unclassified or changed migration file: " + file) + const source = new TextDecoder("utf-8", { fatal: true }).decode(bytes) + const statements = parseMigrationStatements(source) + if (!statements.length) refuse("Empty migration file") + const controls = statements.flatMap((statement, index) => { + const first = words(statement)[0] + if (!["BEGIN", "START", "COMMIT", "END", "ROLLBACK", "SAVEPOINT", "RELEASE", "PREPARE"].includes(first)) return [] + if (statement.tokens.length !== 1 || !["BEGIN", "COMMIT"].includes(first)) refuse("Unsupported migration transaction boundary") + return [{ index, kind: first, line: statement.line }] + }) + const transaction: MigrationTransaction = controls.length ? "self-managed-single-transaction" : "owner-transaction" + if (controls.length && (controls.length !== 2 || controls[0].index !== 0 || controls[0].kind !== "BEGIN" || controls[1].index !== statements.length - 1 || controls[1].kind !== "COMMIT")) refuse("Migration has split or nonterminal transaction boundaries") + if (transaction !== review.transaction) refuse("Migration transaction contract changed") + const opaque = statements.flatMap((statement, index) => { + const head = words(statement).slice(0, 4).join(" "), isDo = head.startsWith("DO"), isFunction = /^CREATE (?:OR REPLACE )?FUNCTION\b/.test(head) + const bodies = statement.tokens.filter(token => token.kind === "dollar") + if ((isDo || isFunction) && bodies.length !== 1) refuse("Executable/stored migration body requires an explicitly reviewed dollar body") + return bodies.map(token => ({ statement: index, line: token.line, sha256: hash(token.value), kind: isDo ? "executed-do-body" : isFunction ? "stored-function-body" : "dollar-literal" })) + }) + if (opaque.length !== review.opaqueBodies || review.bodyReview.length !== opaque.length) refuse("Migration opaque body lacks exact semantic review") + const selfMarkers = statements.flatMap((statement, index) => { + if (words(statement)[0] !== "INSERT" || !["schema_migrations", "public.schema_migrations"].includes(target(statement) ?? "")) return [] + const tokens = statement.tokens, literal = tokens.filter(token => token.kind === "string") + const shape = tokens.map(token => token.kind === "string" ? "?" : token.value).join(" ") + if (!/^INSERT INTO (?:PUBLIC \. )?SCHEMA_MIGRATIONS \( ID \) VALUES \( \? \) ON CONFLICT \( ID \) DO NOTHING$/.test(shape) || literal.length !== 1 || literal[0].value !== file.slice(0, -4)) refuse("Migration self-marker is not the exact declared id INSERT") + return [{ statement: index, line: statement.line, id: literal[0].value }] + }) + if (selfMarkers.length !== Number(review.selfMarker) || selfMarkers.length && (transaction !== "self-managed-single-transaction" || selfMarkers[0].statement !== statements.length - 2)) refuse("Migration self-marker is not atomic with all file effects") + for (const statement of statements) { + const tokens = words(statement), command = tokens[0] + if (!["BEGIN", "COMMIT", "CREATE", "ALTER", "INSERT", "DROP", "DO", "LOCK"].includes(command)) refuse("Unclassified migration command: " + command) + if (tokens.includes("CONCURRENTLY") || tokens.includes("SETVAL") || tokens.includes("NEXTVAL") || tokens.slice(0, 2).join(" ") === "ALTER SYSTEM") refuse("Migration contains an unclassified nontransactional effect") + } + return { + file, id: file.slice(0, -4), sha256: review.sha256, byteLength: bytes.length, transaction, + topLevelTransactionControls: controls, statementCount: statements.length, + selfMarker: selfMarkers[0] ? { ...selfMarkers[0], atomicWithAllFileEffects: true } : null, + currentOuterMarkerGap: !review.selfMarker, + futureDispatcher: transaction === "owner-transaction" ? "may-run-verbatim-in-owned-transaction-with-marker-and-receipt-after-durable-intent" : review.selfMarker ? "verbatim-self-managed-file-with-audited-atomic-legacy-marker-and-separate-durable-receipt" : "verbatim-self-managed-file-needs-durable-pre-intent-and-post-file-terminal-receipt-gap-refusal", + effects: [...review.effects], publicTableReset: review.publicTableReset === true, + opaqueBodies: opaque.map((body, index) => ({ ...body, review: review.bodyReview[index] })), + nontransactionalExecution: "none-observed-in-exact-reviewed-bytes; identity definitions and nondeterministic defaults are separately described", + operations: statements.map((statement, index) => ({ statement: index, line: statement.line, endLine: statement.endLine, command: words(statement)[0], objectKind: words(statement).find(word => ["TABLE", "INDEX", "FUNCTION", "TRIGGER"].includes(word)) ?? (words(statement)[0] === "INSERT" ? "TABLE" : null), target: target(statement) ?? null })), + } +} + +// Exact byte pins and semantic reviews. Additions or any byte change require +// an explicit reviewed entry; the inventory never auto-enrolls discovered SQL. +const classifiedMigrationPins: readonly MigrationReview[] = [ + { + "file": "0000_postgresql.sql", + "sha256": "eb6ce1449add311611634845aa6608327821ca2cbe33c1e7b214dde37d85027e", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Initial public application tables, indexes, FK cascades, identity sequence definitions and settings/data_migration seeds; no marker INSERT in file; generated identity definitions are not evidence of nextval execution." + ] + }, + { + "file": "0002_compact_schema.sql", + "sha256": "0ffcbb8d54dd386a1d709ee03a2e556f4caee102c1b9ef7b0faeef6019056deb", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 1, + "bodyReview": [ + "Executed DO: catalog loop dynamically DROP TABLE IF EXISTS public.%I CASCADE, excludes ONLY public schema_migrations; data-destructive after commit; no top-level transaction statements hidden in this body." + ], + "effects": [ + "DESTRUCTIVE executed DO enumerates public pg_tables and DROP TABLE ... CASCADE for every table except schema_migrations; destroys application data and any public recovery tables.", + "Recreates compact public tables/indexes/FK cascades and identity definitions; initializes settings/data_migration, which removes the previous import-completion state; no file marker." + ], + "publicTableReset": true + }, + { + "file": "0003_backup_s3.sql", + "sha256": "59cda0660a0f487330058735ad2ee75021ca236480c28a81a2cf32bb4cd77be7", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 1, + "bodyReview": [ + "Executed DO: catalog existence check and conditional ALTER TABLE backup_entries ADD CONSTRAINT; no transaction control, external call or nontransactional sequence advance." + ], + "effects": [ + "Creates/seeds backup_storage_settings; adds backup_entries storage/S3 columns; executed DO conditionally adds storage CHECK constraint." + ] + }, + { + "file": "0004_geoip_settings.sql", + "sha256": "a0c6e7b918df2c43dd4dd2f76aa0fea3c24b6c6e304390708d9c601521b6343c", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates/seeds geoip_settings; enabled/update defaults are schema metadata, no downloader/network execution." + ] + }, + { + "file": "0005_flow_facts.sql", + "sha256": "26254ce080c9bc1b8d33431c4c00ed7f8601d49c035a3bb0c0d9cf00d268c8a1", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates partitioned flow_hour_facts and flow_daily_facts parents with server FK cascades; no leaf creation or retention execution." + ] + }, + { + "file": "0006_config_revisions.sql", + "sha256": "aed6b7abaeb8b895c019b32ee5badf5c68fb457d50a4187fbb5c58a8c8728b8f", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates config_revisions with server FK cascade and server/section/time index; retention is comment only." + ] + }, + { + "file": "0007_flow_buckets_nat.sql", + "sha256": "2cab88bea14f338b72127c3c7e39ab7d88baa2d11a728389724ba5574ecdecdd", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds four NAT address/port columns to flow_buckets." + ] + }, + { + "file": "0008_alert_outbox_outcomes.sql", + "sha256": "d16e97a48da5d957b9797309bc0e7b48a826ef861d6734cf4da1ee756539b825", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Drops/replaces alert_outbox status CHECK to permit processing/unknown states; no historical row UPDATE." + ] + }, + { + "file": "0009_alert_outbox_resolution.sql", + "sha256": "e3d66a133862e443320d83213899e96d7de84e1c8144962615895176c8c62b11", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds alert_outbox resolution_version and nonnegative CHECK." + ] + }, + { + "file": "0010_ping_measurement_known.sql", + "sha256": "4cc2f13a489f3b300d238a205a41b75d495a4aa26e3d1468f26b1f682c4a94aa", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds uptime_probe_samples measurement_known default true and observation_error; default backfill changes existing row interpretation transactionally." + ] + }, + { + "file": "0011_traffic_collection_state.sql", + "sha256": "83e3b6958ea289e594cc68b0646c894c703104ca95a9c73e203886950fc94b9b", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates traffic_collection_state with server FK cascade." + ] + }, + { + "file": "0012_traffic_counter_identity.sql", + "sha256": "491bb39fe85554d16881696a10d5d5732f60ecec1be92a8dd7b9584ce5a8a9b8", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds traffic_samples native_counter_identity." + ] + }, + { + "file": "0013_scheduler_retention_index.sql", + "sha256": "8ae429ba11fa02163bb3a12f25fc8fdd9708b98badf4a37dfc3a5124740ced0e", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates ordinary scheduler_runs finished_at index; CONCURRENTLY appears ONLY in comments/deployment advice, not executed SQL." + ] + }, + { + "file": "0014_router_logs.sql", + "sha256": "6f82eeb26101bf20574a0c6cb1f5778d6e8151290f0733e7896c5b6e26fe985f", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates/seeds router_log_settings, collection state, identity-backed log entries and six log indexes; log history deliberately lacks server FK." + ] + }, + { + "file": "0015_mcp.sql", + "sha256": "13c279fe2ecbabeef1b1c588fbf58a374141d931b45914aac5803efc496496b7", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates initial public MCP settings/connections/credentials/audit/requests/protections/plans/operations/steps/budgets/read leases/grants/secrets/artifacts/OAuth/routing profiles/drift tables and indexes; settings seed; identity definition on audit; no file marker." + ] + }, + { + "file": "0016_mcp_routing_observations.sql", + "sha256": "7b35e4ca6522a7d0e086c518a9f84bfb7bf6eabc3d6ce3a6f537117cc49b6e8e", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates MCP routing observations and leg index." + ] + }, + { + "file": "0017_mcp_controller.sql", + "sha256": "bb48c740b4770d979babc2684d9ad7d9e7577083197a74fe00c47e3ce59c1673", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds routing profile credential/generation/policy-version fields and enabled/open-drift indexes." + ] + }, + { + "file": "0018_mcp_owner_validation.sql", + "sha256": "e52f618f7dfd2085b888032367d0d2b56a4004efa4fd6a0561dc3fb1e5f83a61", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds owner_valid_until default now to connections; transactionally backfills existing owner-validity metadata." + ] + }, + { + "file": "0019_mcp_recovery.sql", + "sha256": "820899500c0d00c67a9b34dfd6769cd146c7688e39014d8b4b4cc7bfb6180bb0", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates encrypted operation checkpoints." + ] + }, + { + "file": "0020_mcp_management.sql", + "sha256": "cdd5c527965793060818a4eb600443376470f315e621e451d6ef882840b140ab", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates management channels with server FK cascade." + ] + }, + { + "file": "0021_mcp_path_checks.sql", + "sha256": "8f7c67a99848ecc6ce48c2573ac95f44bd9222c4c74da2aa3283598dff304d35", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates path checks and latest-check index." + ] + }, + { + "file": "0022_mcp_recovery_intents.sql", + "sha256": "a40c06e73a552400ee903848f9149923d85d2176e211c6c91681fff61ffc2b04", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates recovery intents tied to operation/profile." + ] + }, + { + "file": "0023_mcp_protocol_limits.sql", + "sha256": "0feac9580a7f6af649b664cfc129135c5c6a682bc50f0534550ed65f3b82fb10", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates unauthorized request buckets." + ] + }, + { + "file": "0024_user_interface_types.sql", + "sha256": "dd381cb3a33a41e9dc1ca351c632646f9dce7f018b6f40e93e0c393437c9e7b5", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Drops/replaces interface type CHECK, adding ipsec." + ] + }, + { + "file": "0025_mcp_portal_revocation.sql", + "sha256": "c47b01c4976f39f13d77c67606c6681c17fb1c83b0036c9157134ccf322f6f37", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates portal owner revocations/events and event-time index." + ] + }, + { + "file": "0026_mcp_owner_claims_issued.sql", + "sha256": "5569287afd46676a9e8d738dcce2a143c7125b329ad8d892d2f59719b221e815", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds owner_claims_issued_at default epoch; transactionally backfills legacy claim timestamp." + ] + }, + { + "file": "0027_mcp_readiness.sql", + "sha256": "c90b16fcd376bac8c907be715b8e2d4ecd4a7171e624b9e93f884dd5f12ac159", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates handshake cooldown, credential-expiry/readiness and grant-credential indexes." + ] + }, + { + "file": "0028_mcp_artifact_scope.sql", + "sha256": "ad76f4f4c981d6cc3dfabdaf31eddf1f65d4ba3444940e6ad6ad5b7f5bc71ac4", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds artifact access_scope and expiry index." + ] + }, + { + "file": "0029_mcp_routing_sources.sql", + "sha256": "4e9f739407a6a6d969c746485f90de3dc0f62c09292f7ccea425697fe6317c5c", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 2, + "bodyReview": [ + "Stored PL/pgSQL trigger function: assigns gen_random_uuid to NEW epoch only when endpoint fields change, otherwise preserves OLD epoch; RETURN NEW; body is not executed by CREATE.", + "Executed DO: conditionally adds mcp_routing_observation_source_proof CHECK; no transaction control or external effects." + ], + "effects": [ + "Adds server routing_endpoint_epoch with gen_random_uuid default/backfill; creates/replaces epoch trigger function, drops/recreates UPDATE trigger, adds observation source_proof and guarded CHECK.", + "Stored trigger function generates fresh endpoint UUID on future endpoint-field changes; function definition is not top-level execution of its body; preserves unsigned historical observations." + ] + }, + { + "file": "0030_mcp_scoped_configuration_checkpoint.sql", + "sha256": "4f99d83cd803be616e6512c39227eaa63e24e5335e23994af059ce8c6f10c148", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds sealed scoped configuration checkpoint/proof fields." + ] + }, + { + "file": "0031_mcp_scoped_recovery.sql", + "sha256": "3d6380a57f18b940168f9bf584586ccbe99de0b07293e6dae033ea7071088a3f", + "transaction": "self-managed-single-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates scoped recovery intents and consistency checks." + ] + }, + { + "file": "0032_flow_write_receipts.sql", + "sha256": "ca89054c375261f847d1f4a6e87ccf8f83f1f7279f957b89b0ae447b6a468020", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates immutable-purpose flow_write_receipts schema; no pruning or receipt insertion in migration." + ] + }, + { + "file": "0033_flow_settings_intents.sql", + "sha256": "a341295c9a6f4ab09d62dbc7b30c4b0e6c947845a6ceac36bb581ff44964f748", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates flow_settings_intents claim schema and seeds traffic_flow_settings." + ] + }, + { + "file": "0034_vrf_production_tickets.sql", + "sha256": "b5317f326f54e46e3efaf4b1969673d33b6dd2835c12ca61fa7d73abbc9b3d10", + "transaction": "owner-transaction", + "selfMarker": false, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Creates precondition-only VRF production tickets and bounded validity/state CHECKs; no executor capability." + ] + }, + { + "file": "0035_server_probe_journal.sql", + "sha256": "ccc2a61b8329b083c6a5aac4d68b7dca3cf7cc715e702912a4bd1c15de36706d", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 3, + "bodyReview": [ + "Stored trigger: reject DELETE/terminal identity changes or quarantined execution, RETURN NEW.", + "Stored trigger: lock execution row, reject DELETE/noncontiguous stages/invalid terminal change, RETURN NEW.", + "Stored trigger: always reject quarantine mutation; definitions execute no journal DML." + ], + "effects": [ + "Creates probe executions/stages/quarantines with three stored immutability functions/triggers; one exact file-id marker is penultimate before COMMIT.", + "Trigger bodies reject future journal mutations but do not execute UPDATE/DELETE during migration; no existing journal history deletion." + ] + }, + { + "file": "0036_diagnostic_cancellations.sql", + "sha256": "3a9058ac307cf5b63d35f1eec081a5442baa381d29dd9bc6c1df4dded625f007", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Stored trigger function always raises immutability exception; not an executable migration transaction BEGIN." + ], + "effects": [ + "Creates cancellation requests with stored rejecting immutability function/trigger; exact file-id marker is penultimate before COMMIT; no cancellation request inserted." + ] + }, + { + "file": "0037_server_probe_storage_v2.sql", + "sha256": "b691f5748362e13feeef58b53734feb0ca3e7cc907fc7a803f103f3ce97f0748", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 11, + "bodyReview": [ + "Executed DO under ACCESS EXCLUSIVE locks refuses any legacy anchor/stage/quarantine/cancellation, any existing named v2 relation in any namespace, or unavailable retention; no backfill or target data write.", + "Stored GC predicate requires opt-in transaction GUC, serialized storage advisory lock, elapsed original authority/retention/safety, actual terminal MCP source and exact owner/connection/cancellation mapping; no DELETE is executed.", + "Stored admission trigger serializes storage; fresh admission must carry current authority or cancellation; immutable scope and irreversible consume/purge/cancel transitions; DELETE only through lifetime predicate.", + "Stored anchor trigger locks matching consumed admission and rejects expired, cancelled, purged or mismatched owner/execution/binding before INSERT/UPDATE.", + "Replacement execution trigger preserves immutable identities and terminal receipts, rejects quarantined completion, and permits DELETE only through lifetime predicate.", + "Stored stage trigger requires a live consumed matching admission before INSERT/UPDATE under storage advisory lock.", + "Stored quarantine trigger requires a committed matching admission under storage advisory lock before INSERT.", + "Stored payload trigger rejects UPDATE; INSERT requires live admission, exact terminal identity/proof and no quarantine; DELETE requires prior payload purge or full lifetime predicate.", + "Replacement stage trigger preserves serial contiguous execution and immutable received state; DELETE only through lifetime predicate.", + "Replacement quarantine trigger rejects mutation except DELETE independently admitted by lifetime predicate.", + "Replacement cancellation trigger rejects mutation except DELETE whose exact operation/connection/owner maps to admission and satisfies lifetime predicate." + ], + "effects": [ + "Atomically locks all four legacy diagnostic tables and refuses any history or unmarked v2 relation before schema DDL; fresh-only, never a historical source-generation/backfill authorization.", + "Creates usage/admission/payload tables, one owner-age index and guarded storage functions/triggers; seeds only zero global usage, with no copied payload, admission, cancellation or owner usage.", + "Removes inline ciphertext only from the locked empty anchor table and installs separated-payload terminal CHECK; no historical data transformation; exact file marker is penultimate before COMMIT.", + "Stored GC definitions schedule nothing and execute no cleanup during migration; the GUC alone is insufficient to authorize future DELETE." + ] + }, + { + "file": "0038_server_poll_journal_and_writer_guards.sql", + "sha256": "2db0bb00f561dd01ace54066d6649478e06fa26e7e417db113d0801dcdac9fa2", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 3, + "bodyReview": [ + "Executed DO first owns exclusive transaction advisory key8723110, then refuses any preexisting unmarked poll relation, journal/writer function or writer trigger before schema installation; no user rows copied or inferred.", + "Stored immutable journal trigger always raises on UPDATE/DELETE; no mutation executes during function creation.", + "Stored partition writer row trigger acquires shared transaction advisory key8723110 and returns OLD for DELETE or NEW otherwise; not a substitute for pre-relation writer admission." + ], + "effects": [ + "Owns exclusive transaction advisory writer exclusion before target DDL; installs five append-only poll tables and five immutability triggers without inserting poll data.", + "Installs ALWAYS row guards on every pinned partition root, inherited by descendants; no ordinary data or partition deletion, exact atomic file marker immediately precedes COMMIT. Mixed unadopted replicas require deployment drain." + ] + }, + { + "file": "0039_server_poll_storage.sql", + "sha256": "ce3957c2c0196114e85856c93db63fc6e31b61a4866486bc685e017477023f27", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 6, + "bodyReview": [ + "Executed DO refuses any nonempty prior poll journal, missing authoritative retention, or unmarked storage table/function/trigger before schema changes; no historical rows copied or inferred.", + "Stored GC predicate owns the global storage advisory lock and locks the actual current retention row and requires explicit GC mode, original expiry and persisted/current retention after latest activity, with exact terminal MCP poll operation/connection/owner proof; no deletion occurs in the predicate.", + "Stored admission guard atomically charges fixed global/owner quotas on fresh identity; only one-token consume and monotonic actual-row metering are allowed; guarded expired all-child-free deletion atomically refunds both counters.", + "Stored row guard requires matching charged admission, consumed live identity and bounded private-plan stages; quarantine uses matching original charged scope, and server clock normalizes all journal activity.", + "Stored AFTER INSERT meter charges only actual inserted journal rows, bounded to 8MiB and the private stage-row budget; skipped ON CONFLICT rows never charge.", + "Stored replacement immutability guard continues to reject every UPDATE; DELETE requires the independently verified complete original lifetime and terminal source, not merely the GC session flag." + ], + "effects": [ + "Locks five prior journal tables and installs two storage tables, bounded fixed reservation triggers and expiry guards in one transaction; only the global zero-usage seed is inserted.", + "No automatic GC/worker/public route activation or backfill is granted; exact file marker immediately precedes COMMIT. Nonempty intermediate installations fail without data loss." + ] + } + ,{ + "file": "0040_mcp_routing_reservations.sql", + "sha256": "7fd6533442a6756065685c1905cf71af333c284ca4f96e571284602d5e0852ef", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Executed preflight under exclusive shared-slot lock refuses every legacy reserved row and unmarked proposal relation, column or constraint; no evidence, debit or authority is reconstructed." + ], + "effects": [ + "Adds two empty reservation ledgers and same-source slot columns and shape constraints, preserving all existing manual ciphertext and proofs; no capabilities, backfill, controller or public dispatch activation.", + "Existing legacy reserved data fails before DDL; atomic file marker immediately precedes COMMIT." + ] + } + ,{ + "file": "0041_server_poll_terminal_archives.sql", + "sha256": "fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 13, + "bodyReview": [ + "Executed preflight owns the existing poll storage advisory lock and refuses unmarked archive relations, fields, functions or triggers before schema changes.", + "Executed locked preflight checks the entire nonempty previous full-admission ledger and exact original accounting CHECK names/definitions before replacing only those two constraints; no old receipt, ciphertext, proof or identity is rewritten or inferred.", + "Stored eligibility predicate locks the admission, nonblockingly acquires real operation/execution/receipt advisory keys and NOWAIT source rows, and requires approved applied error-free MCP source, one verified global step, original plan/approval/admission time ordering, exact receipt mapping, all four confirmed native stages and complete contiguous confirmed publications recorded before original expiry, no quarantine and elapsed original expiry/activity plus safety; final generic publication may finish later without renewing authority.", + "Stored source producer returns the complete raw admission, every journal row and exact terminal operation/plan/step in stable order only after eligibility; retains all fields and bounds decoded JSON to16MiB; no archive is produced during migration.", + "Stored aggregate predicate compares global and every owner heavy/compact/archive-byte charges with all admission rows and archive presence; no usage certificate or refunded free slot is fabricated.", + "Stored archive INSERT guard requires a one-shot compaction claim from a distinct prior SQL transaction and recomputes exact source-text SHA256 and length under ownership locks, validates fixed encoded gzip/encryption bounds and original activity/expiry, and stamps current SQL transaction and time; private archive and historical MAC authentication remains the fixed application producer/reader responsibility.", + "Replacement journal immutability function permits DELETE only for a current-transaction immutable archive transition or independently admitted original-lifetime GC; no session flag grants compaction.", + "Replacement admission guard permits one independently durable null-to-UUID compaction claim only for an eligible verified full source, with server-stamped time/xid8 and every other column unchanged; caller cannot supply the claim xid and claim cannot reset, retimestamp or change during metering/compaction. It preserves original identities and used counters, atomically charges heavy or compact reservations, refuses replay/reopen and exact-ledger drift, and refunds only original-lifetime child-free GC; old byte/row caps remain unchanged.", + "Stored AFTER archive INSERT removes every old stage/receipt/anchor and transitions the matching admission with exact two-row and ciphertext-plus32768 charge; no data is compacted during migration.", + "Replacement GC predicate preserves persisted/current retention and original activity/expiry for both layouts; terminal source locks are nonblocking and never lock current connection authority; unknown full records retain their existing full-lifetime GC path.", + "Stored archive immutability function rejects every UPDATE and permits DELETE only under the independent full original lifetime GC predicate.", + "Stored deferred final guard checks no old journal children survive compaction and the complete final ledger/archive shape; partial deletion, orphan archive or incomplete charge rolls back the whole transaction.", + "Stored early row guard rejects every late insertion, including quarantine, against a compacted tombstone; no source or claim is reopened." + ], + "effects": [ + "Additively preserves nonempty0040 full poll admissions and journal bytes while adding one immutable archive table, full/compact state and exact aggregate accounting under existing fixed owner/global byte and row caps.", + "No archive, compaction, historical backfill, current authority renewal, worker schedule, native request or production public activation occurs; only the atomic file marker is inserted immediately before COMMIT." + ] + } + ,{ + "file": "0042_server_poll_settings_epoch.sql", + "sha256": "b0bcfe9afffda346c9f2fd2ba867ce1708716af0823735583a6bc9217c72de60", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Stored fixed-target invoker trigger stamps a fresh UUID on INSERT and every complete configuration change; only last_collected_at,last_duration_ms,last_error,updated_at and the epoch itself are excluded from UPDATE comparison. Caller epochs are ignored, unknown future fields remain significant and ENABLE ALWAYS enforces replica-mode writes. No native work or capability is issued." + ], + "effects": [ + "Locks the three settings tables and additively backfills one non-null UUID column while preserving every existing field; repeated or colliding unmarked DDL refuses atomically.", + "Adds one pinned fixed-search-path function and three always row triggers. No journal source, plan, receipt, quota, runtime flag or old migration is rewritten; delayed approval remains inactive until separately integrated." + ] + }, + { + "file": "0043_mcp_routing_forward_charges.sql", + "sha256": "ef556e6d83dfad47bee7476b9aed0e83127e8dc15649048a6d77b2df079ed84d", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 3, + "bodyReview": [ + "Executed collision preflight refuses an existing unmarked forward-charge table; all DDL and the exact file marker share one transaction.", + "Stored invoker source trigger checks actual pending operation, executing v7 routing plan, profile and connection identity at INSERT. It is consistency only, never a permission or MAC verifier.", + "Stored invoker immutable guard refuses UPDATE/TRUNCATE and recent or unresolved DELETE. Retention uses locked current settings and SQL clock; logical historical identities intentionally have no lifetime FK or cascade." + ], + "effects": [ + "Adds one immutable one-unit persistent-forward charge table, two bounded-read indexes and three triggers; existing0040 two-slot reservations and0031 inverse accounting remain unchanged.", + "No backfill, timestamp aging, operation replay, new grants, worker activation, source enrollment or READY occurs. Historical issuer42 artifacts do not attest this schema." + ] + }, + { + "file": "0044_mcp_routing_manual_inverse_charges.sql", + "sha256": "37b88d1679be9bb47f16ccc8cae7373f20f9c05e7a8f662c3775f18d78bb5613", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 4, + "bodyReview": [ + "Collision preflight refuses an unmarked manual-inverse table; additive DDL and exact marker commit atomically.", + "Invoker source trigger checks actual pending recover7/executing plan, source switch7, profile version and owner/connection; it does not authenticate MAC or confer permission.", + "Invoker immutable parent trigger refuses UPDATE/TRUNCATE and retained/current-hour or pending/running DELETE; no family lifetime FK or cascade.", + "Invoker manual-intent INSERT fence acquires the existing global admission advisory key nonblockingly and requires exact running recover7/generic intent; old6 fresh claims refuse. Already-claimed old writers still require drained cutover." + ], + "effects": [ + "Adds one one-unit immutable manual inverse charge table, two charge indexes, one manual-history page index, and four triggers. Existing43 bytes, reserved2 and generic debit ownership remain unchanged.", + "No inferred/backfilled identity, legacy timestamp debit, permission change, source enrollment, public controller activation or READY. Full RT3/all-writer/family-deletion proof remains open." + ] + }, + { + "file": "0045_mcp_operation_replay_tombstones.sql", + "sha256": "8edf5b074c7eef49dad7835c44ac2539df4347f19eb9dd9b2aa7005bafaadbc0", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds one empty replay identity table with plan/operation uniqueness and connection-scoped idempotency digest uniqueness.", + "Table creation refuses an unmarked collision; the exact marker commits with the schema. No backfill, operation deletion, or worker activation." + ] + }, + { + "file": "0046_server_poll_operation_retirement.sql", + "sha256": "75d7956ea8134b01bcc88da083b77eb52f6e1ad70ad854918ed3380cb0e8cf88", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 4, + "bodyReview": [ + "Archive guard permits only fixed receipt-scoped V1 to V2 updates with matching approved terminal operation, tombstone and SQL 180-day cutoff.", + "Deferred guard independently requires the tombstone, physically absent operation and plan, and exact ledger accounting.", + "Admission guard retains existing paths and applies exact V2 archive-byte delta to both owners without changing row charges or lifetime fields.", + "Existing GC retains its authority/activity cutoff and admits retired identity only with exact poll tombstone and compatible V2 archive when present." + ], + "effects": [ + "Extends the same archive table to a bounded version 2, changes three guards and adds one deferred retirement guard.", + "No automatic conversion, worker activation, operation deletion or retention-window adjustment in migration." + ] + }, + { + "file": "0047_server_poll_retirement_claims.sql", + "sha256": "8c01ab14d241f715af021e8a7d7f5bb273b450b32f566140359260282524e01d", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Protects exact current-transaction V1 claim insertion and permits deletion only as the owning admission is cascaded by GC.", + "Requires exact receipt, V1 archive proof and claim token already committed in another transaction before V2 update." + ], + "effects": [ + "Adds one durable one-shot poll retirement claim per admission, cascading only with that admission's existing guarded GC.", + "No migration-time conversion, backfill, dispatch, or quota change." + ] + }, + { + "file": "0048_mcp_ordinary_retirement_claims.sql", + "sha256": "5bc3598c9f5d09611e98d05345accda6d39b199a583bba789b97a24140fafbfd", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Makes ordinary claims immutable and allows insert only for the exact linked settled and aged logs.settings.update v1 operation." + ], + "effects": [ + "Adds one durable ordinary-operation claim with an authenticated source digest. No migration-time data purge or backfill." + ] + }, + { + "file": "0049_mcp_ordinary_telemetry_retirement.sql", + "sha256": "68993e924d20db6b08973846a0da367fe89fe2948c414c810e9b171dbbae6761", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Extends the same immutable ordinary claim guard to four explicitly versioned settled telemetry settings families." + ], + "effects": [ + "Replaces only the ordinary claim guard function. No backfill, data purge, or changes to poll or diagnostic storage." + ] + }, + { + "file": "0050_server_probe_retired_sources.sql", + "sha256": "0468ac7e53638b532557273f55d065ff4cde092d8b08073f5860bab35263aead", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Refuses update, delete, and truncate of a signed retired diagnostic source pending a separately reviewed lifecycle." + ], + "effects": [ + "Creates empty bounded storage linking exact diagnostic admission and replay tombstone IDs. No backfill, retirement, or GC activation." + ] + }, + { + "file": "0051_server_probe_retirement_claims.sql", + "sha256": "b38876b5da44cf14e20a9df0baaa233ea4de81e471b38dddd370e149c8116243", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Restricts diagnostic claims to their original settled 180-day source and makes each claim permanent.", + "Restricts archival insertion to the exact committed claim and current original source and tombstone identity." + ], + "effects": [ + "Creates empty diagnostic retirement claims and guards the archive insert. No conversion, backfill, or GC activation." + ] + }, + { + "file": "0052_server_probe_retired_gc.sql", + "sha256": "ea56a1f1bdabfce606bdfec56b6a7b31d45b0187fbea4c900bc656ae320f2fdd", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 3, + "bodyReview": [ + "Validates the exact retired diagnostic identity and the later of current and persisted retention plus the original safety window.", + "Only permits archive removal during an exact guarded admission cascade after journal removal.", + "Adds the exact retired GC branch while retaining the live admission eligibility predicate." + ], + "effects": [ + "Changes only the retired source admission FK to guarded cascade and adds an explicit exact GC path. No collection runs during migration." + ] + }, + { + "file": "0053_server_probe_terminal_proofs.sql", + "sha256": "759ec8c349fcb24c43af410ad6ade714432678b3a021c94bd982080d6a8986be", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "Pins terminal proof to the current retained original on insert and an exact committed retirement claim on delete.", + "Extends claims only to the identical signed terminal proof when the original diagnostic admission is gone." + ], + "effects": [ + "Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged." + ] + }, + { + "file": "0054_mcp_geoip_retirement.sql", + "sha256": "680390f98076e3047dbb5a34eab417b253853703932d16c17e4fa3a11060accc", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Adds only exact settled GeoIP settings version 1 to the existing immutable ordinary retirement claim guard." + ], + "effects": [ + "Replaces the ordinary claim guard function without changing settings or retiring existing rows during migration." + ] + }, + { + "file": "0055_mcp_backup_schedule_retirement.sql", + "sha256": "07e63c1245ea547176d613933a81ad354e734fa64218dadced67ee8f965c01b6", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 1, + "bodyReview": [ + "Adds only exact settled backup schedule settings version 1 to the existing immutable ordinary retirement claim guard." + ], + "effects": [ + "Replaces the ordinary claim guard function without changing backup settings, metadata or entries during migration." + ] + }, + { + "file": "0056_mcp_manual_inverse_recovery_v8.sql", + "sha256": "1dece1d9c5a9cf69e51336f35f868561dad9da6d21556ea31ef5b102634ed9ee", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 2, + "bodyReview": [ + "The manual inverse charge-source guard requires recovery version 8 and preserves its pending operation, source version, owner and profile checks.", + "The manual inverse intent guard requires recovery version 8 and preserves its advisory lock and exact running service intent checks." + ], + "effects": [ + "Replaces the two live version guard functions and preserves existing charges, intents, and plans." + ] + }, + { + "file": "0057_mcp_routing_charge_audit_index.sql", + "sha256": "499cb91836618e2859e2bb014f4642d8e95770267e9527844c3dcb95ecb32b7f", + "transaction": "self-managed-single-transaction", + "selfMarker": true, + "opaqueBodies": 0, + "bodyReview": [], + "effects": [ + "Adds the partial routing charge audit lookup index. Retains all rows and permissions." + ] + } +] + +export const MIGRATION_MANIFEST: readonly MigrationReview[] = Object.freeze(classifiedMigrationPins.map(entry => Object.freeze({ ...entry, bodyReview: Object.freeze([...entry.bodyReview]), effects: Object.freeze([...entry.effects]) }))) + +export function validateMigrationFileSet(files: readonly string[]) { + if (JSON.stringify([...files].sort()) !== JSON.stringify(MIGRATION_MANIFEST.map(entry => entry.file))) refuse("Migration file set changed; classify additions/removals before dispatch") +} + +export async function inventoryMigrationDirectory(directory: string) { + const entries = (await readdir(directory, { withFileTypes: true })).filter(entry => entry.name.toLowerCase().endsWith(".sql")) + if (entries.some(entry => !entry.isFile())) refuse("Migration SQL entry is not an ordinary file") + const files = entries.map(entry => entry.name).sort() + validateMigrationFileSet(files) + return Promise.all(MIGRATION_MANIFEST.map(async entry => analyzeMigration(entry.file, await readFile(join(directory, entry.file)), entry))) +} +export const migrationManifestDigest = () => hash(JSON.stringify(MIGRATION_MANIFEST)) + +// Mandatory current table presence derived from every pinned CREATE TABLE, +// resetting the accumulated public table set at the reviewed destructive0002. +// This is a necessary presence check, not a substitute for exact live column, +// constraint/index/function/trigger comparison by the control protocol. +export const CURRENT_REQUIRED_TABLES: readonly string[] = Object.freeze(["acme_settings","alert_engine_confirm_pending","alert_engine_cursor","alert_engine_prev_live","alert_engine_state","alert_groups","alert_history","alert_outbox","alert_rule_conditions","alert_rule_targets","alert_rules","alert_telegram_settings","app_users","backup_entries","backup_schedule_settings","backup_storage_settings","certificate_issue_jobs","certificate_renew_settings","config_revisions","data_migration","events","evobgp_settings","filter_rules","flow_asn_meta","flow_buckets","flow_daily_dims","flow_daily_facts","flow_hour_facts","flow_ip_meta","flow_minute_dims","flow_minute_stats","flow_settings_intents","flow_write_receipts","geoip_settings","internet_path_settings","internet_path_snapshots","mcp_artifacts","mcp_audit","mcp_budget_buckets","mcp_connections","mcp_credentials","mcp_diagnostic_cancellations","mcp_drift","mcp_grants","mcp_management_channels","mcp_oauth_clients","mcp_oauth_requests","mcp_operation_checkpoints","mcp_operation_replay_tombstones","mcp_operation_steps","mcp_operations","mcp_ordinary_retirement_claims","mcp_owner_revocations","mcp_path_checks","mcp_plans","mcp_portal_events","mcp_protections","mcp_read_leases","mcp_recovery_intents","mcp_requests","mcp_routing_forward_charges","mcp_routing_manual_inverse_charges","mcp_routing_observations","mcp_routing_profiles","mcp_routing_reservation_slots","mcp_routing_reservations","mcp_scoped_recovery_intents","mcp_secrets","mcp_settings","mcp_unauthorized_buckets","recursive_routes","router_log_collection_state","router_log_entries","router_log_settings","scheduler_runs","schema_migrations","server_poll_acks","server_poll_archives","server_poll_executions","server_poll_intents","server_poll_quarantines","server_poll_receipts","server_poll_retirement_claims","server_poll_storage_admissions","server_poll_storage_usage","server_probe_execution_payloads","server_probe_execution_quarantines","server_probe_execution_stages","server_probe_executions","server_probe_retired_sources","server_probe_retirement_claims","server_probe_storage_admissions","server_probe_storage_usage","server_probe_terminal_proofs","server_snapshots","servers","servers_api_ping_settings","servers_rest_ping_samples","traffic_collection_state","traffic_flow_settings","traffic_samples","traffic_settings","uptime_probe_samples","uptime_probes","uptime_resource_samples","uptime_settings","uptime_speed_probes","uptime_speed_test_runs","user_interface_bindings","vrf_production_tickets"]) + +export interface LegacyBaselineEvidence { + inventoryComplete: boolean; databaseGeneration: string; unresolvedAttemptIds: readonly string[] + markerTable: "absent" | "present"; markerIds: readonly string[] + applicationObjects: readonly { schema: string; name: string; kind: string }[] + /** Receipt from a separately trusted complete schema comparator. This module + * only checks its binding; it does not manufacture a live schema attestation. */ + verifiedSchema?: { manifestDigest: string; databaseGeneration: string; exactCurrentSchema: true; observedSchemaDigest: string } +} +export function assessLegacyMigrationBaseline(evidence: LegacyBaselineEvidence) { + const blocked = (reason: string) => ({ state: "blocked" as const, reason, dispatchAuthorized: false as const }) + if (!evidence.inventoryComplete || !evidence.databaseGeneration || evidence.unresolvedAttemptIds.length) return blocked("Incomplete inventory, missing generation or unresolved durable attempt") + if (evidence.applicationObjects.length > 50000 || evidence.applicationObjects.some(object => !object.schema || !object.name || !object.kind)) return blocked("Malformed or unbounded schema object evidence") + const ids = MIGRATION_MANIFEST.map(entry => entry.file.slice(0, -4)), actual = [...evidence.markerIds].sort() + if (new Set(actual).size !== actual.length || actual.some(id => !ids.includes(id)) || evidence.markerTable === "absent" && actual.length) return blocked("Unknown, duplicate or inconsistent legacy marker evidence") + const objects = evidence.applicationObjects.filter(object => !(object.schema === "public" && object.name === "schema_migrations" && object.kind === "table")) + if (!actual.length && !objects.length) return { state: "clean-database-candidate" as const, reason: "Complete object inventory shows no application objects, independently of marker absence", dispatchAuthorized: false as const } + if (JSON.stringify(actual) !== JSON.stringify(ids)) return blocked("Unmarked application state or legacy prefix requires private reconciliation; missing marker is not fresh/never-run evidence") + const proof = evidence.verifiedSchema + const present = new Set(evidence.applicationObjects.filter(object => object.schema === "public" && ["table", "partitioned-table"].includes(object.kind)).map(object => object.name)) + if (!proof || !proof.exactCurrentSchema || proof.manifestDigest !== migrationManifestDigest() || proof.databaseGeneration !== evidence.databaseGeneration || !/^[a-f0-9]{64}$/.test(proof.observedSchemaDigest) || CURRENT_REQUIRED_TABLES.some(table => !present.has(table))) return blocked("All markers need a separately verified exact current-schema attestation and mandatory table inventory before enrollment") + return { state: "legacy-current-enrollment-candidate" as const, reason: "All exact known ids plus complete schema comparator receipt; historical executed file hashes remain unproven", dispatchAuthorized: false as const } +} diff --git a/backend/src/db/migration-schema-inventory.sql.test.ts b/backend/src/db/migration-schema-inventory.sql.test.ts new file mode 100644 index 00000000..9fb3606d --- /dev/null +++ b/backend/src/db/migration-schema-inventory.sql.test.ts @@ -0,0 +1,119 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { basename, dirname, resolve } from "node:path" +import { fileURLToPath } from "node:url" + +assert.equal(process.env.MMAPP_OWN_MIGRATION_SCHEMA_INVENTORY, "true") +const databaseUrl = new URL(process.env.DATABASE_URL ?? "") +assert.equal(databaseUrl.hostname, "127.0.0.1"); assert.equal(databaseUrl.pathname, "/postgres") +assert(["postgres:", "postgresql:"].includes(databaseUrl.protocol)) +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) + +const { Pool } = await import("pg") +const { pool, closePool } = await import("./index.js") +const { applySqlMigrations } = await import("./migrate.js") +const { inventoryMigrationDirectory, migrationManifestDigest, MIGRATION_MANIFEST } = await import("./migration-manifest.js") +const { captureMigrationSchema, compareMigrationSchemas, MigrationSchemaInventoryError } = await import("./migration-schema-inventory.js") +const suffix = randomUUID().replaceAll("-", "").slice(0, 12), names: string[] = [] +const identifier = (value: string) => { assert(/^mm_schema_[a-z0-9_]+$/.test(value)); return '"' + value + '"' } +const database = async (label: string, template?: string) => { + const name = `mm_schema_${suffix}_${label}`; identifier(name) + await pool.query(`CREATE DATABASE ${identifier(name)} TEMPLATE ${template ? identifier(template) : "template0"}`) + names.push(name) + const url = new URL(databaseUrl); url.pathname = "/" + name + return { name, connection: new Pool({ connectionString: url.toString(), max: 2 }) } +} +async function capture(connection: InstanceType, options?: Parameters[1]) { + const client = await connection.connect() + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + await client.query("SET LOCAL search_path=pg_catalog") + await client.query("SET LOCAL TimeZone='UTC'") + await client.query("SET LOCAL DateStyle='ISO, YMD'") + await client.query("SET LOCAL statement_timeout='5s'") + const result = await captureMigrationSchema(client, options) + await client.query("ROLLBACK") + return result + } catch (error) { await client.query("ROLLBACK").catch(() => {}); throw error } + finally { client.release() } +} +const cases: string[] = [] +try { + await applySqlMigrations(pool) + const inventory = await inventoryMigrationDirectory(fileURLToPath(new URL("../../drizzle/", import.meta.url))) + assert.equal(inventory.length, MIGRATION_MANIFEST.length) + const reference = await database("reference") + let baseline: Awaited> + try { + await applySqlMigrations(reference.connection) + baseline = await capture(reference.connection) + assert.equal(baseline.format, "pg18-application-schema-v3") + assert(baseline.sections.relations.length >= 89) + assert(baseline.sections.columns.length > 500) + assert(baseline.sections.constraints.length > 100) + assert(baseline.sections.functions.length > 0) + assert.deepEqual(compareMigrationSchemas(baseline, await capture(reference.connection)), { equal: true, evidenceOnly: true, enrollmentAuthorized: false, sha256: baseline.sha256 }) + cases.push(`all ${inventory.length} pinned migrations produce repeatable complete PG18 catalog evidence on an actual checked-out read-only backend`) + } finally { await reference.connection.end() } + const independent = await database("independent") + try { + await applySqlMigrations(independent.connection) + const observed = await capture(independent.connection) + try { compareMigrationSchemas(baseline!, observed) } catch (error) { + // This reference is built exclusively from the pinned repository SQL; + // report field names, not function bodies/default values or user data. + const differences = Object.keys(baseline!.sections).flatMap(key => { + const section = key as keyof typeof baseline.sections, before = baseline!.sections[section], after = observed.sections[section] + if (JSON.stringify(before) === JSON.stringify(after)) return [] + return [{ section, beforeRows: before.length, afterRows: after.length, fields: [...new Set(before.flatMap((row, i) => Object.keys(row).filter(field => JSON.stringify(row[field]) !== JSON.stringify(after[i]?.[field]))))] }] + }) + console.error("Independent schema differences:", JSON.stringify(differences)); throw error + } + cases.push("independently migrated database with different physical OIDs matches the same exact logical schema") + } finally { await independent.connection.end() } + + const mismatches: [string, string][] = [ + ["table", "CREATE TABLE public.mm_schema_extra(id integer)"], + ["column-default", "ALTER TABLE public.servers ALTER COLUMN name SET DEFAULT 'owned schema drift'"], + ["column-type", "ALTER TABLE public.servers ALTER COLUMN name TYPE varchar(999)"], + ["index", "CREATE INDEX mm_schema_index ON public.servers(host)"], + ["constraint", "ALTER TABLE public.servers ADD CONSTRAINT mm_schema_port CHECK(port>0) NOT VALID"], + ["foreign-key", "ALTER TABLE public.servers ADD CONSTRAINT mm_schema_self FOREIGN KEY(id) REFERENCES public.servers(id) DEFERRABLE INITIALLY DEFERRED"], + ["sequence", "ALTER SEQUENCE public.servers_id_seq INCREMENT BY 2"], + ["comment", "COMMENT ON TABLE public.servers IS 'owned schema drift'"], + ["function", "CREATE FUNCTION public.mm_schema_function() RETURNS integer LANGUAGE sql IMMUTABLE AS 'SELECT 7'"], + ["view", "CREATE VIEW public.mm_schema_view AS SELECT id FROM public.servers"], + ["partition", "CREATE TABLE public.mm_schema_partitioned(id integer) PARTITION BY RANGE(id); CREATE TABLE public.mm_schema_leaf PARTITION OF public.mm_schema_partitioned FOR VALUES FROM(0) TO(10)"], + ["trigger", "CREATE FUNCTION public.mm_schema_trigger_fn() RETURNS trigger LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END'; CREATE TRIGGER mm_schema_trigger BEFORE UPDATE ON public.servers FOR EACH ROW EXECUTE FUNCTION public.mm_schema_trigger_fn()"], + ] + for (const [label, sql] of mismatches) { + const target = await database(label.replaceAll("-", "_"), reference.name) + try { + await target.connection.query(sql) + const observed = await capture(target.connection) + assert.throws(() => compareMigrationSchemas(baseline!, observed), MigrationSchemaInventoryError, label) + cases.push("actual " + label + " drift differs from the complete captured baseline") + } finally { await target.connection.end() } + } + const unsupported = await database("unsupported", reference.name) + try { + await unsupported.connection.query("ALTER TABLE public.servers ENABLE ROW LEVEL SECURITY; CREATE POLICY mm_schema_policy ON public.servers USING(true)") + await assert.rejects(capture(unsupported.connection), MigrationSchemaInventoryError) + cases.push("actual unsupported RLS policy refuses capture rather than disappearing from equality") + } finally { await unsupported.connection.end() } + const control = await database("control", reference.name) + try { + await control.connection.query("CREATE SCHEMA mm_control; CREATE TABLE mm_control.mm_schema_private(id integer)") + await assert.rejects(capture(control.connection), MigrationSchemaInventoryError) + const scoped = await capture(control.connection, { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["mm_control"] } }) + assert(scoped.sections.namespaces.some(row => row.namespace === "mm_control")) + assert(!scoped.sections.relations.some(row => row.namespace === "mm_control")) + await control.connection.query("CREATE VIEW public.mm_schema_external_view AS SELECT id FROM mm_control.mm_schema_private") + await assert.rejects(capture(control.connection, { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["mm_control"] } }), MigrationSchemaInventoryError) + cases.push("extra namespace refuses by default; explicit control exclusion is inventoried and application cross-dependency still refuses") + } finally { await control.connection.end() } + const client = await pool.connect() + try { await assert.rejects(captureMigrationSchema(client), MigrationSchemaInventoryError); cases.push("autocommit/read-write connection cannot claim a stable read-only catalog snapshot") } + finally { client.release() } + console.log("Owned migration schema inventory SQL:", JSON.stringify({ cases: cases.length, names: cases, databases: names.length, actualSql: true, pinnedFiles: inventory.length, manifestDigest: migrationManifestDigest(), logicalSchemaDigest: baseline!.sha256, dispatcherChanged: false, enrollmentAuthorized: false, productionDatabaseTouched: false })) +} finally { await closePool() } diff --git a/backend/src/db/migration-schema-inventory.test.ts b/backend/src/db/migration-schema-inventory.test.ts new file mode 100644 index 00000000..6154b402 --- /dev/null +++ b/backend/src/db/migration-schema-inventory.test.ts @@ -0,0 +1,360 @@ +import assert from "node:assert/strict" +import { readFileSync } from "node:fs" +import { captureMigrationSchema, compareMigrationSchemas, MIGRATION_SCHEMA_QUERIES, MIGRATION_SCHEMA_PARAMETERS, MigrationSchemaInventoryError, type SchemaSection } from "./migration-schema-inventory.js" + +type Row = Record +type Data = Record +const metadata = { backend_pid: "71", transaction_started: "2026-10-25 10:00:00+00", read_only: "on", isolation: "repeatable read", search_path: "pg_catalog", timezone: "UTC", date_style: "ISO, YMD", standard_strings: "on", server_version: "180000", statement_timeout: "5s" } +const relation = (name: string, kind = "r", extra: Row = {}): Row => ({ namespace: "public", name, kind, persistence: "p", owner: "postgres", acl: null, options: null, access_method: ["r", "p"].includes(kind) ? "heap" : kind === "i" ? "btree" : null, tablespace: null, toast_options: null, of_type: null, row_security: false, force_row_security: false, replica_identity: "d", is_partition: false, populated: true, partition_key: null, partition_bound: null, ...extra }) +const column = (relation = "servers", name = "id", extra: Row = {}): Row => ({ namespace: "public", relation, position: 1, name, dropped: false, type: "bigint", type_modifier: -1, dimensions: 0, not_null: true, identity: "a", generated: "", storage: "p", compression: "", statistics_target: null, has_missing: false, missing_value: null, acl: null, options: null, comment: null, collation: null, default_expression: null, is_local: true, inherited_count: 0, ...extra }) +function fixture(): Data { + const data = Object.fromEntries(Object.keys(MIGRATION_SCHEMA_QUERIES).map(key => [key, []])) as unknown as Data + data.database = [{ encoding: "UTF8", collate: "C.UTF-8", ctype: "C.UTF-8", locale_provider: "c", locale: null, locale_rules: null, collation_version: null, default_tablespace: "pg_default", owner: "postgres", acl: null }] + data.namespaces = [{ namespace: "public", owner: "pg_database_owner", acl: ["=U/pg_database_owner", "pg_database_owner=UC/pg_database_owner"] }] + data.relations = [relation("servers"), relation("servers_id_seq", "S"), relation("servers_pkey", "i"), relation("server_view", "v"), relation("traffic_samples", "p", { partition_key: "RANGE (sampled_at)" }), relation("traffic_samples_d_20261025", "r", { is_partition: true, partition_bound: "FOR VALUES FROM ('2026-10-25') TO ('2026-10-26')" })] + data.columns = [column(), column("servers", "computed", { position: 2, identity: "", generated: "s", default_expression: "(id + 1)" })] + data.constraints = [{ namespace: "public", name: "servers_pkey", relation: "servers", domain: null, kind: "p", deferrable: false, deferred: false, validated: true, is_local: true, inherited_count: 0, no_inherit: true, enforced: true, period: false, definition: "PRIMARY KEY (id)", parent_namespace: null, parent_relation: null, parent_constraint: null, foreign_namespace: null, foreign_relation: null }] + data.indexes = [{ namespace: "public", name: "servers_pkey", relation: "servers", definition: "CREATE UNIQUE INDEX servers_pkey ON public.servers USING btree (id)", unique_index: true, primary_index: true, exclusion: false, immediate: true, clustered: false, valid: true, ready: true, live: true, replica_identity: false, check_xmin: false, nulls_not_distinct: false, attributes: 1, key_attributes: 1, keys: "1", key_options: "0", operator_classes: ["pg_catalog.int8_ops"], collations: [""], expressions: null, predicate: null }] + data.inheritance = [{ namespace: "public", child: "traffic_samples_d_20261025", parent_namespace: "public", parent: "traffic_samples", position: 1, detach_pending: false }] + data.sequences = [{ namespace: "public", name: "servers_id_seq", type: "bigint", start: "1", increment: "1", maximum: "9223372036854775807", minimum: "1", cache: "1", cycle: false, owner_namespace: "public", owner_relation: "servers", owner_column: "id", owner_dependency: "i" }] + data.views = [{ namespace: "public", name: "server_view", kind: "v", definition: "SELECT id FROM public.servers;" }] + data.types = [{ namespace: "public", name: "servers", kind: "c", category: "C", owner: "postgres", acl: null, length: -1, by_value: false, alignment: "d", storage: "x", not_null: false, preferred: false, defined: true, delimiter: ",", dimensions: 0, input_function: "record_in(cstring,oid,integer)", output_function: "record_out(record)", receive_function: "record_recv(internal,oid,integer)", send_function: "record_send(record)", modifier_input: "-", modifier_output: "-", analyze_function: "-", subscript_function: "-", base_type: null, element_type: null, relation_namespace: "public", relation: "servers", default_expression: null, default_literal: null, collation: null, enum_labels: [], range_definition: null }] + data.functions = [{ namespace: "public", name: "guard", arguments: "", kind: "f", language: "plpgsql", owner: "postgres", acl: null, definition: "CREATE OR REPLACE FUNCTION public.guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$", external_library: null, security_definer: false, leakproof: false, strict: false, volatility: "v", parallel: "u", cost: "100", rows: "0", support: "-", configuration: null }] + data.triggers = [{ namespace: "public", relation: "servers", name: "guard", internal: false, enabled: "A", type: 19, definition: "CREATE TRIGGER guard BEFORE UPDATE ON public.servers FOR EACH ROW EXECUTE FUNCTION public.guard()", function_namespace: "public", function_name: "guard", function_arguments: "", arguments: "", condition: null, deferrable: false, initially_deferred: false, old_table: null, new_table: null, parent_relation: null, parent_trigger: null, constraint_name: null }] + for (const section of ["database", "namespaces", "relations", "columns", "constraints", "types", "functions", "triggers"] as const) for (const row of data[section]) row.comment = null + for (const row of data.constraints) row.restore_definition = row.kind === "c" ? row.definition : null + return data +} +function client(data: Data, transform: (envelope: Row, section: SchemaSection, index: number) => void = () => {}) { + let count = 0 + const calls: string[] = [] + return { calls, query: async (sql: string, values?: unknown[]) => { + const section = Object.entries(MIGRATION_SCHEMA_QUERIES).find(([, query]) => query === sql)?.[0] as SchemaSection | undefined + assert(section, "Only exact fixed catalog SQL may execute") + assert.deepEqual(values, ["database", "namespaces"].includes(section) ? [] : [["public"]]); calls.push(section) + const envelope: Row = { ...metadata, objects: structuredClone(data[section]) } + transform(envelope, section, count++); return { rows: [envelope] } + } } +} +const snapshot = (data = fixture()) => captureMigrationSchema(client(data)) +let cases = 0 +async function test(name: string, work: () => Promise | void) { await work(); cases++; console.log("ok", name) } +const expected = await snapshot() +function constraintTriggerFixture(): Data { + const data = fixture() + data.constraints.push({ ...data.constraints[0], name: "guard", kind: "t", deferrable: true, deferred: true, definition: "TRIGGER DEFERRABLE INITIALLY DEFERRED" }) + Object.assign(data.triggers[0], { type: 21, constraint_name: "guard", deferrable: true, initially_deferred: true, definition: "CREATE CONSTRAINT TRIGGER guard AFTER INSERT OR UPDATE ON public.servers DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION public.guard()" }) + return data +} +await test("user constraint triggers require a complete portable matching trigger and preserve definition evidence", async () => { + const before = await snapshot(constraintTriggerFixture()) + const data = constraintTriggerFixture(); data.triggers[0].definition += " /* changed */" + const after = await snapshot(data) + assert.throws(() => compareMigrationSchemas(before, after), MigrationSchemaInventoryError) + assert.equal(before.sections.constraints.find(row => row.kind === "t")?.name, "guard") +}) +await test("orphan, mismatched, foreign and parented user constraint triggers refuse", async () => { + for (const mutate of [ + (data: Data) => { data.triggers = [] }, + (data: Data) => { data.triggers[0].name = "other" }, + (data: Data) => { data.triggers[0].relation = "other" }, + (data: Data) => { data.triggers[0].constraint_name = null }, + (data: Data) => { data.triggers[0].deferrable = false }, + (data: Data) => { data.triggers[0].initially_deferred = false }, + (data: Data) => { data.triggers[0].internal = true }, + (data: Data) => { data.triggers[0].function_namespace = "external" }, + (data: Data) => { data.constraints[1].domain = "domain" }, + (data: Data) => { data.constraints[1].foreign_namespace = "public"; data.constraints[1].foreign_relation = "servers" }, + (data: Data) => { data.constraints[1].parent_namespace = "public"; data.constraints[1].parent_relation = "servers"; data.constraints[1].parent_constraint = "parent" }, + (data: Data) => { data.triggers[0].parent_relation = "servers"; data.triggers[0].parent_trigger = "parent" }, + ]) { const data = constraintTriggerFixture(); mutate(data); await assert.rejects(snapshot(data), MigrationSchemaInventoryError) } +}) +await test("same canonical actual content compares only as nonauthorizing evidence", async () => { + const data = fixture() + for (const rows of Object.values(data)) rows.reverse() + data.namespaces[0].acl = [...data.namespaces[0].acl as string[]].reverse() + const observed = await snapshot(data) + assert.equal(expected.sha256, observed.sha256) + assert.deepEqual(compareMigrationSchemas(expected, observed), { equal: true, evidenceOnly: true, enrollmentAuthorized: false, sha256: expected.sha256 }) +}) +await test("control event trigger evidence pins mode and requires a captured control function", async () => { + const data = fixture() + data.namespaces.push({ namespace: "mmapp_control", owner: "operator", acl: null, comment: null }) + data.functions.push({ ...data.functions[0], namespace: "mmapp_control", name: "ddl_guard" }) + const event: Row = { + name: "ddl_guard", event: "ddl_command_start", owner: "operator", enabled: "A", tags: ["ALTER TABLE"], + comment: null, function_namespace: "mmapp_control", function_name: "ddl_guard", function_arguments: "", + } + const capture = (trigger: Row) => captureMigrationSchema({ async query(sql) { + if (sql.startsWith("/* migration-schema:control-event-triggers */")) return { rows: [{ ...metadata, objects: [trigger] }] } + const section = Object.entries(MIGRATION_SCHEMA_QUERIES).find(([, query]) => query === sql)?.[0] ?? + (sql.startsWith("/* migration-schema:unsupported */") ? "unsupported" : undefined) + assert(section) + return { rows: [{ ...metadata, objects: data[section as SchemaSection] }] } + } }, { scope: { applicationNamespaces: ["public", "mmapp_control"] }, includeControlEventTriggers: true }) + const active = await capture(event) + const disabled = await capture({ ...event, enabled: "D" }) + assert.equal(active.format, "pg18-application-schema-v4") + assert.equal(compareMigrationSchemas(active, structuredClone(active)).equal, true) + assert.throws(() => compareMigrationSchemas(active, disabled), /differs/) + await assert.rejects(capture({ ...event, function_namespace: "public", function_name: "guard" }), /uncaptured control function/) +}) +await test("fixed queries cover all required object classes without mutable sequence values", () => { + assert.equal(Object.keys(MIGRATION_SCHEMA_QUERIES).length, 13) + for (const sql of Object.values(MIGRATION_SCHEMA_QUERIES)) { assert(sql.includes("WITH captured AS MATERIALIZED")); assert(sql.includes("LIMIT ")); assert(!sql.includes("last_value")); assert(!sql.includes("set_config(")); assert(!sql.includes("nextval(")) } + assert(MIGRATION_SCHEMA_QUERIES.constraints.includes("conenforced")); assert(MIGRATION_SCHEMA_QUERIES.constraints.includes("conperiod")) + assert(MIGRATION_SCHEMA_QUERIES.triggers.includes("CREATE CONSTRAINT TRIGGER ")) +}) +await test("each exact catalog query binds only its declared positional parameters", async () => { + assert.deepEqual(Object.keys(MIGRATION_SCHEMA_PARAMETERS).sort(), Object.keys(MIGRATION_SCHEMA_QUERIES).sort()) + const data = fixture(), calls = new Set() + await captureMigrationSchema({ query: async (sql, values) => { + const section = (Object.keys(MIGRATION_SCHEMA_QUERIES) as SchemaSection[]).find(key => MIGRATION_SCHEMA_QUERIES[key] === sql) + assert(section, "Query identity must be the fixed catalog statement") + const expectedValues = section === "database" || section === "namespaces" ? [] : [["public"]] + const placeholders = [...new Set([...sql.matchAll(/\$(\d+)/g)].map(match => Number(match[1])))].sort((a, b) => a - b) + assert.deepEqual(placeholders, expectedValues.length ? [1] : [], section) + assert.equal(MIGRATION_SCHEMA_PARAMETERS[section], expectedValues.length ? "application-namespaces" : "none", section) + assert.deepEqual(values, expectedValues, section) + calls.add(section) + return { rows: [{ ...metadata, objects: structuredClone(data[section]) }] } + } }) + assert.deepEqual([...calls].sort(), Object.keys(MIGRATION_SCHEMA_QUERIES).sort()) +}) +await test("catalog internal char concatenation operands have explicit text casts", () => { + assert(MIGRATION_SCHEMA_QUERIES.unsupported.includes("||a.defaclobjtype::text FROM pg_catalog.pg_default_acl")) + // Audit every concatenation against internal-char catalog fields used by this + // inventory. Names (nspname/relname/etc.) are PostgreSQL name, not char. + const internalChar = /(?:\b\w+\.(?:defaclobjtype|relkind|relpersistence|relreplident|attidentity|attgenerated|attstorage|attcompression|contype|deptype|typtype|typcategory|typalign|typstorage|typdelim|prokind|provolatile|proparallel|tgenabled|datlocprovider))(?![A-Za-z_0-9])/g + for (const sql of Object.values(MIGRATION_SCHEMA_QUERIES)) for (const match of sql.matchAll(internalChar)) { + const before = sql.slice(0, match.index).trimEnd(), after = sql.slice(match.index! + match[0].length) + if (before.endsWith("||") || /^\s*(?:::text)?\s*\|\|/.test(after)) assert(after.startsWith("::text"), match[0]) + } + assert(MIGRATION_SCHEMA_QUERIES.triggers.includes("||t.tgtype::text")) + assert(MIGRATION_SCHEMA_QUERIES.triggers.includes("||pt.tgtype::text")) +}) +await test("query failure identifies only its fixed section and preserves original cause", async () => { + const original = Object.assign(new Error("untrusted database detail"), { code: "42725" }) + const sql = client(fixture()) + await assert.rejects(captureMigrationSchema({ query: async (query, values) => { + if (query === MIGRATION_SCHEMA_QUERIES.unsupported) throw original + return sql.query(query, values) + } }), error => { + assert(error instanceof MigrationSchemaInventoryError) + assert.equal(error.message, "Catalog query failed: unsupported") + assert.equal(error.cause, original) + return true + }) +}) +await test("PG18 null/default, zero/disabled and positive statistics targets remain distinct", async () => { + const captured: Awaited>[] = [] + for (const statisticsTarget of [null, 0, 100]) { + const data = fixture(); data.columns[0].statistics_target = statisticsTarget + const result = await snapshot(data) + assert.equal(result.sections.columns.find(row => row.name === "id")?.statistics_target, statisticsTarget) + assert.equal(compareMigrationSchemas(result, await snapshot(data)).equal, true) + captured.push(result) + } + for (let a = 0; a < captured.length; a++) for (let b = a + 1; b < captured.length; b++) { + assert.notEqual(captured[a].sha256, captured[b].sha256) + assert.throws(() => compareMigrationSchemas(captured[a], captured[b]), MigrationSchemaInventoryError) + } + assert(MIGRATION_SCHEMA_QUERIES.columns.includes("a.attstattarget::int AS statistics_target")) + assert(!MIGRATION_SCHEMA_QUERIES.columns.includes("COALESCE(a.attstattarget")) +}) +await test("nullable statistics target does not permit nonintegers or other null integers", async () => { + for (const value of ["100", true, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1]) { + const data = fixture(); data.columns[0].statistics_target = value + await assert.rejects(snapshot(data), MigrationSchemaInventoryError) + } + for (const field of ["position", "type_modifier", "dimensions", "inherited_count"]) { + const data = fixture(); data.columns[0][field] = null + await assert.rejects(snapshot(data), MigrationSchemaInventoryError) + } +}) +await test("pinned runtime default caches retain physical evidence but compare the same logical schema", async () => { + const migration = readFileSync(new URL("../../drizzle/0018_mcp_owner_validation.sql", import.meta.url), "utf8") + assert(migration.includes("ADD COLUMN IF NOT EXISTS owner_valid_until timestamptz NOT NULL DEFAULT now()")) + const captures: Awaited>[] = [] + for (const cached of ['{"2026-10-25 10:00:00+00"}', '{"2026-10-25 10:01:00+00"}', null]) { + const data = fixture() + Object.assign(data.columns[0], { type: "timestamp with time zone", identity: "", default_expression: "now()", has_missing: cached !== null, missing_value: cached }) + const result = await snapshot(data) + assert.equal(result.sections.columns.find(row => row.name === "id")?.missing_value, cached) + assert.equal(result.observedPhysical.columns.find(row => row.name === "id")?.missing_value, cached) + captures.push(result) + } + for (const captured of captures.slice(1)) { + assert.equal(compareMigrationSchemas(captures[0], captured).equal, true) + assert.equal(captures[0].sha256, captured.sha256) + assert.notEqual(captures[0].observedPhysical.sha256, captured.observedPhysical.sha256) + } + for (const change of [{ default_expression: "clock_timestamp()" }, { type: "timestamp without time zone" }, { not_null: false }, { identity: "a" }]) { + const data = fixture() + Object.assign(data.columns[0], { type: "timestamp with time zone", identity: "", default_expression: "now()", ...change }) + const changed = await snapshot(data) + assert.throws(() => compareMigrationSchemas(captures[0], changed), MigrationSchemaInventoryError) + } +}) +await test("physical cache shapes and independently tampered physical evidence refuse", async () => { + for (const change of [{ has_missing: true, missing_value: null }, { has_missing: false, missing_value: "{1}" }, { has_missing: true, missing_value: 1 }, { has_missing: true, missing_value: [1] }]) { + const data = fixture(); Object.assign(data.columns[0], change) + await assert.rejects(snapshot(data), MigrationSchemaInventoryError) + } + const tampered = structuredClone(expected); tampered.observedPhysical.columns[0].missing_value = "{forged}" + assert.throws(() => compareMigrationSchemas(expected, tampered), MigrationSchemaInventoryError) + const rawTampered = structuredClone(expected) + Object.assign(rawTampered.sections.columns[0], { has_missing: true, missing_value: "{changed}" }) + assert.throws(() => compareMigrationSchemas(expected, rawTampered), MigrationSchemaInventoryError) +}) +for (const [name, change] of [ + ["column type", (data: Data) => { data.columns[0].type = "integer" }], + ["identity", (data: Data) => { data.columns[0].identity = "d" }], + ["generated expression", (data: Data) => { data.columns[1].default_expression = "(id + 2)" }], + ["constraint enforcement", (data: Data) => { data.constraints[0].enforced = false }], + ["index predicate", (data: Data) => { data.indexes[0].predicate = "(id > 1)" }], + ["partition bound", (data: Data) => { data.relations[5].partition_bound = "DEFAULT" }], + ["sequence cache", (data: Data) => { data.sequences[0].cache = "10" }], + ["view definition", (data: Data) => { data.views[0].definition = "SELECT id+1 FROM public.servers;" }], + ["function body", (data: Data) => { data.functions[0].definition = "different stored source" }], + ["trigger enabled state", (data: Data) => { data.triggers[0].enabled = "D" }], + ["TOAST options", (data: Data) => { data.relations[0].toast_options = ["autovacuum_enabled=false"] }], +] as const) await test("same table names do not hide " + name + " drift", async () => { + const data = fixture(); change(data) + assert.deepEqual(data.relations.map(row => row.name), fixture().relations.map(row => row.name)) + const observed = await snapshot(data) + assert.throws(() => compareMigrationSchemas(expected, observed), MigrationSchemaInventoryError) +}) +await test("hash/boolean as attestation and altered captured content are refused", () => { + assert.throws(() => compareMigrationSchemas({ sha256: expected.sha256, equal: true } as never, expected), MigrationSchemaInventoryError) + const changed = structuredClone(expected); changed.sections.columns[0].type = "changed" + assert.throws(() => compareMigrationSchemas(expected, changed), MigrationSchemaInventoryError) +}) +await test("unsupported object classes and external procedural semantics refuse", async () => { + for (const mutate of [ + (data: Data) => { data.unsupported = [{ kind: "publication", identity: "unknown" }] }, + (data: Data) => { data.relations[0].kind = "f" }, + (data: Data) => { data.functions[0].language = "c" }, + (data: Data) => { data.functions[0].kind = "p" }, + (data: Data) => { data.types[0].kind = "b" }, + (data: Data) => { data.inheritance[0].detach_pending = true }, + (data: Data) => { data.triggers[0].function_namespace = "excluded_control" }, + (data: Data) => { data.namespaces.push({ namespace: "unknown_schema", owner: "postgres", acl: null }) }, + ]) { const data = fixture(); mutate(data); await assert.rejects(snapshot(data), MigrationSchemaInventoryError) } +}) +await test("unsupported preflight runs before any detailed expression/type deparser", async () => { + const data = fixture(); data.unsupported = [{ kind: "unsafe-type-io", identity: "public.external_type" }] + const sql = client(data) + await assert.rejects(captureMigrationSchema(sql), MigrationSchemaInventoryError) + assert.deepEqual(sql.calls, ["database", "namespaces", "unsupported"]) +}) +await test("unknown/missing row fields, null PG18 semantics and overflow refuse", async () => { + for (const transform of [ + (row: Row) => { row.objects = null }, + (row: Row) => { row.objects = Array.from({ length: 20001 }, () => fixture().database[0]) }, + (row: Row) => { (row.objects as Row[])[0].unknown = true }, + (row: Row) => { delete (row.objects as Row[])[0].encoding }, + ]) await assert.rejects(captureMigrationSchema(client(fixture(), (row, section) => { if (section === "database") transform(row) })), MigrationSchemaInventoryError) + const data = fixture(); data.constraints[0].enforced = null; await assert.rejects(snapshot(data), MigrationSchemaInventoryError) +}) +await test("readonly snapshot and fixed PG18 deparse context are mandatory", async () => { + for (const replacement of [{ read_only: "off" }, { isolation: "read committed" }, { search_path: "public" }, { timezone: "Europe/Moscow" }, { date_style: "ISO, MDY" }, { server_version: "170000" }, { statement_timeout: "0" }, { statement_timeout: "60s" }]) await assert.rejects(captureMigrationSchema(client(fixture(), row => Object.assign(row, replacement))), MigrationSchemaInventoryError) +}) +await test("backend/transaction/session changes midcapture refuse", async () => { + for (const replacement of [{ backend_pid: "72" }, { transaction_started: "2026-10-25 10:01:00+00" }, { server_version: "180001" }]) await assert.rejects(captureMigrationSchema(client(fixture(), (row, _section, index) => { if (index > 0) Object.assign(row, replacement) })), MigrationSchemaInventoryError) +}) +await test("explicit control exclusion is exact and cannot hide unknown namespaces", async () => { + const data = fixture(); data.namespaces.push({ namespace: "bootstrap_control", owner: "postgres", acl: null, comment: null }) + await assert.rejects(snapshot(data), MigrationSchemaInventoryError) + const observed = await captureMigrationSchema(client(data), { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["bootstrap_control"] } }) + assert.throws(() => compareMigrationSchemas(expected, observed), MigrationSchemaInventoryError) +}) +await test("invalid scope and expired deadline dispatch zero SQL", async () => { + for (const options of [{ scope: { applicationNamespaces: ["public;DROP"] } }, { scope: { applicationNamespaces: ["public"], excludedControlNamespaces: ["public"] } }, { deadlineAt: Date.now() - 1 }]) { + const sql = client(fixture()); await assert.rejects(captureMigrationSchema(sql, options), MigrationSchemaInventoryError); assert.equal(sql.calls.length, 0) + } +}) +function withDroppedSlot(data: Data): Data { + data.columns[1].position = 3 + data.columns.push(column("servers", "........pg.dropped.2........", { position: 2, dropped: true, type: null, identity: "", not_null: false })) + return data +} +await test("v3 tombstone compaction preserves visible order and exact raw/physical evidence", async () => { + const compact = await snapshot(fixture()), gapped = await snapshot(withDroppedSlot(fixture())) + assert.equal(gapped.format, "pg18-application-schema-v3") + assert.equal(compareMigrationSchemas(gapped, compact).equal, true) + assert.equal(gapped.sections.columns.find(c => c.name === "computed")?.position, 3) + assert(gapped.sections.columns.some(c => c.dropped === true)) + assert.notEqual(gapped.observedPhysical.sha256, compact.observedPhysical.sha256) + const legacy = structuredClone(compact) as unknown as { format: string } + legacy.format = "pg18-application-schema-v2" + assert.throws(() => compareMigrationSchemas(legacy as never, legacy as never), /format\/version/) +}) +await test("v3 normalizes every indkey key and INCLUDE slot by live column identity while expression0 remains distinct", async () => { + const compact = fixture(), gapped = withDroppedSlot(fixture()) + for (const [data, keys] of [[compact, "2 0 1"], [gapped, "3 0 1"]] as const) Object.assign(data.indexes[0], { + keys, attributes: 3, key_attributes: 2, expressions: "(id + 1)", key_options: "0 0", + definition: "CREATE UNIQUE INDEX servers_pkey ON public.servers (computed,((id + 1))) INCLUDE(id)" + }) + const before = await snapshot(gapped), after = await snapshot(compact) + assert.equal(compareMigrationSchemas(before, after).equal, true) + assert.equal(before.sections.indexes[0].keys, "3 0 1") + assert(JSON.parse(before.canonical).sections.indexes[0].keys.includes("expression")) + for (const keys of ["1 0 2", "2 0 2", "2 1 1"]) { + const changed = structuredClone(compact); changed.indexes[0].keys = keys + const observed = await snapshot(changed) + assert.throws(() => compareMigrationSchemas(after, observed), MigrationSchemaInventoryError) + } +}) +await test("v3 rejects dangling/dropped/system attnums, expression ambiguity, duplicate identity and vector arity", async () => { + for (const keys of ["2", "99", "-1", "0", "1 1"]) { + const data = withDroppedSlot(fixture()); data.indexes[0].keys = keys + await assert.rejects(snapshot(data), MigrationSchemaInventoryError) + } + const duplicatePosition = fixture(); duplicatePosition.columns[1].position = 1 + await assert.rejects(snapshot(duplicatePosition), MigrationSchemaInventoryError) + const duplicateName = fixture(); duplicateName.columns[1].name = "id" + await assert.rejects(snapshot(duplicateName), MigrationSchemaInventoryError) +}) +await test("v3 visible reorder is refused even when index still names the same logical column", async () => { + const data = fixture(); data.columns[0].position = 2; data.columns[1].position = 1; data.indexes[0].keys = "2" + const observed = await snapshot(data) + assert.throws(() => compareMigrationSchemas(expected, observed), MigrationSchemaInventoryError) +}) +await test("v3 drop/readd same middle name cannot hide changed visible order", async () => { + const original = fixture(); original.columns.push(column("servers", "tail", { position: 3, identity: "" })) + const changed = withDroppedSlot(fixture()); changed.columns.find(c => c.name === "computed")!.position = 4 + changed.columns.push(column("servers", "tail", { position: 3, identity: "" })) + const before = await snapshot(original), after = await snapshot(changed) + assert.throws(() => compareMigrationSchemas(before, after), MigrationSchemaInventoryError) +}) +await test("v3 retains inheritance order and every live type/default/collation/identity/generated/index expression/INCLUDE field", async () => { + for (const mutate of [ + (data: Data) => { data.inheritance[0].position = 2 }, + (data: Data) => { data.columns[1].type = "integer" }, + (data: Data) => { data.columns[1].collation = "pg_catalog.C" }, + (data: Data) => { data.columns[1].default_expression = "(id + 9)" }, + (data: Data) => { data.columns[1].generated = "v" }, + (data: Data) => { data.columns[0].identity = "d" }, + (data: Data) => { data.indexes[0].expressions = "(id + 7)" }, + (data: Data) => { data.indexes[0].definition += " INCLUDE(computed)" }, + ]) { const data = fixture(); mutate(data); const observed = await snapshot(data); assert.throws(() => compareMigrationSchemas(expected, observed), MigrationSchemaInventoryError) } +}) +await test("v3 PostgreSQL CHECK restore rendering preserves logical equality and strict original raw definition fingerprint", async () => { + const make = (definition: string, restore = "CHECK (id > 0 AND id < 10)") => { + const data = fixture(); data.constraints.push({ ...data.constraints[0], name: "owned_check", kind: "c", no_inherit: false, definition, restore_definition: restore }); return data + } + const before = await snapshot(make("CHECK (((id > 0) AND (id < 10)))")), after = await snapshot(make("CHECK ((id > 0 AND id < 10))")) + assert.equal(compareMigrationSchemas(before, after).equal, true) + assert.notEqual(before.observedRaw.sha256, after.observedRaw.sha256) + assert.equal(before.observedPhysical.sha256, after.observedPhysical.sha256) + const tampered = structuredClone(before); tampered.sections.constraints.find(c => c.name === "owned_check")!.definition = "tampered raw definition" + assert.throws(() => compareMigrationSchemas(before, tampered), /raw catalog/) + for (const restore of ["CHECK (id > 0 OR id < 10)", "CHECK (id >= 0 AND id < 10)", "CHECK (id IS NULL AND id < 10)"]) { + const changed = await snapshot(make(restore, restore)); assert.throws(() => compareMigrationSchemas(before, changed), MigrationSchemaInventoryError) + } + const unvalidated = make("CHECK (id > 0 AND id < 10)"); unvalidated.constraints.find(c => c.name === "owned_check")!.validated = false + const changed = await snapshot(unvalidated); assert.throws(() => compareMigrationSchemas(before, changed), MigrationSchemaInventoryError) +}) +console.log(`migration-schema-inventory v3: ${cases} inert raw-catalog capture/comparison scenarios passed; no configured SQL`) + diff --git a/backend/src/db/migration-schema-inventory.ts b/backend/src/db/migration-schema-inventory.ts new file mode 100644 index 00000000..80a59fc3 --- /dev/null +++ b/backend/src/db/migration-schema-inventory.ts @@ -0,0 +1,474 @@ +import { createHash } from "node:crypto" + +/** Read-only evidence capture. No configured pool, schema mutation, enrollment + * decision, attestation/signing or dispatcher import exists in this module. */ +export interface SchemaInventoryClient { query(sql: string, values?: unknown[]): Promise<{ rows: Record[] }> } +export interface SchemaInventoryScope { applicationNamespaces: readonly string[]; excludedControlNamespaces?: readonly string[] } +export class MigrationSchemaInventoryError extends Error { readonly code = "MIGRATION_SCHEMA_REFUSED" } +type Json = null | boolean | number | string | Json[] | { [key: string]: Json } +function fail(message: string): never { throw new MigrationSchemaInventoryError(message) } +const digest = (value: string) => createHash("sha256").update(value).digest("hex") +const acl = (value: string) => `CASE WHEN ${value} IS NULL THEN NULL ELSE ARRAY(SELECT x::text FROM pg_catalog.unnest(${value}) x ORDER BY x::text) END` +const options = (value: string) => `CASE WHEN ${value} IS NULL THEN NULL ELSE ARRAY(SELECT x FROM pg_catalog.unnest(${value}) x ORDER BY x) END` +const application = "n.nspname=ANY($1::text[])" +const rowLimit = 20000 +const metadata = `pg_catalog.pg_backend_pid()::text AS backend_pid, + pg_catalog.transaction_timestamp()::text AS transaction_started, + pg_catalog.current_setting('transaction_read_only') AS read_only, + pg_catalog.current_setting('transaction_isolation') AS isolation, + pg_catalog.current_setting('search_path') AS search_path, + pg_catalog.current_setting('TimeZone') AS timezone, + pg_catalog.current_setting('DateStyle') AS date_style, + pg_catalog.current_setting('standard_conforming_strings') AS standard_strings, + pg_catalog.current_setting('server_version_num') AS server_version, + pg_catalog.current_setting('statement_timeout') AS statement_timeout` +const envelope = (section: string, sql: string, limit = rowLimit) => `/* migration-schema:${section} */ WITH captured AS MATERIALIZED (${sql} LIMIT ${limit + 1}) SELECT ${metadata}, + CASE WHEN (SELECT count(*)>${limit} OR COALESCE(max(pg_catalog.octet_length(pg_catalog.to_jsonb(captured)::text)),0)>1048576 OR COALESCE(sum(pg_catalog.octet_length(pg_catalog.to_jsonb(captured)::text)),0)>33554432 FROM captured) + THEN NULL ELSE COALESCE((SELECT pg_catalog.jsonb_agg(pg_catalog.to_jsonb(captured)) FROM captured),'[]'::jsonb) END AS objects` +const unsupportedEventTriggers = "UNION ALL SELECT 'event-trigger',e.evtname FROM pg_catalog.pg_event_trigger e" +const controlEventTriggersSql = envelope("control-event-triggers", `SELECT e.evtname AS name, e.evtevent AS event, + pg_catalog.pg_get_userbyid(e.evtowner) AS owner, e.evtenabled::text AS enabled, + ${options("e.evttags")} AS tags, pg_catalog.obj_description(e.oid, 'pg_event_trigger') AS comment, + n.nspname AS function_namespace, p.proname AS function_name, + pg_catalog.pg_get_function_identity_arguments(p.oid) AS function_arguments + FROM pg_catalog.pg_event_trigger e + JOIN pg_catalog.pg_proc p ON p.oid=e.evtfoid + JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace`, 1024) + +/** Fixed catalog contract: only SELECTs, only namespace names as bound values. + * LIMIT+1 overflow refuses; nothing is silently truncated or normalized away. + * Caller configures one checked-out PG18 repeatable-read READ ONLY transaction + * with search_path=pg_catalog, TimeZone=UTC, DateStyle='ISO, YMD', bounded + * statement_timeout before capture. Capture never SETs/BEGINs/COMMITs for it. */ +export const MIGRATION_SCHEMA_QUERIES = Object.freeze({ + database: envelope("database", `SELECT pg_catalog.pg_encoding_to_char(d.encoding) AS encoding,d.datcollate AS collate,d.datctype AS ctype, + d.datlocprovider::text AS locale_provider,pg_catalog.to_jsonb(d)->>'datlocale' AS locale,pg_catalog.to_jsonb(d)->>'daticurules' AS locale_rules,d.datcollversion AS collation_version, + ts.spcname AS default_tablespace,pg_catalog.shobj_description(d.oid,'pg_database') AS comment, + pg_catalog.pg_get_userbyid(d.datdba) AS owner,${acl("d.datacl")} AS acl FROM pg_catalog.pg_database d JOIN pg_catalog.pg_tablespace ts ON ts.oid=d.dattablespace WHERE d.datname=pg_catalog.current_database()`), + namespaces: envelope("namespaces", `SELECT n.nspname AS namespace,pg_catalog.pg_get_userbyid(n.nspowner) AS owner,${acl("n.nspacl")} AS acl,pg_catalog.obj_description(n.oid,'pg_namespace') AS comment + FROM pg_catalog.pg_namespace n WHERE n.nspname !~ '^pg_' AND n.nspname<>'information_schema'`), + relations: envelope("relations", `SELECT n.nspname AS namespace,c.relname AS name,c.relkind::text AS kind,c.relpersistence::text AS persistence, + pg_catalog.pg_get_userbyid(c.relowner) AS owner,${acl("c.relacl")} AS acl,${options("c.reloptions")} AS options,pg_catalog.obj_description(c.oid,'pg_class') AS comment, + am.amname AS access_method,t.spcname AS tablespace,${options("toast.reloptions")} AS toast_options, + CASE WHEN c.reloftype=0 THEN NULL ELSE pg_catalog.format_type(c.reloftype,NULL) END AS of_type, + c.relrowsecurity AS row_security,c.relforcerowsecurity AS force_row_security, + c.relreplident::text AS replica_identity,c.relispartition AS is_partition,c.relispopulated AS populated, + CASE WHEN c.relkind='p' THEN pg_catalog.pg_get_partkeydef(c.oid) ELSE NULL END AS partition_key, + CASE WHEN c.relispartition THEN pg_catalog.pg_get_expr(c.relpartbound,c.oid,false) ELSE NULL END AS partition_bound + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_catalog.pg_am am ON am.oid=c.relam LEFT JOIN pg_catalog.pg_tablespace t ON t.oid=c.reltablespace LEFT JOIN pg_catalog.pg_class toast ON toast.oid=c.reltoastrelid WHERE ${application}`), + columns: envelope("columns", `SELECT n.nspname AS namespace,c.relname AS relation,a.attnum::int AS position,a.attname AS name,a.attisdropped AS dropped, + CASE WHEN a.atttypid=0 THEN NULL ELSE pg_catalog.format_type(a.atttypid,a.atttypmod) END AS type,a.atttypmod AS type_modifier, + a.attndims::int AS dimensions,a.attnotnull AS not_null,a.attidentity::text AS identity,a.attgenerated::text AS generated, + a.attstorage::text AS storage,a.attcompression::text AS compression,a.attstattarget::int AS statistics_target, + a.atthasmissing AS has_missing,a.attmissingval::text AS missing_value,${acl("a.attacl")} AS acl,${options("a.attoptions")} AS options,pg_catalog.col_description(c.oid,a.attnum) AS comment, + CASE WHEN co.oid IS NULL THEN NULL ELSE cn.nspname||'.'||co.collname END AS collation, + pg_catalog.pg_get_expr(ad.adbin,ad.adrelid,false) AS default_expression,a.attislocal AS is_local,a.attinhcount::int AS inherited_count + FROM pg_catalog.pg_attribute a JOIN pg_catalog.pg_class c ON c.oid=a.attrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN pg_catalog.pg_attrdef ad ON ad.adrelid=a.attrelid AND ad.adnum=a.attnum + LEFT JOIN pg_catalog.pg_collation co ON co.oid=a.attcollation LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid=co.collnamespace + WHERE ${application} AND a.attnum>0`, 100000), + constraints: envelope("constraints", `SELECT n.nspname AS namespace,c.conname AS name,r.relname AS relation,t.typname AS domain,c.contype::text AS kind, + c.condeferrable AS deferrable,c.condeferred AS deferred,c.convalidated AS validated,c.conislocal AS is_local,c.coninhcount::int AS inherited_count,pg_catalog.obj_description(c.oid,'pg_constraint') AS comment, + c.connoinherit AS no_inherit,pg_catalog.to_jsonb(c)->'conenforced' AS enforced,pg_catalog.to_jsonb(c)->'conperiod' AS period,pg_catalog.pg_get_constraintdef(c.oid,false) AS definition, + CASE WHEN c.contype='c' THEN pg_catalog.pg_get_constraintdef(c.oid,true) ELSE NULL END AS restore_definition, + pn.nspname AS parent_namespace,pr.relname AS parent_relation,pc.conname AS parent_constraint, + fn.nspname AS foreign_namespace,fr.relname AS foreign_relation + FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_namespace n ON n.oid=c.connamespace + LEFT JOIN pg_catalog.pg_class r ON r.oid=c.conrelid LEFT JOIN pg_catalog.pg_type t ON t.oid=c.contypid + LEFT JOIN pg_catalog.pg_constraint pc ON pc.oid=c.conparentid LEFT JOIN pg_catalog.pg_class pr ON pr.oid=pc.conrelid LEFT JOIN pg_catalog.pg_namespace pn ON pn.oid=pr.relnamespace + LEFT JOIN pg_catalog.pg_class fr ON fr.oid=c.confrelid LEFT JOIN pg_catalog.pg_namespace fn ON fn.oid=fr.relnamespace WHERE ${application}`), + indexes: envelope("indexes", `SELECT n.nspname AS namespace,ic.relname AS name,tc.relname AS relation, + pg_catalog.pg_get_indexdef(i.indexrelid) AS definition,i.indisunique AS unique_index,i.indisprimary AS primary_index, + i.indisexclusion AS exclusion,i.indimmediate AS immediate,i.indisclustered AS clustered,i.indisvalid AS valid,i.indisready AS ready,i.indislive AS live, + i.indisreplident AS replica_identity,i.indcheckxmin AS check_xmin,i.indnullsnotdistinct AS nulls_not_distinct,i.indnatts::int AS attributes,i.indnkeyatts::int AS key_attributes, + ARRAY(SELECT onsp.nspname||'.'||op.opcname FROM pg_catalog.unnest(i.indclass) WITH ORDINALITY x(opid,pos) JOIN pg_catalog.pg_opclass op ON op.oid=x.opid JOIN pg_catalog.pg_namespace onsp ON onsp.oid=op.opcnamespace ORDER BY x.pos) AS operator_classes, + ARRAY(SELECT COALESCE(cn.nspname||'.'||co.collname,'') FROM pg_catalog.unnest(i.indcollation) WITH ORDINALITY x(colid,pos) LEFT JOIN pg_catalog.pg_collation co ON co.oid=x.colid LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid=co.collnamespace ORDER BY x.pos) AS collations, + i.indkey::text AS keys,i.indoption::text AS key_options,pg_catalog.pg_get_expr(i.indexprs,i.indrelid,false) AS expressions,pg_catalog.pg_get_expr(i.indpred,i.indrelid,false) AS predicate + FROM pg_catalog.pg_index i JOIN pg_catalog.pg_class ic ON ic.oid=i.indexrelid JOIN pg_catalog.pg_namespace n ON n.oid=ic.relnamespace + JOIN pg_catalog.pg_class tc ON tc.oid=i.indrelid WHERE ${application}`), + inheritance: envelope("inheritance", `SELECT n.nspname AS namespace,c.relname AS child,pn.nspname AS parent_namespace,p.relname AS parent,i.inhseqno::int AS position,i.inhdetachpending AS detach_pending + FROM pg_catalog.pg_inherits i JOIN pg_catalog.pg_class c ON c.oid=i.inhrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + JOIN pg_catalog.pg_class p ON p.oid=i.inhparent JOIN pg_catalog.pg_namespace pn ON pn.oid=p.relnamespace WHERE ${application}`), + sequences: envelope("sequences", `SELECT n.nspname AS namespace,c.relname AS name,pg_catalog.format_type(s.seqtypid,NULL) AS type, + s.seqstart::text AS start,s.seqincrement::text AS increment,s.seqmax::text AS maximum,s.seqmin::text AS minimum,s.seqcache::text AS cache,s.seqcycle AS cycle, + own.namespace AS owner_namespace,own.relation AS owner_relation,own.column_name AS owner_column,own.dependency AS owner_dependency + FROM pg_catalog.pg_sequence s JOIN pg_catalog.pg_class c ON c.oid=s.seqrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + LEFT JOIN LATERAL (SELECT rn.nspname AS namespace,r.relname AS relation,a.attname AS column_name,d.deptype::text AS dependency + FROM pg_catalog.pg_depend d JOIN pg_catalog.pg_class r ON r.oid=d.refobjid JOIN pg_catalog.pg_namespace rn ON rn.oid=r.relnamespace + JOIN pg_catalog.pg_attribute a ON a.attrelid=r.oid AND a.attnum=d.refobjsubid + WHERE d.classid='pg_catalog.pg_class'::regclass AND d.objid=c.oid AND d.refclassid='pg_catalog.pg_class'::regclass AND d.deptype IN ('a','i')) own ON true + WHERE ${application}`), + views: envelope("views", `SELECT n.nspname AS namespace,c.relname AS name,c.relkind::text AS kind,pg_catalog.pg_get_viewdef(c.oid,false) AS definition + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE ${application} AND c.relkind IN ('v','m')`), + types: envelope("types", `SELECT n.nspname AS namespace,t.typname AS name,t.typtype::text AS kind,t.typcategory::text AS category, + pg_catalog.pg_get_userbyid(t.typowner) AS owner,${acl("t.typacl")} AS acl,pg_catalog.obj_description(t.oid,'pg_type') AS comment,t.typlen::int AS length,t.typbyval AS by_value,t.typalign::text AS alignment,t.typstorage::text AS storage, + t.typnotnull AS not_null,t.typispreferred AS preferred,t.typisdefined AS defined,t.typdelim::text AS delimiter,t.typndims::int AS dimensions, + t.typinput::regprocedure::text AS input_function,t.typoutput::regprocedure::text AS output_function,t.typreceive::regprocedure::text AS receive_function,t.typsend::regprocedure::text AS send_function, + t.typmodin::regprocedure::text AS modifier_input,t.typmodout::regprocedure::text AS modifier_output,t.typanalyze::regprocedure::text AS analyze_function,t.typsubscript::regprocedure::text AS subscript_function, + CASE WHEN t.typbasetype=0 THEN NULL ELSE pg_catalog.format_type(t.typbasetype,t.typtypmod) END AS base_type, + CASE WHEN t.typelem=0 THEN NULL ELSE pg_catalog.format_type(t.typelem,NULL) END AS element_type, + rn.nspname AS relation_namespace,r.relname AS relation,pg_catalog.pg_get_expr(t.typdefaultbin,0,false) AS default_expression,t.typdefault AS default_literal, + CASE WHEN co.oid IS NULL THEN NULL ELSE cn.nspname||'.'||co.collname END AS collation, + COALESCE((SELECT pg_catalog.jsonb_agg(e.enumlabel ORDER BY e.enumsortorder) FROM pg_catalog.pg_enum e WHERE e.enumtypid=t.oid),'[]'::jsonb) AS enum_labels, + CASE WHEN rg.rngtypid IS NULL THEN NULL ELSE pg_catalog.jsonb_build_object('subtype',pg_catalog.format_type(rg.rngsubtype,NULL),'multirange',pg_catalog.format_type(rg.rngmultitypid,NULL),'canonical',rg.rngcanonical::regprocedure::text,'subdiff',rg.rngsubdiff::regprocedure::text,'opclass',opn.nspname||'.'||op.opcname,'collation',rcn.nspname||'.'||rc.collname) END AS range_definition + FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid=t.typnamespace + LEFT JOIN pg_catalog.pg_class r ON r.oid=t.typrelid LEFT JOIN pg_catalog.pg_namespace rn ON rn.oid=r.relnamespace + LEFT JOIN pg_catalog.pg_collation co ON co.oid=t.typcollation LEFT JOIN pg_catalog.pg_namespace cn ON cn.oid=co.collnamespace + LEFT JOIN pg_catalog.pg_range rg ON rg.rngtypid=t.oid OR rg.rngmultitypid=t.oid LEFT JOIN pg_catalog.pg_opclass op ON op.oid=rg.rngsubopc LEFT JOIN pg_catalog.pg_namespace opn ON opn.oid=op.opcnamespace + LEFT JOIN pg_catalog.pg_collation rc ON rc.oid=rg.rngcollation LEFT JOIN pg_catalog.pg_namespace rcn ON rcn.oid=rc.collnamespace WHERE ${application}`), + functions: envelope("functions", `SELECT n.nspname AS namespace,p.proname AS name,pg_catalog.pg_get_function_identity_arguments(p.oid) AS arguments,p.prokind::text AS kind, + l.lanname AS language,pg_catalog.pg_get_userbyid(p.proowner) AS owner,${acl("p.proacl")} AS acl,pg_catalog.obj_description(p.oid,'pg_proc') AS comment, + CASE WHEN p.prokind='f' THEN pg_catalog.pg_get_functiondef(p.oid) ELSE NULL END AS definition,p.probin AS external_library, + p.prosecdef AS security_definer,p.proleakproof AS leakproof,p.proisstrict AS strict,p.provolatile::text AS volatility,p.proparallel::text AS parallel, + p.procost::text AS cost,p.prorows::text AS rows,p.prosupport::regprocedure::text AS support,${options("p.proconfig")} AS configuration + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace JOIN pg_catalog.pg_language l ON l.oid=p.prolang WHERE ${application}`), + triggers: envelope("triggers", `SELECT n.nspname AS namespace,c.relname AS relation, + CASE WHEN t.tgisinternal THEN 'constraint:'||COALESCE(tcon.conname,'')||':'||fn.nspname||'.'||p.proname||':'||t.tgtype::text ELSE t.tgname END AS name, + t.tgisinternal AS internal,t.tgenabled::text AS enabled,t.tgtype::int AS type,pg_catalog.obj_description(t.oid,'pg_trigger') AS comment, + CASE WHEN t.tgisinternal THEN pg_catalog.replace(pg_catalog.pg_get_triggerdef(t.oid,false),pg_catalog.format('CREATE CONSTRAINT TRIGGER %I',t.tgname),'CREATE CONSTRAINT TRIGGER ') ELSE pg_catalog.pg_get_triggerdef(t.oid,false) END AS definition, + fn.nspname AS function_namespace,p.proname AS function_name,pg_catalog.pg_get_function_identity_arguments(p.oid) AS function_arguments, + pg_catalog.encode(t.tgargs,'hex') AS arguments,CASE WHEN t.tgqual IS NULL THEN NULL ELSE pg_catalog.pg_get_triggerdef(t.oid,false) END AS condition, + t.tgdeferrable AS deferrable,t.tginitdeferred AS initially_deferred,t.tgoldtable AS old_table,t.tgnewtable AS new_table, + pc.relname AS parent_relation, + CASE WHEN pt.tgisinternal THEN 'constraint:'||COALESCE(pcon.conname,'')||':'||pfn.nspname||'.'||pf.proname||':'||pt.tgtype::text ELSE pt.tgname END AS parent_trigger, + tcon.conname AS constraint_name + FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + JOIN pg_catalog.pg_proc p ON p.oid=t.tgfoid JOIN pg_catalog.pg_namespace fn ON fn.oid=p.pronamespace + LEFT JOIN pg_catalog.pg_constraint tcon ON tcon.oid=t.tgconstraint + LEFT JOIN pg_catalog.pg_trigger pt ON pt.oid=t.tgparentid LEFT JOIN pg_catalog.pg_class pc ON pc.oid=pt.tgrelid + LEFT JOIN pg_catalog.pg_proc pf ON pf.oid=pt.tgfoid LEFT JOIN pg_catalog.pg_namespace pfn ON pfn.oid=pf.pronamespace LEFT JOIN pg_catalog.pg_constraint pcon ON pcon.oid=pt.tgconstraint WHERE ${application}`), + unsupported: envelope("unsupported", `SELECT 'extension' AS kind,e.extname AS identity FROM pg_catalog.pg_extension e WHERE e.extname<>'plpgsql' + UNION ALL SELECT 'unsafe-type-io',n.nspname||'.'||t.typname FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid=t.typnamespace + LEFT JOIN pg_catalog.pg_proc ip ON ip.oid=t.typinput LEFT JOIN pg_catalog.pg_namespace ins ON ins.oid=ip.pronamespace + LEFT JOIN pg_catalog.pg_proc op ON op.oid=t.typoutput LEFT JOIN pg_catalog.pg_namespace ons ON ons.oid=op.pronamespace + WHERE ${application} AND (t.typtype NOT IN ('c','d','e','r','m') AND NOT(t.typtype='b' AND t.typcategory='A' AND t.typelem<>0 AND ip.proname='array_in' AND op.proname='array_out') OR ins.nspname IS DISTINCT FROM 'pg_catalog' OR ons.nspname IS DISTINCT FROM 'pg_catalog') + UNION ALL SELECT 'unsupported-routine',n.nspname||'.'||p.proname FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace JOIN pg_catalog.pg_language l ON l.oid=p.prolang WHERE ${application} AND (p.prokind<>'f' OR l.lanname NOT IN ('sql','plpgsql') OR p.probin IS NOT NULL) + UNION ALL SELECT 'unsupported-relation',n.nspname||'.'||c.relname FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace LEFT JOIN pg_catalog.pg_am am ON am.oid=c.relam WHERE ${application} AND (c.relkind NOT IN ('r','p','i','I','S','v','m','c') OR c.relpersistence NOT IN ('p','u') OR am.amname IS NOT NULL AND am.amname NOT IN ('heap','btree','hash','gist','gin','spgist','brin')) + ${unsupportedEventTriggers} + UNION ALL SELECT 'publication',p.pubname FROM pg_catalog.pg_publication p + UNION ALL SELECT 'subscription',s.subname FROM pg_catalog.pg_subscription s + UNION ALL SELECT 'policy',n.nspname||'.'||c.relname||'.'||p.polname FROM pg_catalog.pg_policy p JOIN pg_catalog.pg_class c ON c.oid=p.polrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE ${application} + UNION ALL SELECT 'rewrite-rule',n.nspname||'.'||c.relname||'.'||r.rulename FROM pg_catalog.pg_rewrite r JOIN pg_catalog.pg_class c ON c.oid=r.ev_class JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE ${application} AND r.rulename<>'_RETURN' + UNION ALL SELECT 'foreign-table',n.nspname||'.'||c.relname FROM pg_catalog.pg_foreign_table f JOIN pg_catalog.pg_class c ON c.oid=f.ftrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE ${application} + UNION ALL SELECT 'extended-statistics',n.nspname||'.'||s.stxname FROM pg_catalog.pg_statistic_ext s JOIN pg_catalog.pg_namespace n ON n.oid=s.stxnamespace WHERE ${application} + UNION ALL SELECT 'collation',n.nspname||'.'||c.collname FROM pg_catalog.pg_collation c JOIN pg_catalog.pg_namespace n ON n.oid=c.collnamespace WHERE ${application} + UNION ALL SELECT 'conversion',n.nspname||'.'||c.conname FROM pg_catalog.pg_conversion c JOIN pg_catalog.pg_namespace n ON n.oid=c.connamespace WHERE ${application} + UNION ALL SELECT 'operator',n.nspname||'.'||o.oprname FROM pg_catalog.pg_operator o JOIN pg_catalog.pg_namespace n ON n.oid=o.oprnamespace WHERE ${application} + UNION ALL SELECT 'operator-class',n.nspname||'.'||o.opcname FROM pg_catalog.pg_opclass o JOIN pg_catalog.pg_namespace n ON n.oid=o.opcnamespace WHERE ${application} + UNION ALL SELECT 'operator-family',n.nspname||'.'||o.opfname FROM pg_catalog.pg_opfamily o JOIN pg_catalog.pg_namespace n ON n.oid=o.opfnamespace WHERE ${application} + UNION ALL SELECT 'text-search-configuration',n.nspname||'.'||o.cfgname FROM pg_catalog.pg_ts_config o JOIN pg_catalog.pg_namespace n ON n.oid=o.cfgnamespace WHERE ${application} + UNION ALL SELECT 'text-search-dictionary',n.nspname||'.'||o.dictname FROM pg_catalog.pg_ts_dict o JOIN pg_catalog.pg_namespace n ON n.oid=o.dictnamespace WHERE ${application} + UNION ALL SELECT 'text-search-parser',n.nspname||'.'||o.prsname FROM pg_catalog.pg_ts_parser o JOIN pg_catalog.pg_namespace n ON n.oid=o.prsnamespace WHERE ${application} + UNION ALL SELECT 'text-search-template',n.nspname||'.'||o.tmplname FROM pg_catalog.pg_ts_template o JOIN pg_catalog.pg_namespace n ON n.oid=o.tmplnamespace WHERE ${application} + UNION ALL SELECT 'default-acl',COALESCE(n.nspname,'global')||':'||a.defaclobjtype::text FROM pg_catalog.pg_default_acl a LEFT JOIN pg_catalog.pg_namespace n ON n.oid=a.defaclnamespace WHERE a.defaclnamespace=0 OR ${application} + UNION ALL SELECT 'security-label',s.provider||':'||s.classoid::regclass::text FROM pg_catalog.pg_seclabel s + UNION ALL SELECT 'shared-security-label',s.provider||':'||s.classoid::regclass::text FROM pg_catalog.pg_shseclabel s + UNION ALL SELECT 'transform',t.trftype::regtype::text FROM pg_catalog.pg_transform t + UNION ALL SELECT 'cast',s.typname||'->'||t.typname FROM pg_catalog.pg_cast c JOIN pg_catalog.pg_type s ON s.oid=c.castsource JOIN pg_catalog.pg_namespace sn ON sn.oid=s.typnamespace JOIN pg_catalog.pg_type t ON t.oid=c.casttarget JOIN pg_catalog.pg_namespace tn ON tn.oid=t.typnamespace WHERE sn.nspname=ANY($1::text[]) OR tn.nspname=ANY($1::text[]) + UNION ALL SELECT 'foreign-wrapper',f.fdwname FROM pg_catalog.pg_foreign_data_wrapper f + UNION ALL SELECT 'foreign-server',s.srvname FROM pg_catalog.pg_foreign_server s + UNION ALL SELECT 'large-object',m.oid::text FROM pg_catalog.pg_largeobject_metadata m + UNION ALL SELECT 'external-schema-dependency',i.identity FROM pg_catalog.pg_depend d CROSS JOIN LATERAL pg_catalog.pg_identify_object(d.refclassid,d.refobjid,d.refobjsubid) i + WHERE i.schema IS NOT NULL AND i.schema !~ '^pg_' AND i.schema<>'information_schema' AND NOT(i.schema=ANY($1::text[])) AND ( + d.classid='pg_catalog.pg_class'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE c.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_proc'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace WHERE p.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_type'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_type t JOIN pg_catalog.pg_namespace n ON n.oid=t.typnamespace WHERE t.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_rewrite'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_rewrite r JOIN pg_catalog.pg_class c ON c.oid=r.ev_class JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE r.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_constraint'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_constraint c JOIN pg_catalog.pg_namespace n ON n.oid=c.connamespace WHERE c.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_trigger'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE t.oid=d.objid AND ${application}) + OR d.classid='pg_catalog.pg_attrdef'::regclass AND EXISTS(SELECT 1 FROM pg_catalog.pg_attrdef a JOIN pg_catalog.pg_class c ON c.oid=a.adrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE a.oid=d.objid AND ${application}))`), +}) +export type SchemaSection = keyof typeof MIGRATION_SCHEMA_QUERIES +/** Bind contract belongs to each fixed query, including the two global census + * queries that intentionally have no namespace placeholder. */ +export const MIGRATION_SCHEMA_PARAMETERS = Object.freeze({ + database: "none", namespaces: "none", + relations: "application-namespaces", columns: "application-namespaces", + constraints: "application-namespaces", indexes: "application-namespaces", + inheritance: "application-namespaces", sequences: "application-namespaces", + views: "application-namespaces", types: "application-namespaces", + functions: "application-namespaces", triggers: "application-namespaces", + unsupported: "application-namespaces", +} satisfies Record) +// Preflight only names/catalog flags before any expression/default deparser: +// unknown custom type output routines must never be invoked just to inspect a +// constant. Caller must separately quiesce concurrent DDL for stable evidence. +const sections: SchemaSection[] = ["database", "namespaces", "unsupported", ...(Object.keys(MIGRATION_SCHEMA_QUERIES) as SchemaSection[]).filter(section => !["database", "namespaces", "unsupported"].includes(section))] +const keys: Record = { + database: "encoding collate ctype locale_provider locale locale_rules collation_version default_tablespace comment owner acl", + namespaces: "namespace owner acl comment", + relations: "namespace name kind persistence owner acl options comment access_method tablespace toast_options of_type row_security force_row_security replica_identity is_partition populated partition_key partition_bound", + columns: "namespace relation position name dropped type type_modifier dimensions not_null identity generated storage compression statistics_target has_missing missing_value acl options comment collation default_expression is_local inherited_count", + constraints: "namespace name relation domain kind deferrable deferred validated is_local inherited_count comment no_inherit enforced period definition restore_definition parent_namespace parent_relation parent_constraint foreign_namespace foreign_relation", + indexes: "namespace name relation definition unique_index primary_index exclusion immediate clustered valid ready live replica_identity check_xmin nulls_not_distinct attributes key_attributes keys key_options operator_classes collations expressions predicate", + inheritance: "namespace child parent_namespace parent position detach_pending", + sequences: "namespace name type start increment maximum minimum cache cycle owner_namespace owner_relation owner_column owner_dependency", + views: "namespace name kind definition", + types: "namespace name kind category owner acl comment length by_value alignment storage not_null preferred defined delimiter dimensions input_function output_function receive_function send_function modifier_input modifier_output analyze_function subscript_function base_type element_type relation_namespace relation default_expression default_literal collation enum_labels range_definition", + functions: "namespace name arguments kind language owner acl comment definition external_library security_definer leakproof strict volatility parallel cost rows support configuration", + triggers: "namespace relation name internal enabled type comment definition function_namespace function_name function_arguments arguments condition deferrable initially_deferred old_table new_table parent_relation parent_trigger constraint_name", + unsupported: "kind identity", +} +function canonical(value: unknown, depth = 0): Json { + if (depth > 12) fail("Catalog JSON depth exceeds bound") + if (value === null || typeof value === "boolean") return value + if (typeof value === "number") { if (!Number.isSafeInteger(value)) fail("Nonintegral/unsafe catalog number"); return value } + if (typeof value === "string") { if (Buffer.byteLength(value) > 1048576 || value.includes("\0")) fail("Catalog string exceeds bound"); return value } + if (Array.isArray(value)) { if (value.length > 100001) fail("Catalog array exceeds bound"); return value.map(item => canonical(item, depth + 1)) } + if (!value || typeof value !== "object" || Object.getPrototypeOf(value) !== Object.prototype || Object.keys(value).length > 100) fail("Invalid catalog JSON value") + return Object.fromEntries(Object.keys(value).sort().map(key => [key, canonical(Reflect.get(value, key), depth + 1)])) +} +function scopeValue(scope: SchemaInventoryScope) { + if (!scope || !Array.isArray(scope.applicationNamespaces) || scope.excludedControlNamespaces !== undefined && !Array.isArray(scope.excludedControlNamespaces)) fail("Missing exact schema scope") + const app = [...scope.applicationNamespaces].sort(), control = [...scope.excludedControlNamespaces ?? []].sort() + if (!app.length || app.length + control.length > 16 || new Set([...app, ...control]).size !== app.length + control.length || [...app, ...control].some(name => !/^[A-Za-z_][A-Za-z_0-9]{0,62}$/.test(name) || /^pg_/i.test(name) || name === "information_schema")) fail("Invalid exact schema scope") + return { applicationNamespaces: app, excludedControlNamespaces: control } +} +function session(row: Record) { + const expected = "backend_pid transaction_started read_only isolation search_path timezone date_style standard_strings server_version statement_timeout objects".split(" ").sort() + if (JSON.stringify(Object.keys(row).sort()) !== JSON.stringify(expected)) fail("Unexpected catalog envelope") + if (typeof row.backend_pid !== "string" || !/^\d+$/.test(row.backend_pid) || typeof row.transaction_started !== "string" || !Number.isFinite(Date.parse(row.transaction_started)) || row.read_only !== "on" || !["repeatable read", "serializable"].includes(String(row.isolation)) || row.search_path !== "pg_catalog" || row.timezone !== "UTC" || row.date_style !== "ISO, YMD" || row.standard_strings !== "on" || typeof row.server_version !== "string" || !/^18\d{4}$/.test(row.server_version)) fail("PG18 fixed-context checked-out readonly snapshot required") + const timeout = /^(\d+(?:\.\d+)?)(ms|s|min)?$/.exec(String(row.statement_timeout)) + const millis = timeout ? Number(timeout[1]) * ({ ms: 1, s: 1000, min: 60000 }[timeout[2] ?? "ms"] ?? 0) : 0 + if (millis < 1 || millis > 10000) fail("Bounded read-only statement timeout required") + return JSON.stringify(Object.fromEntries(expected.filter(key => key !== "objects").map(key => [key, row[key]]))) +} +function rowsFor(section: SchemaSection, raw: unknown) { + if (!Array.isArray(raw) || raw.length > (section === "columns" ? 100000 : rowLimit)) fail("Catalog section overflow: " + section) + const expected = keys[section].split(" ").sort() + const rows = raw.map(value => { + if (!value || typeof value !== "object" || Array.isArray(value) || JSON.stringify(Object.keys(value).sort()) !== JSON.stringify(expected)) fail("Unknown catalog row shape: " + section) + const row = canonical(value) as Record + const booleans = new Set("row_security force_row_security is_partition populated dropped not_null has_missing is_local deferrable deferred validated no_inherit enforced period unique_index primary_index exclusion immediate clustered valid ready live check_xmin nulls_not_distinct detach_pending cycle by_value preferred defined security_definer leakproof strict internal initially_deferred".split(" ")) + const numbers = new Set("position type_modifier dimensions statistics_target inherited_count attributes key_attributes length".split(" ")) + for (const [field, item] of Object.entries(row)) { + if (booleans.has(field) || field === "replica_identity" && section === "indexes") { if (typeof item !== "boolean") fail("Invalid catalog boolean: " + section + "." + field) } + // PG18 pg_attribute.attstattarget is nullable int2: NULL means system + // default, while 0 disables statistics. Preserve that distinction. + else if (section === "columns" && field === "statistics_target") { if (item !== null && typeof item !== "number") fail("Invalid catalog integer: " + section + "." + field) } + else if (numbers.has(field) || field === "type" && section === "triggers") { if (typeof item !== "number") fail("Invalid catalog integer: " + section + "." + field) } + else if (["acl", "options", "toast_options", "configuration", "enum_labels", "operator_classes", "collations"].includes(field)) { if (item !== null && (!Array.isArray(item) || item.some(value => typeof value !== "string"))) fail("Invalid catalog string array") } + else if (field === "range_definition") { if (item !== null && (!item || Array.isArray(item) || typeof item !== "object" || JSON.stringify(Object.keys(item).sort()) !== JSON.stringify(["canonical", "collation", "multirange", "opclass", "subdiff", "subtype"]) || Object.values(item).some(value => value !== null && typeof value !== "string"))) fail("Invalid range definition") } + else if (item !== null && typeof item !== "string") fail("Invalid catalog scalar: " + section + "." + field) + } + for (const field of ["namespace", "name"]) if (field in row && (typeof row[field] !== "string" || !row[field])) fail("Missing catalog object identity") + for (const field of ["acl", "options", "toast_options", "configuration"]) if (Array.isArray(row[field])) { + const values = row[field] as string[] + if (new Set(values.map(value => field === "acl" ? value : value.split("=", 1)[0])).size !== values.length) fail("Duplicate unordered catalog option") + row[field] = [...values].sort() + } + return row + }) + const serialized = rows.map(row => JSON.stringify(row)).sort() + if (new Set(serialized).size !== serialized.length) fail("Duplicate catalog row: " + section) + return serialized.map(value => JSON.parse(value) as Record) +} +function validateSemantics(data: Record[]>, scope: ReturnType) { + if (data.database.length !== 1 || data.unsupported.length) fail("Unsupported database/schema object class") + if (!["b", "c", "i"].includes(String(data.database[0].locale_provider))) fail("Unknown database locale provider") + const namespaces = data.namespaces.map(row => row.namespace).sort() + if (JSON.stringify(namespaces) !== JSON.stringify([...scope.applicationNamespaces, ...scope.excludedControlNamespaces].sort())) fail("Unexpected or missing application/control namespace") + for (const section of sections.filter(section => !["database", "namespaces", "unsupported"].includes(section))) for (const row of data[section]) if (!scope.applicationNamespaces.includes(String(row.namespace))) fail("Catalog row escaped exact namespace scope") + for (const row of data.relations) if (!["r", "p", "i", "I", "S", "v", "m", "c"].includes(String(row.kind)) || !["p", "u"].includes(String(row.persistence)) || !["d", "n", "f", "i"].includes(String(row.replica_identity)) || row.access_method !== null && !["heap", "btree", "hash", "gist", "gin", "spgist", "brin"].includes(String(row.access_method))) fail("Unsupported relation or access method") + for (const row of data.types) if (!["c", "d", "e", "r", "m"].includes(String(row.kind)) && !(row.kind === "b" && row.category === "A" && typeof row.element_type === "string")) fail("Unsupported user base/pseudo type semantics") + for (const row of data.types) if (row.defined !== true || row.kind === "d" && typeof row.base_type !== "string" || ["r", "m"].includes(String(row.kind)) && row.range_definition === null || row.kind === "c" && (row.relation_namespace !== row.namespace || typeof row.relation !== "string")) fail("Incomplete user type semantics") + for (const row of data.functions) if (row.kind !== "f" || !["sql", "plpgsql"].includes(String(row.language)) || !["i", "s", "v"].includes(String(row.volatility)) || !["u", "r", "s"].includes(String(row.parallel)) || typeof row.definition !== "string" || row.external_library !== null) fail("Unsupported procedure/aggregate/window/external function semantics") + for (const row of data.columns) if (!Number.isSafeInteger(row.position) || Number(row.position) < 1 || !["", "a", "d"].includes(String(row.identity)) || !["", "s", "v"].includes(String(row.generated))) fail("Unsupported column semantics") + for (const row of data.columns) if (row.has_missing ? typeof row.missing_value !== "string" : row.missing_value !== null) fail("Invalid fast-default physical metadata") + for (const row of data.inheritance) if (!scope.applicationNamespaces.includes(String(row.parent_namespace)) || row.detach_pending !== false) fail("Unsupported external or pending-detach inheritance") + for (const row of data.constraints) { + if (row.kind === "c" ? typeof row.restore_definition !== "string" || !row.restore_definition : row.restore_definition !== null) fail("Missing fixed PostgreSQL CHECK restore definition") + if (row.kind === "t") { + // User constraint triggers have a pg_constraint row as well as the full + // pg_trigger definition captured below. Join by portable identity rather + // than OIDs, which legitimately change after dump/restore. + const triggers = data.triggers.filter(trigger => trigger.namespace === row.namespace && trigger.relation === row.relation && trigger.name === row.name && trigger.constraint_name === row.name) + if (typeof row.relation !== "string" || row.domain !== null || row.foreign_namespace !== null || row.foreign_relation !== null || row.parent_namespace !== null || row.parent_relation !== null || row.parent_constraint !== null || triggers.length !== 1) fail("Unsupported user constraint trigger shape") + const trigger = triggers[0] + if (trigger.internal !== false || trigger.deferrable !== row.deferrable || trigger.initially_deferred !== row.deferred || trigger.parent_relation !== null || trigger.parent_trigger !== null || !scope.applicationNamespaces.includes(String(trigger.function_namespace))) fail("Inconsistent user constraint trigger semantics") + } else if (!["p", "u", "f", "c", "x", "n"].includes(String(row.kind)) || row.foreign_namespace !== null && !scope.applicationNamespaces.includes(String(row.foreign_namespace))) fail("Unsupported external constraint semantics") + } + for (const row of data.triggers) if (!["O", "D", "R", "A"].includes(String(row.enabled)) || row.internal && (typeof row.constraint_name !== "string" || row.function_namespace !== "pg_catalog" || !String(row.function_name).startsWith("RI_FKey_")) || !row.internal && !scope.applicationNamespaces.includes(String(row.function_namespace))) fail("Unsupported internal/external trigger semantics") + for (const row of data.sequences) if (row.owner_namespace !== null && !scope.applicationNamespaces.includes(String(row.owner_namespace))) fail("Unsupported external sequence ownership") + const relationNames = new Set(data.relations.map(row => row.namespace + "." + row.name)) + for (const row of data.columns) if (!relationNames.has(row.namespace + "." + row.relation)) fail("Column without captured relation") + for (const row of data.relations.filter(row => row.kind === "S")) if (!data.sequences.some(sequence => sequence.namespace === row.namespace && sequence.name === row.name)) fail("Missing sequence definition") + for (const row of data.relations.filter(row => ["i", "I"].includes(String(row.kind)))) if (!data.indexes.some(index => index.namespace === row.namespace && index.name === row.name)) fail("Missing index definition") + for (const row of data.relations.filter(row => ["v", "m"].includes(String(row.kind)))) if (!data.views.some(view => view.namespace === row.namespace && view.name === row.name)) fail("Missing view definition") +} +export interface CapturedMigrationSchema { + format: "pg18-application-schema-v3" | "pg18-application-schema-v4" + scope: ReturnType + serverVersion: string + sections: Record[]> + eventTriggers?: Record[] + /** Exact observed fast-default history; independently content-checked, never + * used as proof of row/data-migration equivalence. */ + observedPhysical: { columns: Record[]; canonical: string; sha256: string } + /** v3 binds ALL raw fields, including original CHECK rendering, tombstones, + * numeric index keys and the additive fixed PostgreSQL restore rendering. */ + observedRaw: { format: "pg18-raw-catalog-v1"; sha256: string } + /** v3 restore-portable logical definitions. Raw sections and physical + * fingerprint stay exact; dropped slots/attnums are normalized only here. */ + canonical: string + sha256: string +} +/** Fast-default cached values describe existing tuple history, not the current + * default expression. ALTER ADD DEFAULT now() (pinned migration0018) caches its + * execution time; a table rewrite clears the cache/flag. Neither proves data + * migration equivalence. Keep exact observations separately, never erase them. */ +function canonicalControlEventTriggers(raw: unknown, functions: CapturedMigrationSchema["sections"]["functions"]) { + if (!Array.isArray(raw) || raw.length > 1024) return fail("Bounded control event trigger catalog required") + const names = new Set() + return raw.map(value => { + const row = canonical(value) as Record + if (!row || Object.keys(row).join(" ") !== "comment enabled event function_arguments function_name function_namespace name owner tags" || + typeof row.name !== "string" || !row.name || names.has(row.name) || typeof row.event !== "string" || !row.event || + typeof row.owner !== "string" || !row.owner || !["O", "D", "R", "A"].includes(String(row.enabled)) || + row.comment !== null && typeof row.comment !== "string" || + row.tags !== null && (!Array.isArray(row.tags) || row.tags.some(tag => typeof tag !== "string") || new Set(row.tags).size !== row.tags.length) || + row.function_namespace !== "mmapp_control" || row.function_arguments !== "" || + !functions.some(fn => fn.namespace === row.function_namespace && fn.name === row.function_name && fn.arguments === row.function_arguments)) { + return fail("Unsupported event trigger or uncaptured control function") + } + names.add(row.name) + if (Array.isArray(row.tags)) row.tags = [...row.tags].sort() + return row + }).sort((left, right) => String(left.name) < String(right.name) ? -1 : 1) +} + +function projections(data: CapturedMigrationSchema["sections"], eventTriggers?: Record[]) { + const sortRows = (rows: Record[]) => rows.map(row => JSON.stringify(canonical(row))).sort().map(row => JSON.parse(row) as Record) + const physicalColumns = sortRows(data.columns.map(row => Object.fromEntries(["namespace", "relation", "position", "name", "has_missing", "missing_value"].map(field => [field, row[field]])))) + const physicalCanonical = JSON.stringify(canonical({ columns: physicalColumns })) + const observedRaw = { format: "pg18-raw-catalog-v1" as const, sha256: digest(JSON.stringify(canonical({ + format: "pg18-raw-catalog-v1", sections: data, ...(eventTriggers ? { eventTriggers } : {}), + }))) } + // Never mutate raw catalog rows. A pg_dump/restore legitimately omits dropped + // slots and compacts physical attnums. Visible order remains semantic. + const relationKey = (namespace: Json, relation: Json) => JSON.stringify([namespace, relation]) + const groups = new Map[]>() + for (const row of data.columns) { + const key = relationKey(row.namespace, row.relation), rows = groups.get(key) ?? [] + rows.push(row); groups.set(key, rows) + } + const visibleByAttnum = new Map>() + const columns: Record[] = [] + for (const [key, rows] of groups) { + const ordered = [...rows].sort((a, b) => Number(a.position) - Number(b.position)) + const positions = new Set(), names = new Set(), visible = new Map() + let ordinal = 0 + for (const row of ordered) { + const attnum = Number(row.position) + if (!Number.isSafeInteger(attnum) || attnum < 1 || attnum > 32767 || positions.has(attnum)) fail("Ambiguous physical column position") + positions.add(attnum) + if (row.dropped === true) continue + if (typeof row.name !== "string" || names.has(row.name)) fail("Ambiguous visible column identity") + names.add(row.name); visible.set(attnum, row.name) + columns.push({ ...Object.fromEntries(Object.entries(row).filter(([field]) => field !== "has_missing" && field !== "missing_value")), position: ++ordinal }) + } + visibleByAttnum.set(key, visible) + } + const indexes = data.indexes.map(row => { + // indkey covers key AND INCLUDE attributes. Positive attnums must resolve + // to the exact live column of THIS indexed relation; zero is an expression + // slot, never an absent/dropped column. Definitions/expressions/predicates, + // key order, options, opclasses and collations remain independently pinned. + if (typeof row.keys !== "string" || !/^(?:0|[1-9][0-9]*)(?: (?:0|[1-9][0-9]*))*$/.test(row.keys) || + !Number.isSafeInteger(row.attributes) || Number(row.attributes) < 1 || Number(row.attributes) > 32767) fail("Unsupported index attribute vector") + const tokens = row.keys.split(" ") + if (tokens.length !== row.attributes) fail("Index attribute vector arity mismatch") + const visible = visibleByAttnum.get(relationKey(row.namespace, row.relation)) + const keys = tokens.map(token => { + if (token === "0") { + if (typeof row.expressions !== "string" || !row.expressions) fail("Index expression slot without expression") + return ["expression"] + } + const name = visible?.get(Number(token)) + if (name === undefined) return fail("Dangling or dropped index column reference") + return ["column", name] + }) + return { ...row, keys: JSON.stringify(keys) } + }) + // inheritance.position is inhseqno, not attnum: it and every other catalog + // field/definition stay exact. The v3 marker prevents legacy reinterpretation. + // PostgreSQL itself removes nonsemantic CHECK deparser grouping; no SQL + // regex/rewriting or discarded CHECK is involved. Original false-rendered + // text remains in raw sections and observedRaw, while all constraint flags, + // identity, validation/enforcement and non-CHECK definitions remain exact. + const constraints = data.constraints.map(row => ({ ...Object.fromEntries(Object.entries(row).filter(([field]) => field !== "restore_definition")), definition: row.kind === "c" ? row.restore_definition : row.definition })) + const logical = { ...data, columns: sortRows(columns), indexes: sortRows(indexes), constraints: sortRows(constraints) } + return { logical, observedPhysical: { columns: physicalColumns, canonical: physicalCanonical, sha256: digest(physicalCanonical) }, observedRaw } +} +/** Every query envelope must identify the SAME backend + transaction + deparse + * context. Mutable sequence last_value/statistics/table data are not schema. + * Excluded control namespaces must be explicit and present, never inferred + * from marker/table existence. Their objects are outside this evidence scope. */ +export async function captureMigrationSchema(client: SchemaInventoryClient, options: { + scope?: SchemaInventoryScope; deadlineAt?: number; includeControlEventTriggers?: boolean +} = {}): Promise { + const scope = scopeValue(options.scope ?? { applicationNamespaces: ["public"] }), deadline = options.deadlineAt ?? Date.now() + 60000 + if (options.includeControlEventTriggers && !scope.applicationNamespaces.includes("mmapp_control")) fail("Control event triggers require the control schema in scope") + if (!Number.isSafeInteger(deadline) || deadline <= Date.now() || deadline - Date.now() > 120000) fail("Invalid schema inventory deadline") + const data = {} as Record[]> + let owner: string | undefined, serverVersion = "", size = 0 + for (const section of sections) { + if (Date.now() >= deadline) fail("Schema capture deadline expired") + const parameters = MIGRATION_SCHEMA_PARAMETERS[section] === "none" ? [] : [[...scope.applicationNamespaces]] + let result: Awaited> + const sql = MIGRATION_SCHEMA_QUERIES[section] + try { result = await client.query(section === "unsupported" && options.includeControlEventTriggers ? sql.replace(unsupportedEventTriggers, "") : sql, parameters) } + catch (cause) { throw new MigrationSchemaInventoryError("Catalog query failed: " + section, { cause }) } + if (result.rows.length !== 1) fail("Catalog query envelope unavailable") + const row = result.rows[0], current = session(row) + if (owner !== undefined && owner !== current) fail("Catalog capture changed backend, transaction or deparse context") + owner = current; serverVersion = String(row.server_version) + data[section] = rowsFor(section, row.objects) + if (section === "unsupported" && data.unsupported.length) fail("Unsupported schema semantics refused before detailed deparsing") + if (section === "namespaces" && JSON.stringify(data.namespaces.map(row => row.namespace).sort()) !== JSON.stringify([...scope.applicationNamespaces, ...scope.excludedControlNamespaces].sort())) fail("Unexpected or missing application/control namespace") + size += Buffer.byteLength(JSON.stringify(data[section])); if (size > 33554432 || Date.now() >= deadline) fail("Schema capture byte/deadline bound exceeded") + } + validateSemantics(data, scope) + let eventTriggers: Record[] | undefined + if (options.includeControlEventTriggers) { + const { rows } = await client.query(controlEventTriggersSql) + if (rows.length !== 1 || session(rows[0]) !== owner || Date.now() >= deadline) fail("Control event triggers escaped the catalog snapshot") + const captured = canonicalControlEventTriggers(rows[0].objects, data.functions) + if (size + Buffer.byteLength(JSON.stringify(captured)) > 33554432) fail("Control event trigger byte bound exceeded") + if (captured.length) eventTriggers = captured + } + const format = eventTriggers ? "pg18-application-schema-v4" : "pg18-application-schema-v3" + const value = { format, scope, serverVersion, sections: data, ...(eventTriggers ? { eventTriggers } : {}) } as const + const { logical, observedPhysical, observedRaw } = projections(data, eventTriggers) + const serialized = JSON.stringify(canonical({ ...value, sections: logical })) + return { ...value, observedPhysical, observedRaw, canonical: serialized, sha256: digest(serialized) } +} +/** Recompute and compare captured CONTENT, never trust a supplied boolean/hash. + * Evidence-only equality is not proof that a caller's reference is authorized. + * Equality covers logical schema, not cached tuple history or data migration. + * Reference generation from all36 pinned files belongs to the owner harness. */ +export function compareMigrationSchemas(expected: CapturedMigrationSchema, observed: CapturedMigrationSchema) { + const checked = (snapshot: CapturedMigrationSchema) => { + if (!snapshot || !["pg18-application-schema-v3", "pg18-application-schema-v4"].includes(snapshot.format) || !/^18\d{4}$/.test(snapshot.serverVersion)) fail("Unsupported captured schema format/version") + const eventFormat = snapshot.format === "pg18-application-schema-v4" + const keys = ["canonical", "format", "observedPhysical", "observedRaw", "scope", "sections", "serverVersion", "sha256"] + if (eventFormat) keys.push("eventTriggers") + if (JSON.stringify(Object.keys(snapshot).sort()) !== JSON.stringify(keys.sort())) fail("Unexpected captured schema envelope") + const scope = scopeValue(snapshot.scope) + if (JSON.stringify(Object.keys(snapshot.sections).sort()) !== JSON.stringify([...sections].sort())) fail("Captured schema section set changed") + const data = Object.fromEntries(sections.map(section => [section, rowsFor(section, snapshot.sections[section])])) as Record[]> + validateSemantics(data, scope) + const eventTriggers = eventFormat ? canonicalControlEventTriggers(snapshot.eventTriggers, data.functions) : undefined + if (eventFormat && (!eventTriggers?.length || !scope.applicationNamespaces.includes("mmapp_control"))) fail("Unexpected event trigger scope") + const { logical, observedPhysical, observedRaw } = projections(data, eventTriggers) + if (JSON.stringify(canonical(snapshot.observedPhysical)) !== JSON.stringify(canonical(observedPhysical))) fail("Captured physical metadata/content digest was altered") + if (JSON.stringify(canonical(snapshot.observedRaw)) !== JSON.stringify(canonical(observedRaw))) fail("Captured raw catalog/content digest was altered") + const text = JSON.stringify(canonical({ format: snapshot.format, scope, serverVersion: snapshot.serverVersion, sections: logical, ...(eventTriggers ? { eventTriggers } : {}) })) + if (text !== snapshot.canonical || digest(text) !== snapshot.sha256) fail("Captured schema content/digest was altered") + return text + } + const reference = checked(expected), actual = checked(observed) + if (reference !== actual) fail("Observed schema differs from the exact captured reference") + return Object.freeze({ equal: true as const, evidenceOnly: true as const, enrollmentAuthorized: false as const, sha256: digest(actual) }) +} diff --git a/backend/src/db/migration-schema-restore-layout.sql-helper.ts b/backend/src/db/migration-schema-restore-layout.sql-helper.ts new file mode 100644 index 00000000..f56eeba4 --- /dev/null +++ b/backend/src/db/migration-schema-restore-layout.sql-helper.ts @@ -0,0 +1,114 @@ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { join, basename, dirname, resolve } from "node:path" +import { writeFile, unlink } from "node:fs/promises" +import { Pool } from "pg" +import { captureMigrationData, compareMigrationData } from "./migration-data-inventory.js" +import { compareMigrationSchemas } from "./migration-schema-inventory.js" + +/** Test-only owned-cluster regression. Never called by an application entrypoint. */ +export async function acceptMigrationSchemaRestoreLayoutSql() { +assert.equal(process.env.MMAPP_OWN_MIGRATION_DATA_INVENTORY, "true") +const url = new URL(process.env.DATABASE_URL ?? "") +assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") +assert.equal(url.username,"mcp_test"); assert(url.port&&url.port!=="5432"); assert.equal(process.env.NODE_ENV,"test") +assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const source = new Pool({ connectionString: url.toString(), max: 3 }), names: string[] = [], cases: string[] = [] +const pools: Pool[] = [] +const archive = join(process.env.MCP_STATE_DIR!, "owned-v3-tombstone.dump") +async function database(label: string, template = "template0") { + const name = "mm_v3_" + label + "_" + randomUUID().replaceAll("-", "") + assert(/^mm_v3_[a-z_]+_[a-f0-9]{32}$/.test(name)); assert(template === "template0" || names.includes(template)) + await source.query('CREATE DATABASE "' + name + '" TEMPLATE "' + template + '"'); names.push(name) + const target = new URL(url); target.pathname = "/" + name + const pool = new Pool({ connectionString: target.toString(), max: 2 }); pools.push(pool) + return { name, pool } +} +async function snapshot(pool: Pool) { + const client = await pool.connect() + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + for (const query of ["SET LOCAL search_path=pg_catalog", "SET LOCAL TimeZone='UTC'", "SET LOCAL DateStyle='ISO, YMD'", "SET LOCAL standard_conforming_strings=on", "SET LOCAL statement_timeout='5s'", "SET LOCAL row_security=off", "SET LOCAL extra_float_digits=3", "SET LOCAL bytea_output=hex", "SET LOCAL IntervalStyle=postgres", "SET LOCAL lc_monetary='C'", "SET LOCAL lc_numeric='C'"]) await client.query(query) + return await captureMigrationData(client) + } finally { try { await client.query("ROLLBACK") } finally { client.release(true) } } +} +const execute = promisify(execFile) +try { + await source.query("COMMENT ON DATABASE postgres IS NULL") + await source.query(`CREATE TABLE public.layout_parent(id integer PRIMARY KEY,other integer UNIQUE); INSERT INTO public.layout_parent VALUES(1,1); + CREATE TABLE public.layout_probe( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,doomed text, + keep text COLLATE "C" NOT NULL DEFAULT 'owned',optional integer, + calc integer GENERATED ALWAYS AS(optional+1) STORED,spare text,tail text, + CONSTRAINT layout_fk FOREIGN KEY(optional) REFERENCES public.layout_parent(id), + CONSTRAINT layout_nullable CHECK((optional IS NULL OR optional>=0) AND keep<>'blocked')); + INSERT INTO public.layout_probe(doomed,optional,spare,tail) VALUES('discarded',1,'spare-preserved','tail-preserved'); + ALTER TABLE public.layout_probe DROP COLUMN doomed; + CREATE INDEX layout_rich_idx ON public.layout_probe(keep COLLATE "C",(optional+2)) INCLUDE(calc) WHERE optional IS NOT NULL; + CREATE FUNCTION public.layout_guard_fn() RETURNS trigger LANGUAGE plpgsql AS 'BEGIN RETURN NEW; END'; + CREATE TRIGGER layout_guard BEFORE UPDATE OF keep ON public.layout_probe FOR EACH ROW WHEN(NEW.keep IS DISTINCT FROM OLD.keep) EXECUTE FUNCTION public.layout_guard_fn()`) + const before = await snapshot(source) + const args = ["--host", url.hostname, "--port", url.port, "--username", decodeURIComponent(url.username), "--no-password"] + const env = { ...process.env, PGPASSWORD: decodeURIComponent(url.password) } + const dump = await execute("pg_dump", [...args, "--format=custom", "--dbname=postgres"], { env, encoding: "buffer", timeout: 30000, maxBuffer: 16777216 }) + await writeFile(archive, dump.stdout) + const template = await database("restored") + await execute("pg_restore", [...args, "--single-transaction", "--exit-on-error", "--dbname=" + template.name, archive], { env, timeout: 30000, maxBuffer: 1048576 }) + const restored = await snapshot(template.pool) + assert.equal(compareMigrationSchemas(before.schema, restored.schema).equal, true); assert.equal(compareMigrationData(before, restored).equal, true) + assert.notEqual(before.schema.observedPhysical.sha256, restored.schema.observedPhysical.sha256) + assert.notEqual(before.schema.observedRaw.sha256, restored.schema.observedRaw.sha256) + assert.notEqual(before.schema.sections.indexes.find(r => r.name === "layout_rich_idx")!.keys, restored.schema.sections.indexes.find(r => r.name === "layout_rich_idx")!.keys) + assert(before.schema.sections.columns.some(r => r.dropped)); assert(!restored.schema.sections.columns.some(r => r.dropped)) + await template.pool.end() + cases.push("real pg_dump/restore compacts tombstone and indkey/INCLUDE attnums, keeps logical schema and actual rows/sequences equal, preserves distinct raw/physical fingerprints") + const mismatches: [string, string][] = [ + ["same_middle_name_readded", "ALTER TABLE public.layout_probe DROP COLUMN spare; ALTER TABLE public.layout_probe ADD COLUMN spare text"], + ["type", "ALTER TABLE public.layout_probe ALTER COLUMN spare TYPE varchar(50)"], + ["collation", 'ALTER TABLE public.layout_probe ALTER COLUMN spare TYPE text COLLATE "C"'], + ["default", "ALTER TABLE public.layout_probe ALTER COLUMN keep SET DEFAULT 'changed'"], + ["generated", "ALTER TABLE public.layout_probe ALTER COLUMN calc SET EXPRESSION AS(optional+9)"], + ["identity", "ALTER TABLE public.layout_probe ALTER COLUMN id SET GENERATED BY DEFAULT"], + ["include_reference", 'DROP INDEX public.layout_rich_idx; CREATE INDEX layout_rich_idx ON public.layout_probe(keep COLLATE "C",(optional+2)) INCLUDE(tail) WHERE optional IS NOT NULL'], + ["index_expression", 'DROP INDEX public.layout_rich_idx; CREATE INDEX layout_rich_idx ON public.layout_probe(keep COLLATE "C",(optional+3)) INCLUDE(calc) WHERE optional IS NOT NULL'], + ["and_or_grouping", "ALTER TABLE public.layout_probe DROP CONSTRAINT layout_nullable; ALTER TABLE public.layout_probe ADD CONSTRAINT layout_nullable CHECK(optional IS NULL OR(optional>=0 AND keep<>'blocked'))"], + ["comparison_operator", "ALTER TABLE public.layout_probe DROP CONSTRAINT layout_nullable; ALTER TABLE public.layout_probe ADD CONSTRAINT layout_nullable CHECK((optional IS NULL OR optional>0) AND keep<>'blocked')"], + ["null_semantics", "ALTER TABLE public.layout_probe DROP CONSTRAINT layout_nullable; ALTER TABLE public.layout_probe ADD CONSTRAINT layout_nullable CHECK((optional IS NOT NULL OR optional>=0) AND keep<>'blocked')"], + ["not_valid", "ALTER TABLE public.layout_probe DROP CONSTRAINT layout_nullable; ALTER TABLE public.layout_probe ADD CONSTRAINT layout_nullable CHECK((optional IS NULL OR optional>=0) AND keep<>'blocked') NOT VALID"], + ["fk_reference", "ALTER TABLE public.layout_probe DROP CONSTRAINT layout_fk; ALTER TABLE public.layout_probe ADD CONSTRAINT layout_fk FOREIGN KEY(optional) REFERENCES public.layout_parent(other)"], + ["trigger_state", "ALTER TABLE public.layout_probe DISABLE TRIGGER layout_guard"], + ["function_security", "ALTER FUNCTION public.layout_guard_fn() SECURITY DEFINER"], + ] + for (const [label, sql] of mismatches) { + const target = await database(label, template.name) + try { + await target.pool.query(sql) + const changed = await snapshot(target.pool) + assert.throws(() => compareMigrationSchemas(restored.schema, changed.schema), label) + cases.push("actual " + label + " drift remains refused by logical v3") + } finally { await target.pool.end() } + } + const last = await database("same_last_name_readded", template.name) + await last.pool.query("ALTER TABLE public.layout_probe DROP COLUMN tail; ALTER TABLE public.layout_probe ADD COLUMN tail text") + const lost = await snapshot(last.pool) + // Identical final live DDL cannot prove historical data preservation. The + // raw fingerprint changes and actual data comparison must detect row loss. + assert.equal(compareMigrationSchemas(restored.schema, lost.schema).equal, true) + assert.notEqual(restored.schema.observedPhysical.sha256, lost.schema.observedPhysical.sha256) + assert.throws(() => compareMigrationData(restored, lost)) + await last.pool.end(); cases.push("same last-column name/type/order readd is not mistaken for data preservation: actual lost values refuse data equality despite equal live DDL") + const unsafe = await database("unsupported_policy", template.name) + await unsafe.pool.query("ALTER TABLE public.layout_probe ENABLE ROW LEVEL SECURITY; CREATE POLICY layout_policy ON public.layout_probe USING(true)") + await assert.rejects(snapshot(unsafe.pool)); await unsafe.pool.end(); cases.push("unsupported actual RLS policy still refuses capture") + return { cases: cases.length, names: cases, actualSql: true, ownedDatabases: names.length, actualDumpRestore: true, + logicalSchemaFormat: "pg18-application-schema-v3", rawAndPhysicalDifferencesRetained: true, dataPreservationNotInferredFromDdl: true, + allWriterDrainAuthenticated: false, sourceGenerationAuthenticated: false, readyAuthorized: false, productionDatabaseTouched: false } +} finally { + await Promise.all(pools.map(p => p.end().catch(() => {}))) + for (const name of [...names].reverse()) await source.query('DROP DATABASE "' + name + '" WITH(FORCE)') + await source.end(); await unlink(archive).catch(e => { if (e.code !== "ENOENT") throw e }) +} + +} diff --git a/backend/src/db/operation-lock.ts b/backend/src/db/operation-lock.ts index e750e507..72781f69 100644 --- a/backend/src/db/operation-lock.ts +++ b/backend/src/db/operation-lock.ts @@ -1,18 +1,41 @@ -type Client = { - query: (sql: string, params: number[]) => Promise<{rows: Record[]}> +export type DatabaseOperationLockClient = { + query: (sql: string, params: unknown[]) => Promise<{rows: Record[]; command?: string}> release: (destroy?: boolean) => void } +export const SYSTEM_DATABASE_OPERATION_LOCK = 8_723_102 +export const SYSTEM_DATABASE_REQUEST_LOCK = 8_723_130 + +export async function acquireDatabaseRequestDrain(client: Pick): Promise { + await client.query("SELECT pg_catalog.pg_advisory_lock($1::bigint)", [SYSTEM_DATABASE_REQUEST_LOCK]) +} + +export async function releaseDatabaseRequestDrain(client: Pick): Promise { + const result = await client.query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS released", [SYSTEM_DATABASE_REQUEST_LOCK]) + if (result.rows[0]?.released !== true) throw new Error("Блокировка запросов к базе не освобождена") +} + +export async function withSystemDatabaseOperationLock(connect: () => Promise, run: (client: DatabaseOperationLockClient, discard: () => void) => Promise): Promise { + return withOperationLock(connect, SYSTEM_DATABASE_OPERATION_LOCK, run, "Операция с базой приложения уже выполняется", true) +} /** Nonblocking session lock: a second destructive operation is rejected rather than queued. */ -export async function withOperationLock(connect: () => Promise, key: number, run: () => Promise, busyMessage = "Сброс NetFlow уже выполняется"): Promise { +export async function withOperationLock(connect: () => Promise, key: number, run: (client: DatabaseOperationLockClient, discard: () => void) => Promise, busyMessage = "Сброс NetFlow уже выполняется", drainRequests = false): Promise { const client = await connect() let owned = false, destroy = false + let drained = false + let discard = false try { destroy = true + if (drainRequests) { + await client.query("SET lock_timeout = '10s'", []) + await acquireDatabaseRequestDrain(client) + drained = true + await client.query("RESET lock_timeout", []) + } const result = await client.query("SELECT pg_try_advisory_lock($1) AS acquired", [key]) if (result.rows[0]?.acquired === false) { destroy = false; throw new Error(busyMessage) } if (result.rows[0]?.acquired !== true) throw new Error("Блокировка сброса NetFlow не подтверждена") owned = true; destroy = false - return await run() + return await run(client, () => { discard = true }) } finally { if (owned) { try { @@ -20,8 +43,11 @@ export async function withOperationLock(connect: () => Promise, key: destroy = result.rows[0]?.released !== true } catch { destroy = true } } + if (drained && !destroy) { + try { await releaseDatabaseRequestDrain(client) } catch { destroy = true } + } // A connection whose unlock failed must not return a held session lock to the pool. - client.release(destroy) + client.release(destroy || discard) } } diff --git a/backend/src/db/partition-guard-inheritance.ts b/backend/src/db/partition-guard-inheritance.ts new file mode 100644 index 00000000..ba99989e --- /dev/null +++ b/backend/src/db/partition-guard-inheritance.ts @@ -0,0 +1,186 @@ +import {createHash} from 'node:crypto' +import {assertRestoreProfileQuarantine} from './database-restore-quarantine-profile.js' + +export interface PartitionGuardSql {query(sql:string,values?:unknown[]):Promise<{rows:Record[];command?:string}>} +const KEY=8723110 +const MANAGED=new Set(['flow_buckets','flow_minute_stats','flow_minute_dims','flow_daily_dims','flow_hour_facts','flow_daily_facts','traffic_samples','servers_rest_ping_samples','uptime_probe_samples','uptime_resource_samples','server_snapshots','internet_path_snapshots']) +const ROW='server_partition_writer_guard',STATEMENT='candidate_application_statement_guard',TRUNCATE='candidate_truncate_preadmitted' +// Fixed reviewed SQL bodies. Operator provisioning must reuse these exact +// definitions; this component cannot issue/enroll any trusted profile/state. +const BODIES:Readonly>={ + 'public.server_partition_writer_guard':" BEGIN PERFORM pg_advisory_xact_lock_shared(8723110::bigint); IF TG_OP='DELETE' THEN RETURN OLD; END IF; RETURN NEW; END; ", + 'mmapp_control.candidate_truncate_preadmitted':'\nBEGIN PERFORM mmapp_control.candidate_require_preadmission(); RETURN NULL; END ', + 'mmapp_control.candidate_require_preadmission':"\nBEGIN\n IF NOT EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE pid=pg_catalog.pg_backend_pid()\n AND locktype='advisory' AND classid=0 AND objid=8723110 AND objsubid=1\n AND granted AND mode IN ('ShareLock','ExclusiveLock')) THEN\n RAISE EXCEPTION 'operator SQL requires preheld writer admission' USING ERRCODE='55000';\n END IF;\n PERFORM pg_catalog.pg_advisory_xact_lock_shared(8723110::bigint);\nEND ", +} +type Acl={grantee:string;grantor:string;privilege:string;grantable:boolean} +type Relation={oid:string;name:string;kind:string;partition:boolean;persistence:string;rls:boolean;force_rls:boolean;owner:string;schema_owner:string;acl:Acl[];parents:string[]} +type Trigger={oid:string;name:string;type:number;enabled:string;function_oid:string;function_namespace:string;function_name:string;parent_trigger:string;args:number;arg_bytes:number;unconditional:boolean;constraint_oid:string;deferrable:boolean;deferred:boolean;old_table:string|null;new_table:string|null} +type Context={pid:number;xid:string|null;actor:string;login:string;role_oid:string;database_oid:string;read_only:string;isolation:string;version:number;admitted:boolean} +type FunctionRow={oid:string;namespace:string;name:string;owner:string;schema_owner:string;language:string;return_type:string;return_namespace:string;kind:string;source:string;security_definer:boolean;configuration:string[]|null;strict:boolean;leakproof:boolean;volatility:string;parallel:string;arguments:number;returns_set:boolean;library:string|null;acl:Acl[]} +type Policy={parent:Relation;triggers:Trigger[];functions:FunctionRow[];runtime:{oid:string;name:string};capture:{oid:string;name:string};controlOwner:string} +function refuse(message:string):never{throw Error('Reviewed partition guard inheritance refused: '+message)} +const identifier=(value:string)=>{if(!/^[a-z_][a-z0-9_]{0,62}$/.test(value))refuse('identifier');return '"'+value+'"'} +const canonical=(value:unknown):string=>JSON.stringify(value,(_key,item:unknown)=>item&&typeof item==='object'&&!Array.isArray(item)?Object.fromEntries(Object.entries(item).sort(([a],[b])=>a.localeCompare(b))):item) +const digest=(value:unknown)=>createHash('sha256').update(canonical(value)).digest('hex') +async function rows(client:PartitionGuardSql,sql:string,values:unknown[]=[],bound=32):Promise{const result=await client.query(sql,values);if(!Array.isArray(result.rows)||result.rows.length>bound)refuse('catalog bound');return result.rows as unknown as T[]} +async function context(client:PartitionGuardSql):Promise{ + const result=await rows(client,`SELECT pg_catalog.pg_backend_pid() AS pid,pg_catalog.pg_current_xact_id_if_assigned()::pg_catalog.text AS xid, + CURRENT_USER::pg_catalog.text AS actor,SESSION_USER::pg_catalog.text AS login,r.oid::pg_catalog.text AS role_oid,d.oid::pg_catalog.text AS database_oid, + pg_catalog.current_setting('transaction_read_only') AS read_only,pg_catalog.current_setting('transaction_isolation') AS isolation,pg_catalog.current_setting('server_version_num')::pg_catalog.int4 AS version, + EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() AND classid=0 AND objid=$1::pg_catalog.oid AND objsubid=1 AND granted AND mode IN('ShareLock','ExclusiveLock')) AS admitted + FROM pg_catalog.pg_roles r CROSS JOIN pg_catalog.pg_database d WHERE r.rolname=CURRENT_USER AND d.datname=pg_catalog.current_database()`,[KEY],1) + const value=result[0] + if(!value||!Number.isSafeInteger(value.pid)||value.pid<=0||value.actor!==value.login||value.read_only!=='off'||!value.admitted)refuse('owned writable preheld admission') + return value +} +const ACL_SQL=`SELECT a.grantee::pg_catalog.text AS grantee,a.grantor::pg_catalog.text AS grantor,a.privilege_type AS privilege,a.is_grantable AS grantable FROM pg_catalog.aclexplode(COALESCE(c.relacl,pg_catalog.acldefault('r',c.relowner))) a ORDER BY a.grantee,a.privilege_type LIMIT 33` +async function relation(client:PartitionGuardSql,name:string):Promise{ + const result=await rows(client,`SELECT c.oid::pg_catalog.text AS oid,c.relname AS name,c.relkind::pg_catalog.text AS kind,c.relispartition AS partition,c.relpersistence::pg_catalog.text AS persistence,c.relrowsecurity AS rls,c.relforcerowsecurity AS force_rls, + c.relowner::pg_catalog.text AS owner,n.nspowner::pg_catalog.text AS schema_owner, + COALESCE((SELECT pg_catalog.jsonb_agg(x) FROM (${ACL_SQL}) x),'[]'::pg_catalog.jsonb) AS acl, + ARRAY(SELECT i.inhparent::pg_catalog.text FROM pg_catalog.pg_inherits i WHERE i.inhrelid=c.oid ORDER BY i.inhseqno) AS parents + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='public' AND c.relname=$1`,[name],1) + return result[0] +} +async function triggers(client:PartitionGuardSql,oid:string):Promise{return rows(client,`SELECT t.oid::pg_catalog.text AS oid,t.tgname AS name,t.tgtype::pg_catalog.int4 AS type,t.tgenabled::pg_catalog.text AS enabled, + t.tgfoid::pg_catalog.text AS function_oid,fn.nspname AS function_namespace,p.proname AS function_name,t.tgparentid::pg_catalog.text AS parent_trigger,t.tgnargs::pg_catalog.int4 AS args,pg_catalog.octet_length(t.tgargs)::pg_catalog.int4 AS arg_bytes, + t.tgqual IS NULL AS unconditional,t.tgconstraint::pg_catalog.text AS constraint_oid,t.tgdeferrable AS deferrable,t.tginitdeferred AS deferred,t.tgoldtable AS old_table,t.tgnewtable AS new_table + FROM pg_catalog.pg_trigger t JOIN pg_catalog.pg_proc p ON p.oid=t.tgfoid JOIN pg_catalog.pg_namespace fn ON fn.oid=p.pronamespace WHERE t.tgrelid=$1::pg_catalog.oid AND NOT t.tgisinternal ORDER BY t.tgname LIMIT 9`,[oid],8)} +function aclShape(acl:Acl[],owner:string,kind:'table'|'function'='table'):Acl[]{ + if(!Array.isArray(acl)||acl.length>32||acl.some(item=>!item||item.grantor!==owner||item.grantable!==false||typeof item.grantee!=='string'||typeof item.privilege!=='string'))refuse('ACL/grantor/grant option') + if(canonical(acl.filter(item=>item.grantee===owner).map(item=>item.privilege).sort())!==canonical(kind==='function'?['EXECUTE']:['DELETE','INSERT','MAINTAIN','REFERENCES','SELECT','TRIGGER','TRUNCATE','UPDATE']))refuse('owner ACL structure') + return acl.filter(item=>item.grantee!==owner).sort((a,b)=>(a.grantee+':'+a.privilege).localeCompare(b.grantee+':'+b.privilege)) +} +function guard(trigger:Trigger|undefined,name:string,type:number,fn:string,parentTrigger='0'){ + if(!trigger||trigger.name!==name||trigger.type!==type||trigger.function_oid!==fn||trigger.enabled!=='A'||trigger.parent_trigger!==parentTrigger||trigger.args!==0||trigger.arg_bytes!==0||!trigger.unconditional||trigger.constraint_oid!=='0'||trigger.deferrable||trigger.deferred||trigger.old_table!==null||trigger.new_table!==null)refuse('trigger identity/shape') +} +async function functions(client:PartitionGuardSql):Promise{return rows(client,`SELECT p.oid::pg_catalog.text AS oid,n.nspname AS namespace,p.proname AS name,p.proowner::pg_catalog.text AS owner,n.nspowner::pg_catalog.text AS schema_owner, + l.lanname AS language,t.typname AS return_type,tn.nspname AS return_namespace,p.prokind::pg_catalog.text AS kind,p.prosrc AS source,p.prosecdef AS security_definer,p.proconfig AS configuration,p.proisstrict AS strict,p.proleakproof AS leakproof, + p.provolatile::pg_catalog.text AS volatility,p.proparallel::pg_catalog.text AS parallel,p.pronargs::pg_catalog.int4 AS arguments,p.proretset AS returns_set,p.probin AS library, + COALESCE((SELECT pg_catalog.jsonb_agg(x) FROM (SELECT a.grantee::pg_catalog.text AS grantee,a.grantor::pg_catalog.text AS grantor,a.privilege_type AS privilege,a.is_grantable AS grantable FROM pg_catalog.aclexplode(COALESCE(p.proacl,pg_catalog.acldefault('f',p.proowner))) a ORDER BY a.grantee,a.privilege_type LIMIT 33) x),'[]'::pg_catalog.jsonb) AS acl + FROM pg_catalog.pg_proc p JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace JOIN pg_catalog.pg_language l ON l.oid=p.prolang JOIN pg_catalog.pg_type t ON t.oid=p.prorettype JOIN pg_catalog.pg_namespace tn ON tn.oid=t.typnamespace + WHERE(n.nspname='public' AND p.proname='server_partition_writer_guard')OR(n.nspname='mmapp_control' AND p.proname IN('candidate_require_preadmission','candidate_truncate_preadmitted')) ORDER BY n.nspname,p.proname LIMIT 5`,[],4)} +async function reviewedFunctions(client:PartitionGuardSql,owner:string):Promise { + const fns=await functions(client);if(fns.length!==3)refuse('fixed guard function set') + for(const fn of fns){ + const identity=fn.namespace+'.'+fn.name,body=BODIES[identity] + if(!body||fn.source!==body||fn.arguments!==0||fn.kind!=='f'||fn.language!=='plpgsql'||fn.strict||fn.leakproof||fn.volatility!=='v'||fn.parallel!=='u'||fn.returns_set||fn.library!==null||canonical(fn.configuration)!==canonical(['search_path=pg_catalog'])||fn.return_namespace!=='pg_catalog'||fn.return_type!==(fn.name==='candidate_require_preadmission'?'void':'trigger')||fn.security_definer!==(fn.namespace==='mmapp_control')||fn.owner!==fn.schema_owner||fn.namespace==='public'&&fn.owner!==owner||fn.namespace==='mmapp_control'&&fn.owner===owner)refuse('fixed guard function body/security/owner') + const other=aclShape(fn.acl,fn.owner,'function'),expected=fn.namespace==='public'?[{grantee:'0',grantor:fn.owner,privilege:'EXECUTE',grantable:false}]:fn.name===TRUNCATE?[{grantee:owner,grantor:fn.owner,privilege:'EXECUTE',grantable:false}]:[] + if(canonical(other)!==canonical(expected))refuse('guard function ACL') + } + return fns +} +async function policy(client:PartitionGuardSql,parent:Relation,all:Trigger[],ctx:Context):Promise{ + if(!MANAGED.has(parent.name)||ctx.version<180000||ctx.version>=190000||parent.kind!=='p'||parent.partition||parent.persistence!=='p'||parent.rls||parent.force_rls||parent.owner!==ctx.role_oid||parent.schema_owner!==ctx.role_oid||all.length!==3)refuse('managed parent/owner/scope') + const fns=await reviewedFunctions(client,parent.owner) + const rowFn=fns.find(fn=>fn.namespace==='public')!,truncateFn=fns.find(fn=>fn.name===TRUNCATE)!,controlOwner=truncateFn.owner + guard(all.find(t=>t.name===ROW),ROW,31,rowFn.oid);guard(all.find(t=>t.name===STATEMENT),STATEMENT,30,rowFn.oid);guard(all.find(t=>t.name===TRUNCATE),TRUNCATE,34,truncateFn.oid) + const acl=aclShape(parent.acl,parent.owner),grantees=[...new Set(acl.map(item=>item.grantee))] + if(grantees.length!==2||grantees.includes('0'))refuse('exact two parent grantees') + const runtimeId=grantees.find(oid=>canonical(acl.filter(item=>item.grantee===oid).map(item=>item.privilege).sort())===canonical(['DELETE','INSERT','SELECT','UPDATE'])) + const captureId=grantees.find(oid=>canonical(acl.filter(item=>item.grantee===oid).map(item=>item.privilege))===canonical(['SELECT'])) + if(!runtimeId||!captureId||runtimeId===captureId)refuse('exact runtime/capture ACL') + const roles=await rows<{oid:string;name:string;control_name:string;unsafe:boolean;owner_member:boolean;control_member:boolean;has_membership:boolean;public_usage:boolean;public_create:boolean;control_usage:boolean;control_create:boolean;db_create:boolean;db_temp:boolean;set_replication:boolean;set_event:boolean}>(client,`SELECT r.oid::pg_catalog.text AS oid,r.rolname AS name,pg_catalog.pg_get_userbyid($2::pg_catalog.oid) AS control_name, + (r.rolsuper OR r.rolcreatedb OR r.rolcreaterole OR r.rolreplication OR r.rolbypassrls) AS unsafe, + pg_catalog.pg_has_role(r.oid,$1::pg_catalog.oid,'MEMBER') AS owner_member,pg_catalog.pg_has_role(r.oid,$2::pg_catalog.oid,'MEMBER') AS control_member, + EXISTS(SELECT 1 FROM pg_catalog.pg_auth_members m WHERE m.member=r.oid OR m.roleid=r.oid) AS has_membership, + pg_catalog.has_schema_privilege(r.oid,('public'::pg_catalog.regnamespace)::pg_catalog.oid,'USAGE') AS public_usage,pg_catalog.has_schema_privilege(r.oid,('public'::pg_catalog.regnamespace)::pg_catalog.oid,'CREATE') AS public_create, + pg_catalog.has_schema_privilege(r.oid,('mmapp_control'::pg_catalog.regnamespace)::pg_catalog.oid,'USAGE') AS control_usage,pg_catalog.has_schema_privilege(r.oid,('mmapp_control'::pg_catalog.regnamespace)::pg_catalog.oid,'CREATE') AS control_create, + pg_catalog.has_database_privilege(r.oid,$4::pg_catalog.oid,'CREATE') AS db_create,pg_catalog.has_database_privilege(r.oid,$4::pg_catalog.oid,'TEMP') AS db_temp, + pg_catalog.has_parameter_privilege(r.oid,'session_replication_role','SET') AS set_replication,pg_catalog.has_parameter_privilege(r.oid,'event_triggers','SET') AS set_event + FROM pg_catalog.pg_roles r WHERE r.oid=ANY($3::pg_catalog.oid[]) ORDER BY r.oid`,[parent.owner,controlOwner,[parent.owner,runtimeId,captureId],ctx.database_oid],3) + if(roles.length!==3)refuse('actor roles') + for(const role of roles){identifier(role.name);if(role.unsafe||role.control_member||role.has_membership||role.db_create||role.db_temp||role.set_replication||role.set_event||role.control_create)refuse('role/membership/DDL parameter escape');if(role.oid!==parent.owner&&(role.owner_member||!role.public_usage||role.public_create||role.oid!==runtimeId&&role.control_usage))refuse('limited actor schema/ownership')} + const runtime=roles.find(role=>role.oid===runtimeId)! + if(runtime.control_usage)await assertRestoreProfileQuarantine(client,runtime.name,ctx.actor,runtime.control_name) + return {parent,triggers:all,functions:fns,runtime:roles.find(role=>role.oid===runtimeId)!,capture:roles.find(role=>role.oid===captureId)!,controlOwner} +} + +/** Fixed marker-table guard validation under the migration owner's held admission. */ +export async function assertReviewedMigrationMarkerGuard(client:PartitionGuardSql,state:'installed'|'unenrolled'='installed'):Promise{ + const ctx=await context(client), marker=await relation(client,'schema_migrations') + if(!marker||ctx.version<180000||ctx.version>=190000||marker.kind!=='r'||marker.partition||marker.persistence!=='p'|| + marker.rls||marker.force_rls||marker.parents.length||marker.owner!==ctx.role_oid||marker.schema_owner!==ctx.role_oid)refuse('marker table/owner/scope') + const acl=aclShape(marker.acl,marker.owner) + if(acl.length!==3||new Set(acl.map(entry=>entry.grantee)).size!==3|| + acl.some(entry=>entry.grantee==='0'||entry.privilege!=='SELECT'))refuse('marker table ACL') + const extra=await rows<{unsafe:boolean}>(client,`SELECT ( + c.relhasrules OR EXISTS (SELECT 1 FROM pg_catalog.pg_inherits i WHERE i.inhparent=c.oid) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_attribute a WHERE a.attrelid=c.oid AND a.attnum>0 AND a.attacl IS NOT NULL) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_trigger t WHERE t.tgrelid=c.oid + AND (t.tgisinternal OR t.tgattr::text<>'' OR t.tgconstrrelid<>0 OR t.tgconstrindid<>0)) + ) AS unsafe FROM pg_catalog.pg_class c WHERE c.oid=$1::pg_catalog.oid`,[marker.oid],1) + if(extra.length!==1||extra[0].unsafe!==false)refuse('marker table rules, column ACL or trigger scope') + const all=await triggers(client,marker.oid) + if(all.length!==(state==='unenrolled'?0:3))refuse('exact marker trigger set') + const fns=await reviewedFunctions(client,marker.owner) + if(state==='unenrolled')return + const rowFn=fns.find(fn=>fn.namespace==='public')!,truncateFn=fns.find(fn=>fn.name===TRUNCATE)! + guard(all.find(t=>t.name===ROW),ROW,31,rowFn.oid) + guard(all.find(t=>t.name===STATEMENT),STATEMENT,30,rowFn.oid) + guard(all.find(t=>t.name===TRUNCATE),TRUNCATE,34,truncateFn.oid) +} +async function validateLeaf(client:PartitionGuardSql,leaf:Relation,parent:Policy,complete:boolean){ + if(leaf.kind!=='r'||!leaf.partition||leaf.persistence!=='p'||leaf.rls||leaf.force_rls||leaf.owner!==parent.parent.owner||canonical(leaf.parents)!==canonical([parent.parent.oid]))refuse('direct owned ordinary leaf') + const all=await triggers(client,leaf.oid),row=parent.triggers.find(t=>t.name===ROW)!,statement=parent.triggers.find(t=>t.name===STATEMENT)!,truncate=parent.triggers.find(t=>t.name===TRUNCATE)! + if(all.length!==(complete?3:1))refuse('leaf foreign/partial guards') + guard(all.find(t=>t.name===ROW),ROW,31,row.function_oid,row.oid) + if(complete){guard(all.find(t=>t.name===STATEMENT),STATEMENT,30,statement.function_oid);guard(all.find(t=>t.name===TRUNCATE),TRUNCATE,34,truncate.function_oid)} + const extra=aclShape(leaf.acl,leaf.owner),wanted=aclShape(parent.parent.acl,parent.parent.owner) + if(canonical(extra)!==canonical(wanted)&&!(complete===false&&extra.length===0))refuse('leaf foreign/partial ACL') +} +/** Mechanical inheritance only. Parent SQL is NOT trusted profile issuance. + * Protected enabled-host admission remains independently required. */ +export async function prepareReviewedPartitionGuardInheritance(client:PartitionGuardSql,parentName:string,leafName:string):Promise<()=>Promise>{ + identifier(parentName);identifier(leafName) + const first=await context(client) + // This is reentrant ONLY after confirming a preheld same-backend lease; never + // first wait for a global admission key while owning relation DDL locks. + await client.query('SELECT pg_catalog.pg_advisory_xact_lock_shared($1::pg_catalog.int8)',[KEY]) + const assigned=await rows<{xid:string}>(client,'SELECT pg_catalog.pg_current_xact_id()::pg_catalog.text AS xid',[],1) + const bound={...first,xid:assigned[0]?.xid};if(!bound.xid||!/^[1-9][0-9]*$/.test(bound.xid))refuse('owned full transaction identity') + const check=async()=>{if(canonical(await context(client))!==canonical(bound))refuse('backend/transaction/admission changed')} + await check() + const parent=await relation(client,parentName);if(!parent)refuse('missing parent') + const all=await triggers(client,parent.oid) + const guarded=all.some(t=>t.name===STATEMENT||t.name===TRUNCATE) + let used=false + if(!guarded){ + if(all.length>1||all.length===1&&(all[0].name!==ROW||all[0].type!==31||all[0].enabled!=='A'||all[0].function_namespace!=='public'||all[0].function_name!==ROW||all[0].args!==0||all[0].arg_bytes!==0||!all[0].unconditional))refuse('foreign/partial legacy parent') + return async()=>{if(used)refuse('finished continuation');used=true;await check()} + } + if(bound.isolation!=='read committed')refuse('guarded DDL requires fresh READ COMMITTED metadata') + let template=await policy(client,parent,all,bound),existing=await relation(client,leafName) + if(existing)await validateLeaf(client,existing,template,true) + else{ + // Only a potentially missing leaf needs CREATE's parent lock. Known full + // leaves retain a read-only verification fast path, without a new strong + // parent lock on every normal flow flush. Re-read after a racing creator. + await client.query('LOCK TABLE public.'+identifier(parentName)+' IN ACCESS EXCLUSIVE MODE') + await check() + const lockedParent=await relation(client,parentName);if(!lockedParent||lockedParent.oid!==parent.oid)refuse('locked parent changed') + template=await policy(client,lockedParent,await triggers(client,lockedParent.oid),bound) + existing=await relation(client,leafName) + if(existing)await validateLeaf(client,existing,template,true) + } + const fingerprint=digest(template) + return async()=>{ + if(used)refuse('finished continuation');used=true;await check() + const currentParent=await relation(client,parentName);if(!currentParent)refuse('parent disappeared') + if(digest(await policy(client,currentParent,await triggers(client,currentParent.oid),bound))!==fingerprint)refuse('parent policy changed') + const leaf=await relation(client,leafName);if(!leaf)refuse('missing created leaf') + await validateLeaf(client,leaf,template,Boolean(existing)) + if(!existing){ + const table='public.'+identifier(leafName) + await client.query(`CREATE TRIGGER ${STATEMENT} BEFORE INSERT OR UPDATE OR DELETE ON ${table} FOR EACH STATEMENT EXECUTE FUNCTION public.server_partition_writer_guard()`) + await client.query(`ALTER TABLE ${table} ENABLE ALWAYS TRIGGER ${STATEMENT}`) + await client.query(`CREATE TRIGGER ${TRUNCATE} BEFORE TRUNCATE ON ${table} FOR EACH STATEMENT EXECUTE FUNCTION mmapp_control.candidate_truncate_preadmitted()`) + await client.query(`ALTER TABLE ${table} ENABLE ALWAYS TRIGGER ${TRUNCATE}`) + await client.query(`GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE ${table} TO ${identifier(template.runtime.name)}`) + await client.query(`GRANT SELECT ON TABLE ${table} TO ${identifier(template.capture.name)}`) + } + await validateLeaf(client,(await relation(client,leafName))!,template,true);await check() + const finalParent=(await relation(client,parentName))! + if(digest(await policy(client,finalParent,await triggers(client,finalParent.oid),bound))!==fingerprint)refuse('final parent policy changed') + } +} diff --git a/backend/src/db/partitions-writer.test.ts b/backend/src/db/partitions-writer.test.ts new file mode 100644 index 00000000..a86f3b66 --- /dev/null +++ b/backend/src/db/partitions-writer.test.ts @@ -0,0 +1,80 @@ +import assert from "node:assert/strict" +import pg, { type PoolClient } from "pg" +import { ensurePartitionFor,ensurePartitionsAround,ensurePartitionsBetween,listExpiredPartitions,dropExpiredPartitions,PARTITION_SPECS } from "./partitions.js" +import { withPartitionWrite,PartitionWriteError } from "../services/partition-write-transaction.js" + +/** Explicit MODEL for the legacy ROW-only catalog protocol. Real42/114 owned + * acceptance supplies SQL truth; these rows confer no source/profile authority. + * The real inheritance helper executes unchanged against this bounded model. */ +function fixture(){ + const calls:string[]=[],release:boolean[]=[],direct:string[]=[] + let fail:string|undefined,retention=14,transaction=false,admitted=false,xid:string|null=null,nextXid=1000 + const parentOids=new Map() + const metadata=async(text:string,values?:unknown[])=>{ + if(text==="BEGIN"){transaction=true;admitted=false;xid=null} + if(text.includes("pg_advisory_xact_lock_shared")){assert(transaction,"MODEL admission requires caller BEGIN");assert.deepEqual(values,[8723110],"MODEL tracks the reviewed global admission key");admitted=true} + if(text.includes("pg_current_xact_id_if_assigned"))return {rows:[{pid:12345,xid,actor:"model_operator",login:"model_operator",role_oid:"16001",database_oid:"16390",read_only:"off",isolation:"read committed",version:180004,admitted:transaction&&admitted}]} + if(text.startsWith("SELECT pg_catalog.pg_current_xact_id()")){assert(transaction,"MODEL XID requires owned transaction");xid??=String(++nextXid);return {rows:[{xid}]}} + if(text.includes("c.relname AS name")&&text.includes("n.nspname='public' AND c.relname=$1")){ + const parent=String(values?.[0]);if(!parentOids.has(parent))parentOids.set(parent,String(20000+parentOids.size)) + const acl=["DELETE","INSERT","MAINTAIN","REFERENCES","SELECT","TRIGGER","TRUNCATE","UPDATE"].map(privilege=>({grantee:"16001",grantor:"16001",privilege,grantable:false})) + return {rows:[{oid:parentOids.get(parent),name:parent,kind:"p",partition:false,persistence:"p",rls:false,force_rls:false,owner:"16001",schema_owner:"16001",acl,parents:[]}]} + } + if(text.includes("FROM pg_catalog.pg_trigger t")&&text.includes("t.tgrelid=$1"))return {rows:[{oid:"17000",name:"server_partition_writer_guard",type:31,enabled:"A",function_oid:"17001",function_namespace:"public",function_name:"server_partition_writer_guard",parent_trigger:"0",args:0,arg_bytes:0,unconditional:true,constraint_oid:"0",deferrable:false,deferred:false,old_table:null,new_table:null}]} + if(text.includes("retention_days"))return {rows:[{retention_days:retention}]} + if(text.includes("FROM pg_inherits"))return {rows:values?.[0]==="flow_buckets"?[{relname:"flow_buckets_d_20000101"}]:[]} + if(text==="COMMIT"){transaction=false;admitted=false;xid=null} + return {rows:[],command:text==="COMMIT"?"COMMIT":"SELECT"} + } + const client={query:async(input:string|{text:string;values?:unknown[]},values?:unknown[])=>{const text=typeof input==="string"?input:input.text;calls.push(text);if(fail&&text.startsWith(fail)){fail=undefined;throw Error("Synthetic boundary failure")};return metadata(text,typeof input==="string"?values:input.values)},release:(destroy=false)=>{release.push(destroy)}}as unknown as PoolClient + const pool=new pg.Pool() + pool.connect=(async()=>client)as typeof pool.connect + pool.query=(async(text:string,values?:unknown[])=>{direct.push(text);assert(!/^(CREATE|ALTER|DROP)/.test(text),"Global pool issued partition DDL");return metadata(text,values)})as typeof pool.query + return {pool,client,calls,release,direct,fail:(value:string)=>{fail=value},retention:(days:number)=>{retention=days}} +} +let cases=0 +async function test(name:string,work:()=>Promise){await work();cases++;console.log("ok",name)} +const at=new Date("2099-04-08T12:00:00.000Z") +await test("standalone fact CREATE and ALTER share admitted backend and transaction",async()=>{const f=fixture();const name=await ensurePartitionFor(f.pool,"flow_daily_facts","month",at);assert.equal(name,"flow_daily_facts_m_209904");const shared=f.calls.findIndex(q=>q.includes("pg_advisory_xact_lock_shared")),create=f.calls.findIndex(q=>q.startsWith("CREATE")),alter=f.calls.findIndex(q=>q.startsWith("ALTER"));assert(shared>=0&&sharedq==="BEGIN").length,1);assert.equal(f.calls.at(-1),"COMMIT");assert.deepEqual(f.release,[false]);assert.equal(f.direct.length,0)}) +await test("common authenticated owner borrows existing transaction without nested boundaries",async()=>{const f=fixture();await withPartitionWrite(async owner=>{await ensurePartitionFor(owner.client,"flow_hour_facts","day",at)},{connect:async()=>f.client});assert.equal(f.calls.filter(q=>q==="BEGIN").length,1);assert.equal(f.calls.filter(q=>q==="COMMIT").length,1);const shared=f.calls.flatMap((q,index)=>q.includes("pg_advisory_xact_lock_shared")?[index]:[]);assert.equal(shared.length,2,"One owner admission plus one required reentrant XACT pin, no nested transaction");const checked=f.calls.findIndex(q=>q.includes("pg_current_xact_id_if_assigned")),create=f.calls.findIndex(q=>q.startsWith("CREATE"));assert(shared[0]{const f=fixture();await assert.rejects(ensurePartitionFor(f.client,"traffic_samples","week",at),/authenticated/);assert.equal(f.calls.length,0)}) +await test("closed authenticated handle cannot issue detached DDL",async()=>{const f=fixture();let closed:PoolClient|undefined;await withPartitionWrite(async owner=>{closed=owner.client},{connect:async()=>f.client});const count=f.calls.length;await assert.rejects(ensurePartitionFor(closed!,"traffic_samples","week",at),/authenticated/);assert.equal(f.calls.length,count)}) +await test("exact flow proxy uses actual live ALS client and keeps caller boundaries",async()=>{const f=fixture();const {flowWritePool,withFlowWriteClient}=await import("../services/traffic-flow-write-transaction.js");await f.client.query("BEGIN");await f.client.query("SELECT pg_advisory_xact_lock_shared($1::bigint)",[8723110]);await withFlowWriteClient(f.client,()=>ensurePartitionFor(flowWritePool,"flow_hour_facts","day",at));assert.equal(f.calls.filter(q=>q==="BEGIN").length,1);assert(!f.calls.includes("COMMIT"));assert.equal(f.calls.filter(q=>q.startsWith("CREATE")).length,1);assert.equal(f.calls.filter(q=>q.startsWith("ALTER")).length,1)}) +await test("flow authority lost before DDL sends no publication",async()=>{const f=fixture();const {flowWritePool,withFlowWriteClient}=await import("../services/traffic-flow-write-transaction.js");await assert.rejects(withFlowWriteClient(f.client,()=>ensurePartitionFor(flowWritePool,"flow_buckets","day",at),"shared",async()=>{throw Error("authority lost")}),/authority lost/);assert.equal(f.calls.length,0)}) +await test("SQL CREATE failure has no ALTER, COMMIT, inverse or retry",async()=>{const f=fixture();f.fail("CREATE");await assert.rejects(ensurePartitionFor(f.pool,"flow_hour_facts","day",at),PartitionWriteError);assert.equal(f.calls.filter(q=>q.startsWith("CREATE")).length,1);assert(!f.calls.some(q=>q.startsWith("ALTER")||q==="COMMIT"||q==="ROLLBACK"));assert.deepEqual(f.release,[true])}) +await test("unknown COMMIT destroys backend and does not replay DDL",async()=>{const f=fixture();f.fail("COMMIT");await assert.rejects(ensurePartitionFor(f.pool,"flow_buckets","day",at),(error:unknown)=>error instanceof PartitionWriteError&&error.outcomeUnknown);assert.equal(f.calls.filter(q=>q==="COMMIT").length,1);assert.equal(f.calls.filter(q=>q.startsWith("CREATE")).length,1);assert(!f.calls.includes("ROLLBACK"));assert.deepEqual(f.release,[true])}) +await test("identifier, mismatched known parent kind and invalid UTC instant reject before SQL",async()=>{for(const input of [["traffic_samples;DROP","week",at],["traffic_samples","day",at],["flow_buckets","day",new Date("invalid")]] as const){const f=fixture();await assert.rejects(ensurePartitionFor(f.pool,input[0],input[1],input[2]));assert.equal(f.calls.length,0);assert.equal(f.direct.length,0)}}) +await test("oversize range rejects entire plan before first DDL",async()=>{const f=fixture();await assert.rejects(ensurePartitionsBetween(f.pool,"flow_buckets","day",new Date("2000-01-01"),new Date("2010-01-01")),/budget/);assert.equal(f.calls.length,0)}) +await test("all12 parent footprints remain in bounded around plan",async()=>{const f=fixture();await ensurePartitionsAround(f.pool,at);const names=f.calls.filter(q=>q.startsWith("CREATE"));for(const spec of PARTITION_SPECS)assert(names.some(q=>q.includes(" PARTITION OF "+spec.parent+" ")),spec.parent);assert(names.length<2000);assert.equal(f.calls.filter(q=>q==="COMMIT").length,names.length)}) +await test("invalid configured retention cannot silently reduce maintenance footprint",async()=>{const f=fixture();f.retention(Infinity);await assert.rejects(ensurePartitionsAround(f.pool,at),/retention/);assert.equal(f.calls.length,0)}) +await test("read-only retention preview still accepts bounded query provider",async()=>{const f=fixture();assert.deepEqual(await listExpiredPartitions({query:f.pool.query.bind(f.pool)},at),[{parent:"flow_buckets",name:"flow_buckets_d_20000101"}]);assert.equal(f.calls.length,0)}) +await test("DROP callback executes after admission and immediately before owned DROP",async()=>{const f=fixture();await dropExpiredPartitions(f.pool,at,async partition=>{assert.equal(partition.name,"flow_buckets_d_20000101");assert(f.calls.some(q=>q.includes("pg_advisory_xact_lock_shared")));f.calls.push("CALLBACK")});const index=f.calls.indexOf("CALLBACK");assert(f.calls[index+1].startsWith("DROP TABLE"));assert.equal(f.calls.at(-1),"COMMIT")}) +await test("denied DROP callback sends no DROP and no retry",async()=>{const f=fixture();await assert.rejects(dropExpiredPartitions(f.pool,at,async()=>{throw Error("unapproved")}),PartitionWriteError);assert(!f.calls.some(q=>q.startsWith("DROP")||q==="COMMIT"));assert.deepEqual(f.release,[true])}) +await test("protected runtime verifies partition bounds and refuses pending retention DDL", async () => { + const saved = process.env.DATABASE_RUNTIME_REQUEST_FILE + process.env.DATABASE_RUNTIME_REQUEST_FILE = "approved-request.json" + try { + const f = fixture() + const query = f.pool.query.bind(f.pool) + let present = true + f.pool.query = (async (sql: string, values?: unknown[]) => { + if (!sql.includes("pg_catalog.pg_get_expr(leaf.relpartbound")) return query(sql, values) + assert.match(sql, /^SELECT /) + assert.match(sql, /NOT inheritance\.inhdetachpending/) + assert.match(sql, /\$3::pg_catalog\.date/) + assert.deepEqual(values, ["flow_daily_facts", "flow_daily_facts_m_209904", "2099-04-01", "2099-05-01"]) + return { rows: present ? [{ "?column?": 1 }] : [] } + }) as typeof f.pool.query + assert.equal(await ensurePartitionFor(f.pool, "flow_daily_facts", "month", at), "flow_daily_facts_m_209904") + present = false + await assert.rejects(ensurePartitionFor(f.pool, "flow_daily_facts", "month", at), /operator partition maintenance/) + await assert.rejects(dropExpiredPartitions(f.pool, at, async () => { + assert.fail("Protected runtime must not reach the DROP callback") + }), /operator maintenance/) + assert.equal(f.calls.length, 0) + } finally { + if (saved === undefined) delete process.env.DATABASE_RUNTIME_REQUEST_FILE + else process.env.DATABASE_RUNTIME_REQUEST_FILE = saved + } +}) +console.log(`Partition DDL ownership: ${cases} inert MODEL scenarios passed; no SQL/source authority claimed`) diff --git a/backend/src/db/partitions.ts b/backend/src/db/partitions.ts index d46d51b9..f24bde11 100644 --- a/backend/src/db/partitions.ts +++ b/backend/src/db/partitions.ts @@ -1,4 +1,37 @@ -import type { Pool } from "pg" +import {prepareReviewedPartitionGuardInheritance} from "./partition-guard-inheritance.js" +import pg, { type Pool } from "pg" +import { isPartitionWriteClient, withPartitionWrite } from "../services/partition-write-transaction.js" + +type PartitionQueries=Pick +const MAX_PARTITION_OPERATIONS=2000 +/** CREATE/ALTER/DROP share the exact backend admitted before relation access. + * A query-shaped client is never evidence of transaction ownership. */ +async function withPartitionDdl(target:PartitionQueries,work:(client:PartitionQueries)=>Promise):Promise{ + if(isPartitionWriteClient(target))return work(target) + if(!(target instanceof pg.Pool))throw Error("Partition DDL requires a Pool or a live authenticated partition writer client") + // Delay the flow runtime import until an actual pool is used. Importing the + // partition catalogue itself must not initialize the configured DB pool. + const {flowWritePool,flowWriteContext}=await import("../services/traffic-flow-write-transaction.js") + if(target===flowWritePool){ + const context=flowWriteContext() + if(context){ + const query=(async(...args:unknown[])=>{ + await context.authorize();if(!context.active||flowWriteContext()!==context)throw Error("Flow partition writer context is closed") + const result=await Reflect.apply(context.client.query,context.client,args) + await context.authorize();if(!context.active||flowWriteContext()!==context)throw Error("Flow partition writer context is closed") + return result + })as Pool["query"] + return work({query}) + } + } + return withPartitionWrite(owner=>work(owner.client),{connect:()=>target.connect()}) +} + +function validatePartitionInput(parent:string,kind:PartitionKind,ts:Date):void{ + if(!/^[a-z_][a-z\d_]{0,49}$/.test(parent)||!["day","week","month"].includes(kind)||!(ts instanceof Date)||!Number.isFinite(ts.getTime())||ts.getUTCFullYear()<100||ts.getUTCFullYear()>9998)throw Error("Invalid bounded partition identifier, kind or UTC instant") + const spec=PARTITION_SPECS.find(spec=>spec.parent===parent) + if(spec&&spec.kind!==kind)throw Error("Partition kind does not match the parent catalogue") +} export type PartitionKind = "day" | "week" | "month" @@ -31,32 +64,28 @@ export const PARTITION_SPECS: PartitionSpec[] = [ const WEEK_SLACK_DAYS = 7 -async function resolveKeepDays(pool: Pool): Promise> { +async function resolveKeepDays(pool: Pick): Promise> { const map = new Map(PARTITION_SPECS.map((s) => [s.parent, s.keepDays])) + const configured:{parents:string[];value:unknown}[]=[] try { const traffic = await pool.query<{ retention_days: number }>( `SELECT retention_days FROM traffic_settings WHERE id = 1`, ) - const td = Number(traffic.rows[0]?.retention_days) - if (Number.isFinite(td) && td > 0) map.set("traffic_samples", td + WEEK_SLACK_DAYS) + if(traffic.rows.length)configured.push({parents:["traffic_samples"],value:traffic.rows[0].retention_days}) const uptime = await pool.query<{ retention_days: number }>( `SELECT retention_days FROM uptime_settings WHERE id = 1`, ) - const ud = Number(uptime.rows[0]?.retention_days) - if (Number.isFinite(ud) && ud > 0) { - map.set("uptime_probe_samples", ud + WEEK_SLACK_DAYS) - map.set("uptime_resource_samples", ud + WEEK_SLACK_DAYS) - } + if(uptime.rows.length)configured.push({parents:["uptime_probe_samples","uptime_resource_samples"],value:uptime.rows[0].retention_days}) const path = await pool.query<{ retention_days: number }>( `SELECT retention_days FROM internet_path_settings WHERE id = 1`, ) - const pd = Number(path.rows[0]?.retention_days) - if (Number.isFinite(pd) && pd > 0) map.set("internet_path_snapshots", pd + WEEK_SLACK_DAYS) + if(path.rows.length)configured.push({parents:["internet_path_snapshots"],value:path.rows[0].retention_days}) } catch { /* settings may be absent mid-migration */ } + for(const setting of configured){const days=Number(setting.value);if(!Number.isSafeInteger(days)||days<1||days>3660)throw Error("Invalid bounded partition retention setting");for(const parent of setting.parents)map.set(parent,days+WEEK_SLACK_DAYS)} return map } @@ -88,7 +117,7 @@ function fmtDay(d: Date): string { function partName(parent: string, kind: PartitionKind, start: Date): string { if (kind === "day") return `${parent}_d_${fmtDay(start).replaceAll("-", "")}` if (kind === "week") return `${parent}_w_${fmtDay(start).replaceAll("-", "")}` - return `${parent}_m_${start.getUTCFullYear()}${String(start.getUTCMonth() + 1).padStart(2, "0")}` + return `${parent}_m_${String(start.getUTCFullYear()).padStart(4,"0")}${String(start.getUTCMonth() + 1).padStart(2, "0")}` } function rangeFor(kind: PartitionKind, ts: Date): { start: Date; end: Date } { @@ -106,59 +135,112 @@ function rangeFor(kind: PartitionKind, ts: Date): { start: Date; end: Date } { } export async function ensurePartitionFor( - pool: Pool, + pool: Pick, parent: string, kind: PartitionKind, ts: Date, ): Promise { + validatePartitionInput(parent,kind,ts) const { start, end } = rangeFor(kind, ts) + validatePartitionInput(parent,kind,start);validatePartitionInput(parent,kind,end) const name = partName(parent, kind, start) - const from = fmtDay(start) - const to = fmtDay(end) - await pool.query( - `CREATE TABLE IF NOT EXISTS ${name} PARTITION OF ${parent} FOR VALUES FROM ('${from}') TO ('${to}')`, - ) - if (FACT_PARENTS.has(parent)) { - await pool.query(`ALTER TABLE ${name} SET (${FACT_LEAF_STORAGE})`) + if (process.env.DATABASE_RUNTIME_REQUEST_FILE !== undefined) { + const rangeType = kind === "month" ? "date" : "timestamptz" + const from = kind === "month" ? fmtDay(start) : start.toISOString() + const to = kind === "month" ? fmtDay(end) : end.toISOString() + // Compare the catalog and expected bounds using the same session formatting. + const { rows } = await pool.query( + `SELECT 1 + FROM pg_catalog.pg_inherits inheritance + JOIN pg_catalog.pg_class leaf ON leaf.oid = inheritance.inhrelid + JOIN pg_catalog.pg_class parent ON parent.oid = inheritance.inhparent + JOIN pg_catalog.pg_namespace namespace ON namespace.oid = parent.relnamespace + WHERE namespace.nspname = 'public' AND leaf.relnamespace = namespace.oid + AND parent.relname = $1 AND leaf.relname = $2 + AND parent.relkind = 'p' AND leaf.relkind = 'r' AND leaf.relispartition + AND NOT inheritance.inhdetachpending + AND pg_catalog.pg_get_expr(leaf.relpartbound, leaf.oid, false) = + pg_catalog.format('FOR VALUES FROM (%L) TO (%L)', $3::pg_catalog.${rangeType}, $4::pg_catalog.${rangeType}) + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint required + WHERE required.conrelid = parent.oid AND required.convalidated AND required.conenforced + AND NOT required.connoinherit AND required.contype IN ('p', 'u', 'x', 'f', 'c', 'n') + AND NOT EXISTS ( + SELECT 1 FROM pg_catalog.pg_constraint actual + WHERE actual.conrelid = leaf.oid AND actual.contype = required.contype + AND actual.convalidated AND actual.conenforced + AND (actual.conparentid = required.oid OR ( + required.contype IN ('c', 'n') AND actual.coninhcount > 0 + AND pg_catalog.pg_get_constraintdef(actual.oid) = pg_catalog.pg_get_constraintdef(required.oid) + )) + ) + )`, + [parent, name, from, to], + ) + if (rows.length !== 1) { + throw new Error(`Required PostgreSQL partition ${name} is absent or incompatible; operator partition maintenance is required`) + } + return name } + // Bounds describe UTC instants, independently of the PostgreSQL session zone. + // A date-only literal would move a timestamptz boundary by that zone's offset. + const from = start.toISOString() + const to = end.toISOString() + await withPartitionDdl(pool,async client=>{ + const finishInheritance=await prepareReviewedPartitionGuardInheritance(client,parent,name) + await client.query(`CREATE TABLE IF NOT EXISTS ${name} PARTITION OF ${parent} FOR VALUES FROM ('${from}') TO ('${to}')`) + if (FACT_PARENTS.has(parent))await client.query(`ALTER TABLE ${name} SET (${FACT_LEAF_STORAGE})`) + await finishInheritance() + }) return name } export async function ensurePartitionsBetween( - pool: Pool, + pool: PartitionQueries, parent: string, kind: PartitionKind, from: Date, to: Date, ): Promise { + validatePartitionInput(parent,kind,from);validatePartitionInput(parent,kind,to) const start = from.getTime() <= to.getTime() ? from : to const end = from.getTime() <= to.getTime() ? to : from + const planned:Date[]=[] for (let t = new Date(start.getTime()); t <= end; ) { - await ensurePartitionFor(pool, parent, kind, t) + planned.push(t);if(planned.length*(FACT_PARENTS.has(parent)?2:1)>MAX_PARTITION_OPERATIONS)throw Error("Partition range exceeds the bounded maintenance budget") if (kind === "day") t = addUtcDays(t, 1) else if (kind === "week") t = addUtcDays(t, 7) else t = new Date(Date.UTC(t.getUTCFullYear(), t.getUTCMonth() + 1, 1)) } + for(const t of planned)await ensurePartitionFor(pool,parent,kind,t) } -export async function ensurePartitionsAround(pool: Pool, around = new Date()): Promise { +export async function ensurePartitionsAround(pool: PartitionQueries, around = new Date()): Promise { + validatePartitionInput("server_snapshots","week",around) const keepDays = await resolveKeepDays(pool) + const planned:{parent:string;kind:PartitionKind;at:Date}[]=[] + let cost=0 for (const spec of PARTITION_SPECS) { const keep = keepDays.get(spec.parent) ?? spec.keepDays const daysAhead = spec.kind === "month" ? 40 : spec.kind === "week" ? 21 : 8 const start = addUtcDays(around, -keep) const end = addUtcDays(around, daysAhead) for (let t = new Date(start.getTime()); t < end; ) { - await ensurePartitionFor(pool, spec.parent, spec.kind, t) + planned.push({parent:spec.parent,kind:spec.kind,at:t});cost+=FACT_PARENTS.has(spec.parent)?2:1;if(cost>MAX_PARTITION_OPERATIONS)throw Error("Partition catalogue exceeds the bounded maintenance budget") if (spec.kind === "day") t = addUtcDays(t, 1) else if (spec.kind === "week") t = addUtcDays(t, 7) else t = new Date(Date.UTC(t.getUTCFullYear(), t.getUTCMonth() + 1, 1)) } } + for(const item of planned)await ensurePartitionFor(pool,item.parent,item.kind,item.at) } -export async function dropExpiredPartitions(pool: Pool, around = new Date()): Promise { +export interface ExpiredPartition { parent: string; name: string } +/** One bounded retention manifest is shared by preview and the actual cleanup. */ +export async function listExpiredPartitions(pool: Pick, around = new Date()): Promise { + validatePartitionInput("server_snapshots","week",around) const keepDays = await resolveKeepDays(pool) + const expired: ExpiredPartition[] = [] for (const spec of PARTITION_SPECS) { const keep = keepDays.get(spec.parent) ?? spec.keepDays const cutoff = addUtcDays(around, -keep) @@ -167,12 +249,15 @@ export async function dropExpiredPartitions(pool: Pool, around = new Date()): Pr FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_class p ON p.oid = i.inhparent - WHERE p.relname = $1`, + WHERE p.relname = $1 AND p.relnamespace='public'::regnamespace AND c.relnamespace=p.relnamespace + ORDER BY c.relname LIMIT 1001`, [spec.parent], ) + if(rows.length>1000)throw new Error("Превышен предел разделов для обслуживания") for (const row of rows) { - const m = row.relname.match(/_([dwm])_(\d{8}|\d{6})$/) + const m = row.relname.match(new RegExp(`^${spec.parent}_([dwm])_(\\d{8}|\\d{6})$`)) if (!m) continue + if(m[1]!==spec.kind[0])continue const stamp = m[2] let start: Date if (stamp.length === 6) { @@ -180,11 +265,25 @@ export async function dropExpiredPartitions(pool: Pool, around = new Date()): Pr } else { start = new Date(`${stamp.slice(0, 4)}-${stamp.slice(4, 6)}-${stamp.slice(6, 8)}T00:00:00Z`) } - if (start < cutoff) { - await pool.query(`DROP TABLE IF EXISTS ${row.relname}`) - } + if(!Number.isFinite(start.getTime())||partName(spec.parent,spec.kind,start)!==row.relname)continue + if (start < cutoff) expired.push({parent:spec.parent,name:row.relname}) + if(expired.length>1000)throw new Error("Превышен предел истёкших разделов") } } + return expired +} +export async function dropExpiredPartitions(pool: PartitionQueries, around = new Date(), beforeDrop?: (partition: ExpiredPartition) => Promise): Promise { + const expired=await listExpiredPartitions(pool,around) + if (expired.length > 0 && process.env.DATABASE_RUNTIME_REQUEST_FILE !== undefined) { + throw new Error("Expired PostgreSQL partitions require operator maintenance") + } + for(const partition of expired){ + const quoted=partition.name.replace(/"/g,'""') + await withPartitionDdl(pool,async client=>{ + await beforeDrop?.(partition) + await client.query(`DROP TABLE IF EXISTS "public"."${quoted}"`) + }) + } } export function specForParent(parent: string): PartitionSpec | undefined { diff --git a/backend/src/db/pg-schema.test.ts b/backend/src/db/pg-schema.test.ts index e1b221d2..36e321de 100644 --- a/backend/src/db/pg-schema.test.ts +++ b/backend/src/db/pg-schema.test.ts @@ -3,6 +3,7 @@ import { withPgOrSkip } from "../test/pg.js" import { dbQuery, pool } from "./index.js" import { applySqlMigrations } from "./migrate.js" import { ensurePartitionFor } from "./partitions.js" +import { verifyPartitionTimezones } from "../test/partition-timezones.js" if (!(await withPgOrSkip())) { console.log("pg-schema.test.ts: skip") @@ -10,6 +11,7 @@ if (!(await withPgOrSkip())) { } await applySqlMigrations(pool) +await verifyPartitionTimezones(pool) { const { rows } = await dbQuery<{ n: string }>(`SELECT COUNT(*)::text AS n FROM servers`) diff --git a/backend/src/db/proposals/server-poll-settings-epoch.sql b/backend/src/db/proposals/server-poll-settings-epoch.sql new file mode 100644 index 00000000..3b31dfd4 --- /dev/null +++ b/backend/src/db/proposals/server-poll-settings-epoch.sql @@ -0,0 +1,46 @@ +-- ISOLATED PROPOSAL ONLY: normal startup does not discover this file. +-- Approval-time configuration identity; full execution context remains exact. +-- Caller must own the transaction. No migration marker or production activation. +LOCK TABLE public.traffic_settings, public.uptime_settings, + public.internet_path_settings IN ACCESS EXCLUSIVE MODE; +ALTER TABLE public.traffic_settings ADD COLUMN poll_configuration_epoch uuid NOT NULL DEFAULT pg_catalog.gen_random_uuid(); +ALTER TABLE public.uptime_settings ADD COLUMN poll_configuration_epoch uuid NOT NULL DEFAULT pg_catalog.gen_random_uuid(); +ALTER TABLE public.internet_path_settings ADD COLUMN poll_configuration_epoch uuid NOT NULL DEFAULT pg_catalog.gen_random_uuid(); +CREATE FUNCTION public.server_poll_settings_epoch_guard() RETURNS trigger +LANGUAGE plpgsql SET search_path = pg_catalog AS $epoch$ +BEGIN + IF TG_TABLE_SCHEMA <> 'public' OR TG_TABLE_NAME NOT IN + ('traffic_settings', 'uptime_settings', 'internet_path_settings') + OR TG_LEVEL <> 'ROW' OR TG_WHEN <> 'BEFORE' THEN + RAISE EXCEPTION 'unexpected poll settings epoch trigger target'; + END IF; + IF TG_OP = 'INSERT' THEN + -- Never accept an epoch supplied by an importer or application caller. + NEW.poll_configuration_epoch := pg_catalog.gen_random_uuid(); + ELSIF TG_OP = 'UPDATE' THEN + -- Every unknown/future field is configuration by default. Only these four + -- current collector fields are excluded; created_at stays significant. + IF (pg_catalog.to_jsonb(NEW) - ARRAY['poll_configuration_epoch', + 'last_collected_at', 'last_duration_ms', 'last_error', 'updated_at']) + IS DISTINCT FROM + (pg_catalog.to_jsonb(OLD) - ARRAY['poll_configuration_epoch', + 'last_collected_at', 'last_duration_ms', 'last_error', 'updated_at']) THEN + NEW.poll_configuration_epoch := pg_catalog.gen_random_uuid(); + ELSE + NEW.poll_configuration_epoch := OLD.poll_configuration_epoch; + END IF; + ELSE + RAISE EXCEPTION 'unexpected poll settings epoch trigger operation'; + END IF; + RETURN NEW; +END; +$epoch$; +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.traffic_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.uptime_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +CREATE TRIGGER server_poll_settings_epoch BEFORE INSERT OR UPDATE ON public.internet_path_settings +FOR EACH ROW EXECUTE FUNCTION public.server_poll_settings_epoch_guard(); +ALTER TABLE public.traffic_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; +ALTER TABLE public.uptime_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; +ALTER TABLE public.internet_path_settings ENABLE ALWAYS TRIGGER server_poll_settings_epoch; diff --git a/backend/src/db/proposals/server-poll-storage.sql b/backend/src/db/proposals/server-poll-storage.sql new file mode 100644 index 00000000..d7c09a7d --- /dev/null +++ b/backend/src/db/proposals/server-poll-storage.sql @@ -0,0 +1,170 @@ +-- ISOLATED PROPOSAL ONLY. Not discovered by normal migration startup. +-- Fixed logical quota; physical heap/index/WAL and vacuum remain independent. +-- Requires the 0038 poll journal. No backfill, scheduling or public activation. +BEGIN; +LOCK TABLE server_poll_executions,server_poll_intents,server_poll_acks,server_poll_receipts,server_poll_quarantines IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM server_poll_executions) OR EXISTS(SELECT 1 FROM server_poll_intents) + OR EXISTS(SELECT 1 FROM server_poll_acks) OR EXISTS(SELECT 1 FROM server_poll_receipts) + OR EXISTS(SELECT 1 FROM server_poll_quarantines) + THEN RAISE EXCEPTION 'nonempty poll journal needs explicit reconciliation'; END IF; + IF NOT EXISTS(SELECT 1 FROM mcp_settings WHERE id=1 AND requests_retention_days BETWEEN 1 AND 365) + THEN RAISE EXCEPTION 'poll retention unavailable'; END IF; +END $$; +CREATE TABLE server_poll_storage_usage ( + scope text PRIMARY KEY CHECK(scope='global' OR(scope LIKE 'owner:%' AND length(scope) BETWEEN 7 AND 206)), + admissions bigint NOT NULL DEFAULT 0 CHECK(admissions>=0 AND admissions<=128), + reserved_rows bigint NOT NULL DEFAULT 0 CHECK(reserved_rows=admissions*4019), + reserved_bytes bigint NOT NULL DEFAULT 0 CHECK(reserved_bytes=admissions*8421376), + CHECK(scope='global' OR admissions<=8) +); +CREATE TABLE server_poll_storage_admissions ( + receipt_id uuid PRIMARY KEY, identity_kind text NOT NULL CHECK(identity_kind IN('mcp','rest')), + execution_id uuid NOT NULL, connection_id uuid, owner_id text NOT NULL CHECK(length(owner_id) BETWEEN 1 AND 200 AND btrim(owner_id)=owner_id AND owner_id !~ '[[:cntrl:]]'), + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), context_proof text NOT NULL CHECK(context_proof ~ '^[a-f0-9]{64}$'), + authority_issued_at timestamptz NOT NULL CHECK(isfinite(authority_issued_at)), + authority_expires_at timestamptz NOT NULL CHECK(isfinite(authority_expires_at) AND authority_expires_at>authority_issued_at), + max_stages integer NOT NULL CHECK(max_stages BETWEEN 8 AND 2008), + claim_token uuid, claim_attempted_at timestamptz, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + retention_days integer NOT NULL CHECK(retention_days BETWEEN 1 AND 365), + safety_ms integer NOT NULL DEFAULT 60000 CHECK(safety_ms=60000), + gc_not_before timestamptz NOT NULL CHECK(isfinite(gc_not_before)), + used_rows integer NOT NULL DEFAULT 0 CHECK(used_rows BETWEEN 0 AND 4019 AND used_rows<=max_stages*2+3), + used_bytes bigint NOT NULL DEFAULT 0 CHECK(used_bytes BETWEEN 0 AND 8388608), + UNIQUE(identity_kind,execution_id), + CHECK((identity_kind='mcp' AND connection_id IS NOT NULL) OR(identity_kind='rest' AND connection_id IS NULL)), + CHECK((claim_token IS NOT NULL AND claim_attempted_at IS NULL) OR(claim_token IS NULL AND claim_attempted_at IS NOT NULL)), + CHECK(gc_not_before>=authority_expires_at+make_interval(days=>retention_days)+interval '60 seconds') +); +CREATE INDEX server_poll_storage_expiry ON server_poll_storage_admissions(gc_not_before,receipt_id); +INSERT INTO server_poll_storage_usage(scope) VALUES('global'); + +-- GUARDS: isolated historical fixtures may be inserted before this boundary; +-- production enrollment never copies or inserts historical admissions. +CREATE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' AS $$ +DECLARE a server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz; +BEGIN + IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF; + SELECT requests_retention_days INTO retention FROM mcp_settings WHERE mcp_settings.id=1 FOR SHARE; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM mcp_operations o + JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections c ON c.id=o.connection_id + WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll' + AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + FOR UPDATE OF o,p,c; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at, + COALESCE((SELECT created_at FROM server_poll_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM server_poll_intents WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(created_at) FROM server_poll_acks WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_poll_receipts WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond'=1 AND reserved_rows>=4019 AND reserved_bytes>=8421376; + GET DIAGNOSTICS charged=ROW_COUNT; + IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; + RETURN OLD; + END IF; + IF TG_OP='UPDATE' THEN + IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF; + IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL + AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp() + THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF; + IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at + AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF; + RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself'; + END IF; + SELECT requests_retention_days INTO retention FROM mcp_settings WHERE id=1; + IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() + OR NEW.authority_expires_at<=clock_timestamp() OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL + OR NEW.used_rows<>0 OR NEW.used_bytes<>0 + THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF; + IF EXISTS(SELECT 1 FROM server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) + THEN RAISE EXCEPTION 'poll identity already spent'; END IF; + NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000; + NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds'; + INSERT INTO server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING; + IF(SELECT count(*) FROM server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id) + AND admissions2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF; + UPDATE server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 + WHERE scope IN('global','owner:'||NEW.owner_id); + GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_storage_admission_guard BEFORE INSERT OR UPDATE OR DELETE ON server_poll_storage_admissions + FOR EACH ROW EXECUTE FUNCTION server_poll_storage_admission_guard(); + +CREATE FUNCTION server_poll_storage_row_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_poll_storage_admissions%ROWTYPE; n integer; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + SELECT * INTO a FROM server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'poll journal needs a committed admission'; END IF; + IF TG_TABLE_NAME='server_poll_quarantines' THEN + IF NEW.identity_kind<>a.identity_kind OR NEW.execution_id<>a.execution_id OR NEW.context_proof<>a.context_proof + THEN RAISE EXCEPTION 'poll quarantine admission mismatch'; END IF; + ELSE + IF a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'poll admission is not consumed and live'; END IF; + END IF; + IF TG_TABLE_NAME='server_poll_executions' THEN + IF NEW.identity_kind<>a.identity_kind OR NEW.execution_id<>a.execution_id OR NEW.context_proof<>a.context_proof OR NEW.binding_proof<>a.binding_proof + THEN RAISE EXCEPTION 'poll anchor admission mismatch'; END IF; + ELSIF TG_TABLE_NAME IN('server_poll_intents','server_poll_acks') THEN + IF NEW.stage LIKE 'publish.%' AND substring(NEW.stage FROM 9)::integer>=a.max_stages-4 THEN RAISE EXCEPTION 'poll stage exceeds private reservation'; END IF; + IF TG_TABLE_NAME='server_poll_intents' THEN SELECT count(*) INTO n FROM server_poll_intents WHERE receipt_id=NEW.receipt_id; + ELSE SELECT count(*) INTO n FROM server_poll_acks WHERE receipt_id=NEW.receipt_id; END IF; + IF n>=a.max_stages THEN RAISE EXCEPTION 'poll stage row budget exhausted'; END IF; + END IF; + IF TG_TABLE_NAME<>'server_poll_quarantines' AND EXISTS(SELECT 1 FROM server_poll_quarantines WHERE receipt_id=NEW.receipt_id) + THEN RAISE EXCEPTION 'poll identity quarantined'; END IF; + NEW.created_at:=clock_timestamp(); + RETURN NEW; +END $$; +-- Meter actual inserts, not BEFORE attempts: ON CONFLICT DO NOTHING must not +-- spend another row/byte allowance for a marker which was never inserted. +CREATE FUNCTION server_poll_storage_row_meter() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE cost bigint; n integer; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0)); + cost:=pg_column_size(NEW); + UPDATE server_poll_storage_admissions SET used_rows=used_rows+1,used_bytes=used_bytes+cost + WHERE receipt_id=NEW.receipt_id AND used_rows1 THEN RAISE EXCEPTION 'poll journal byte or row budget exhausted'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_poll_storage_anchor_guard BEFORE INSERT ON server_poll_executions FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_intent_guard BEFORE INSERT ON server_poll_intents FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_ack_guard BEFORE INSERT ON server_poll_acks FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_receipt_guard BEFORE INSERT ON server_poll_receipts FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_quarantine_guard BEFORE INSERT ON server_poll_quarantines FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_guard(); +CREATE TRIGGER server_poll_storage_anchor_meter AFTER INSERT ON server_poll_executions FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_intent_meter AFTER INSERT ON server_poll_intents FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_ack_meter AFTER INSERT ON server_poll_acks FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_receipt_meter AFTER INSERT ON server_poll_receipts FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE TRIGGER server_poll_storage_quarantine_meter AFTER INSERT ON server_poll_quarantines FOR EACH ROW EXECUTE FUNCTION server_poll_storage_row_meter(); +CREATE OR REPLACE FUNCTION server_poll_journal_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' AND server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'server poll journal is immutable before its full lifetime elapses'; +END $$; +COMMIT; diff --git a/backend/src/db/proposals/server-probe-legacy-0037-owned.ts b/backend/src/db/proposals/server-probe-legacy-0037-owned.ts new file mode 100644 index 00000000..43401c6d --- /dev/null +++ b/backend/src/db/proposals/server-probe-legacy-0037-owned.ts @@ -0,0 +1,113 @@ +/** Disposable acceptance kernel. No production dispatcher or migration imports it. */ +import assert from "node:assert/strict" +import { tmpdir } from "node:os" +import { basename, dirname, resolve } from "node:path" +import type { Client, Pool } from "pg" +import { acquireDatabaseSchemaControl } from "../database-schema-control.js" +import { readDatabaseSourceBindingRegistry, sourceBindingSha256, type DatabaseSourceDeploymentPin } from "../database-source-binding.js" +import { migrationManifestDigest } from "../migration-manifest.js" +import { readReviewedLegacyProbe0037 } from "../legacy-probe-0037-body.js" +import { withSystemDatabaseOperationLock } from "../operation-lock.js" +import { captureLegacyProbeLineage, captureLegacyProbeStorage, LEGACY_PROBE_CUSTODY_LOCK, LEGACY_PROBE_RELATIONS } from "../../services/server-probe-legacy-source-inventory.js" +import { verifyLegacyProbeArchiveApproval, type LegacyProbeArchiveApprovalInput } from "../../services/server-probe-legacy-archive-approval.js" +import { PARTITION_WRITER_ADVISORY_KEY } from "../../services/partition-write-transaction.js" +import { randomUUID } from "node:crypto" + +export const ownedLegacyProbe0037Body = readReviewedLegacyProbe0037 + +export async function runOwnedLegacyProbe0037Handoff(input: { + live: Pool; cold: Client; coldDatabase: string; fixtureDirectory: string + originalPin: DatabaseSourceDeploymentPin; archiveApproval: LegacyProbeArchiveApprovalInput; failAfterBody?: boolean +}) { + assert.equal(process.env.MMAPP_OWN_PROBE_LEGACY_0037, "true") + assert.equal(dirname(resolve(input.fixtureDirectory)), resolve(tmpdir())) + assert.match(basename(input.fixtureDirectory), /^mm-archive-leaf-tls-/) + assert.match(input.coldDatabase, /^mm_owned_legacy_cold_[0-9a-f]{32}$/) + const approved = await verifyLegacyProbeArchiveApproval(input.originalPin, input.archiveApproval) + assert.equal(approved.format, "legacy-probe-custody-approval-v1") + const original = (await readDatabaseSourceBindingRegistry(input.originalPin)).body.source + const { body, pin } = await ownedLegacyProbe0037Body() + const liveOwnerSql = `SELECT current_database() AS name, current_user::text AS actor, + pg_catalog.current_setting('data_directory') AS data_dir, inet_server_addr()::text AS host, + (pg_catalog.pg_control_system()).system_identifier::text AS system_identifier, + (SELECT oid::int FROM pg_catalog.pg_database WHERE datname=current_database()) AS database_oid` + const assertLiveOwned = (live: Record) => { + assert(live.name === "postgres" && live.actor === "archive_leaf_fixture" && live.host === "127.0.0.1/32") + assert.equal(resolve(String(live.data_dir)), resolve(input.fixtureDirectory, "pg")) + assert.equal(live.system_identifier, original.systemIdentifier) + assert.equal(live.database_oid, original.databaseOid) + } + assertLiveOwned((await input.live.query(liveOwnerSql)).rows[0]) + let control: Awaited> | undefined + try { + await input.cold.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + const { rows: [cold] } = await input.cold.query("SELECT current_database() AS name, current_user::text AS actor, inet_server_addr()::text AS host") + assert.deepEqual(cold, { name: input.coldDatabase, actor: "owned_legacy_reader", host: "127.0.0.1/32" }) + const preserved = await captureLegacyProbeStorage(input.cold) + const preservedLineage = await captureLegacyProbeLineage(input.cold) + assert(preserved.rowCount > 0) + assert.equal(preserved.sha256, approved.storageSha256) + assert.equal(preservedLineage.sha256, approved.lineageSha256) + const owner = await acquireDatabaseSchemaControl(input.live) + control = owner + + return await withSystemDatabaseOperationLock(() => input.live.connect(), async (app, discard) => { + const { rows: [barrier] } = await app.query("SELECT pg_catalog.pg_try_advisory_lock($1::bigint) AS held", [PARTITION_WRITER_ADVISORY_KEY]) + assert.equal(barrier.held, true) + let intentId: string | undefined, rollbackConfirmed = false, commitSent = false, commitAck = false + try { + await app.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ WRITE", []) + await app.query("SET LOCAL statement_timeout='30s'", []) + await app.query(LEGACY_PROBE_CUSTODY_LOCK.replace("IN SHARE MODE", "IN ACCESS EXCLUSIVE MODE"), []) + assertLiveOwned((await app.query(liveOwnerSql, [])).rows[0]) + const sql = { query: (text: string, values: unknown[] = []) => app.query(text, values) } + assert.deepEqual(await captureLegacyProbeStorage(sql), preserved) + assert.deepEqual(await captureLegacyProbeLineage(sql), preservedLineage) + await app.query("SET LOCAL synchronous_commit=on", []) + await app.query("SET LOCAL search_path=public,pg_catalog,pg_temp", []) + const { rows: [unexpectedTrigger] } = await app.query(`SELECT EXISTS(SELECT FROM pg_catalog.pg_trigger t + JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid WHERE c.relnamespace='public'::regnamespace + AND c.relname::text=ANY($1::text[]) AND (t.tgtype::int & 32)<>0) AS present`, [LEGACY_PROBE_RELATIONS]) + assert.equal(unexpectedTrigger.present, false) + const id = randomUUID() + await owner.transition({ type: "begin", id, kind: "migration", manifestSha256: migrationManifestDigest(), artifactSha256: pin.sha256 }) + intentId = id + await app.query(`TRUNCATE TABLE ${LEGACY_PROBE_RELATIONS.map(relation => `ONLY public.${relation}`).join(", ")} CONTINUE IDENTITY RESTRICT`, []) + await app.query(body, []) + if (input.failAfterBody) await app.query("SELECT 1 / 0", []) + const { rows: [current] } = await app.query(`SELECT + NOT EXISTS(SELECT FROM public.server_probe_storage_admissions) AS admissions_empty, + NOT EXISTS(SELECT FROM public.server_probe_execution_payloads) AS payloads_empty, + (${LEGACY_PROBE_RELATIONS.map(relation => `NOT EXISTS(SELECT FROM public.${relation})`).join(" AND ")}) AS journal_empty, + (SELECT count(*)=1 AND bool_and(scope='global' AND admissions=0 AND reserved_files=0 AND reserved_bytes=0) + FROM public.server_probe_storage_usage) AS usage_empty, + (SELECT count(*)=37 AND max(id)=$1 FROM public.schema_migrations) AS marker_exact`, [pin.file.slice(0, -4)]) + assert.deepEqual(current, { admissions_empty: true, payloads_empty: true, journal_empty: true, usage_empty: true, marker_exact: true }) + commitSent = true + assert.equal((await app.query("COMMIT", [])).command, "COMMIT") + commitAck = true + await owner.transition({ type: "terminal", id, outcome: "observed-success", + observationSha256: sourceBindingSha256(`owned-legacy-0037-handoff-v1\n${id}\n${approved.approvalSha256}`) }) + return { archivedRows: preserved.rowCount, lineageRows: preservedLineage.rowCount, readyAuthorized: false as const } + } catch (error) { + if (!commitSent) { + try { rollbackConfirmed = (await app.query("ROLLBACK", [])).command === "ROLLBACK" } catch { discard() } + } + if (intentId) await owner.transition({ type: "terminal", id: intentId, + outcome: rollbackConfirmed ? "observed-failure" : "interrupted-unknown", + observationSha256: sourceBindingSha256(`owned-legacy-0037-refused-v1\n${intentId}`) }).catch(() => {}) + if (!rollbackConfirmed) discard() + throw error + } finally { + if (rollbackConfirmed || commitAck) { + try { + const { rows: [result] } = await app.query("SELECT pg_catalog.pg_advisory_unlock($1::bigint) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY]) + if (result?.unlocked !== true) discard() + } catch { discard() } + } + } + }) + } finally { + try { await input.cold.query("ROLLBACK") } finally { await control?.close() } + } +} diff --git a/backend/src/db/proposals/server-probe-storage-v2.sql b/backend/src/db/proposals/server-probe-storage-v2.sql new file mode 100644 index 00000000..b62f392d --- /dev/null +++ b/backend/src/db/proposals/server-probe-storage-v2.sql @@ -0,0 +1,255 @@ +-- REVIEW / ISOLATED FIXTURE ONLY. Not a migration; never discovered by migrate. +-- User-approved bounded metadata GC, isolated fixture only. No scheduling or +-- production registration; migrations never discover this proposal. +-- Requires 0035, 0036 and mcp_settings. Stop old writers before a separately approved +-- real migration. Legacy orphan quarantines deliberately refuse backfill. +BEGIN; +LOCK TABLE server_probe_executions,server_probe_execution_stages,server_probe_execution_quarantines,mcp_diagnostic_cancellations IN ACCESS EXCLUSIVE MODE; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM server_probe_execution_quarantines q WHERE NOT EXISTS( + SELECT 1 FROM server_probe_executions e WHERE e.receipt_id=q.receipt_id + AND e.identity_kind=q.identity_kind AND e.execution_id=q.execution_id AND e.binding_proof=q.binding_proof)) + THEN RAISE EXCEPTION 'unmapped probe quarantine prevents storage conversion'; END IF; + IF NOT EXISTS(SELECT 1 FROM mcp_settings WHERE id=1 AND requests_retention_days BETWEEN 1 AND 365) + THEN RAISE EXCEPTION 'authoritative probe retention setting unavailable'; END IF; +END $$; + +CREATE TABLE server_probe_storage_usage ( + scope text PRIMARY KEY CHECK(scope='global' OR (scope LIKE 'owner:%' AND length(scope) BETWEEN 7 AND 206)), + admissions bigint NOT NULL DEFAULT 0 CHECK(admissions>=0), + reserved_files bigint NOT NULL DEFAULT 0 CHECK(reserved_files>=0 AND reserved_files<=admissions), + reserved_bytes bigint NOT NULL DEFAULT 0 CHECK(reserved_bytes>=0), + CHECK(reserved_bytes=admissions*32768+reserved_files*262144) +); +CREATE TABLE server_probe_storage_admissions ( + receipt_id uuid PRIMARY KEY, + owner_id text NOT NULL CHECK(length(owner_id) BETWEEN 1 AND 200), + identity_kind text NOT NULL CHECK(identity_kind IN ('mcp','rest')), execution_id uuid NOT NULL, + binding_proof text NOT NULL CHECK(binding_proof ~ '^[a-f0-9]{64}$'), + authority_expires_at timestamptz NOT NULL, + claim_token uuid, claim_attempted_at timestamptz, + purged_at timestamptz, cancelled_at timestamptz, created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + retention_days integer NOT NULL CHECK(retention_days BETWEEN 1 AND 365), + safety_ms integer NOT NULL DEFAULT 60000 CHECK(safety_ms BETWEEN 60000 AND 86400000), + gc_not_before timestamptz NOT NULL DEFAULT 'infinity', + CHECK(gc_not_before>=authority_expires_at+make_interval(days=>retention_days)+safety_ms*interval '1 millisecond'), + UNIQUE(identity_kind,execution_id), + CHECK(claim_token IS NULL OR (claim_attempted_at IS NULL AND purged_at IS NULL AND cancelled_at IS NULL)), + CHECK(cancelled_at IS NULL OR (claim_token IS NULL AND claim_attempted_at IS NOT NULL)) +); +CREATE INDEX server_probe_storage_admissions_owner_age ON server_probe_storage_admissions(owner_id,authority_expires_at,receipt_id); +CREATE TABLE server_probe_execution_payloads ( + receipt_id uuid PRIMARY KEY REFERENCES server_probe_executions(receipt_id), + receipt_ciphertext text NOT NULL CHECK(octet_length(receipt_ciphertext) BETWEEN 1 AND 262144), + receipt_proof text NOT NULL CHECK(receipt_proof ~ '^[a-f0-9]{64}$') +); +INSERT INTO server_probe_storage_admissions(receipt_id,owner_id,identity_kind,execution_id,binding_proof,authority_expires_at,claim_attempted_at,created_at,retention_days,safety_ms,gc_not_before) + SELECT receipt_id,actor_id,identity_kind,execution_id,binding_proof,authority_expires_at,created_at,created_at, + (SELECT requests_retention_days FROM mcp_settings WHERE id=1),60000, + GREATEST(authority_expires_at,created_at)+make_interval(days=>(SELECT requests_retention_days FROM mcp_settings WHERE id=1))+interval '1 minute' FROM server_probe_executions; +-- Legacy cancellation has no receipt/binding proof of its own. Only an exact +-- native execution/owner/connection/operation mapping can reuse a charged +-- admission. Orphans REFUSE conversion: a separate authenticated preflight and +-- backfill proposal is required; target-only evidence never fabricates one. +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM mcp_diagnostic_cancellations c WHERE NOT EXISTS( + SELECT 1 FROM server_probe_storage_admissions a JOIN server_probe_executions e ON e.receipt_id=a.receipt_id + JOIN mcp_operations o ON o.id=c.operation_id AND o.connection_id=c.connection_id + JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + WHERE a.identity_kind='mcp' AND a.execution_id=c.operation_id AND a.owner_id=c.actor_id + AND e.actor_id=a.owner_id AND e.identity_kind=a.identity_kind AND e.execution_id=a.execution_id + AND e.binding_proof=a.binding_proof AND e.authority_expires_at=a.authority_expires_at + AND p.operation_id='servers.probes.run')) + THEN RAISE EXCEPTION 'unmapped diagnostic cancellation prevents storage conversion; authenticated backfill required'; END IF; +END $$; +UPDATE server_probe_storage_admissions a SET cancelled_at=c.created_at + FROM mcp_diagnostic_cancellations c WHERE a.identity_kind='mcp' AND a.execution_id=c.operation_id AND a.owner_id=c.actor_id; +INSERT INTO server_probe_execution_payloads(receipt_id,receipt_ciphertext,receipt_proof) + SELECT receipt_id,receipt_ciphertext,receipt_proof FROM server_probe_executions WHERE state<>'claimed'; +DO $$ BEGIN + IF EXISTS(SELECT 1 FROM server_probe_executions e LEFT JOIN server_probe_execution_payloads p USING(receipt_id) + WHERE e.state<>'claimed' AND (p.receipt_id IS NULL OR p.receipt_ciphertext IS DISTINCT FROM e.receipt_ciphertext OR p.receipt_proof IS DISTINCT FROM e.receipt_proof)) + THEN RAISE EXCEPTION 'probe payload copy mismatch'; END IF; +END $$; +INSERT INTO server_probe_storage_usage(scope,admissions,reserved_files,reserved_bytes) + SELECT 'global',count(*),count(*),count(*)*294912 FROM server_probe_storage_admissions; +INSERT INTO server_probe_storage_usage(scope,admissions,reserved_files,reserved_bytes) + SELECT 'owner:'||owner_id,count(*),count(*),count(*)*294912 FROM server_probe_storage_admissions GROUP BY owner_id; + +-- PostgreSQL drops table CHECKs dependent on this column together with it. +-- Copy verification above precedes the destructive *proposal* transformation. +ALTER TABLE server_probe_executions DROP COLUMN receipt_ciphertext; +ALTER TABLE server_probe_executions ADD CONSTRAINT server_probe_storage_terminal_state CHECK( + (state='claimed' AND receipt_proof IS NULL AND finished_at IS NULL AND attempted_stages IS NULL AND completed_stages IS NULL) + OR(state<>'claimed' AND receipt_proof IS NOT NULL AND finished_at IS NOT NULL AND attempted_stages IS NOT NULL AND completed_stages IS NOT NULL)); + +-- Predicate only: it performs no DELETE. Every guarded exception below must +-- independently satisfy the persisted lifetime, actual current retention and +-- original terminal operation. The GUC alone never grants cleanup authority. +CREATE FUNCTION server_probe_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; activity timestamptz; current_retention integer; terminal_at timestamptz; +BEGIN + IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF; + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=id FOR UPDATE; + IF NOT FOUND OR a.gc_not_before>=clock_timestamp() OR a.authority_expires_at>=clock_timestamp() THEN RETURN false; END IF; + SELECT settings.requests_retention_days INTO current_retention FROM mcp_settings settings WHERE settings.id=1; + IF current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF; + terminal_at:=a.created_at; + IF a.identity_kind='mcp' THEN + SELECT GREATEST(o.finished_at,p.expires_at,COALESCE(cancellation.created_at,o.finished_at)) INTO terminal_at + FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id + JOIN mcp_connections owner_connection ON owner_connection.id=o.connection_id + LEFT JOIN mcp_diagnostic_cancellations cancellation ON cancellation.operation_id=o.id + WHERE o.id=a.execution_id AND owner_connection.owner_id=a.owner_id AND p.operation_id='servers.probes.run' + AND o.state IN ('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL + AND (cancellation.operation_id IS NULL OR(cancellation.connection_id=o.connection_id AND cancellation.actor_id=a.owner_id)) + FOR UPDATE OF o,p,owner_connection; + IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF; + END IF; + SELECT GREATEST(a.created_at,a.authority_expires_at,terminal_at,COALESCE(a.cancelled_at,a.created_at), + COALESCE((SELECT finished_at FROM server_probe_executions WHERE receipt_id=id),a.created_at), + COALESCE((SELECT created_at FROM server_probe_execution_quarantines WHERE receipt_id=id),a.created_at), + COALESCE((SELECT max(GREATEST(created_at,COALESCE(received_at,created_at))) FROM server_probe_execution_stages WHERE receipt_id=id),a.created_at)) INTO activity; + RETURN activity+make_interval(days=>GREATEST(a.retention_days,current_retention))+a.safety_ms*interval '1 millisecond'clock_timestamp() + THEN RETURN NEW; END IF; + IF OLD.purged_at IS NULL AND NEW.purged_at IS NOT NULL AND NEW.claim_token IS NULL + AND NEW.claim_attempted_at IS NOT DISTINCT FROM OLD.claim_attempted_at THEN + SELECT GREATEST(OLD.authority_expires_at,COALESCE(e.finished_at,OLD.authority_expires_at),COALESCE(q.created_at,OLD.authority_expires_at)) INTO activity + FROM (SELECT 1) x LEFT JOIN server_probe_executions e ON e.receipt_id=OLD.receipt_id + LEFT JOIN server_probe_execution_quarantines q ON q.receipt_id=OLD.receipt_id; + IF activityGREATEST(OLD.retention_days,current_retention)) THEN RETURN NEW; END IF; + END IF; + RAISE EXCEPTION 'probe admission transition refused'; +END $$; +CREATE TRIGGER server_probe_storage_admission_guard BEFORE INSERT OR UPDATE OR DELETE ON server_probe_storage_admissions + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_admission_guard(); + +CREATE FUNCTION server_probe_storage_anchor_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR a.owner_id<>NEW.actor_id OR a.identity_kind<>NEW.identity_kind OR a.execution_id<>NEW.execution_id + OR a.binding_proof<>NEW.binding_proof OR a.authority_expires_at<>NEW.authority_expires_at + OR a.purged_at IS NOT NULL OR a.cancelled_at IS NOT NULL OR a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'probe anchor has no live consumed admission'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_probe_storage_anchor_guard BEFORE INSERT OR UPDATE ON server_probe_executions + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_anchor_guard(); + +CREATE OR REPLACE FUNCTION server_probe_execution_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN + IF server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe execution lifetime has not elapsed'; + END IF; + IF OLD.state<>'claimed' OR (to_jsonb(OLD)-ARRAY['state','attempted_stages','completed_stages','receipt_proof','finished_at']) IS DISTINCT FROM + (to_jsonb(NEW)-ARRAY['state','attempted_stages','completed_stages','receipt_proof','finished_at']) OR NEW.state='claimed' + THEN RAISE EXCEPTION 'probe execution identity or terminal receipt is immutable'; END IF; + IF EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=OLD.receipt_id) + THEN RAISE EXCEPTION 'probe execution quarantined'; END IF; + RETURN NEW; END $$; + +CREATE FUNCTION server_probe_storage_stage_guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id AND purged_at IS NULL AND cancelled_at IS NULL + AND claim_token IS NULL AND claim_attempted_at IS NOT NULL AND authority_expires_at>clock_timestamp() FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'probe stage has no live admission'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_storage_stage_guard BEFORE INSERT OR UPDATE ON server_probe_execution_stages + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_stage_guard(); + +CREATE FUNCTION server_probe_storage_quarantine_guard() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id AND identity_kind=NEW.identity_kind + AND execution_id=NEW.execution_id AND binding_proof=NEW.binding_proof FOR UPDATE; + IF NOT FOUND THEN RAISE EXCEPTION 'probe quarantine has no committed matching admission'; END IF; + RETURN NEW; END $$; +CREATE TRIGGER server_probe_storage_quarantine_guard BEFORE INSERT ON server_probe_execution_quarantines + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_quarantine_guard(); + +CREATE FUNCTION server_probe_storage_payload_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE a server_probe_storage_admissions%ROWTYPE; e server_probe_executions%ROWTYPE; +BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)); + IF TG_OP='UPDATE' THEN RAISE EXCEPTION 'probe payload is immutable'; END IF; + IF TG_OP='DELETE' THEN + PERFORM receipt_id FROM server_probe_storage_admissions WHERE receipt_id=OLD.receipt_id AND purged_at IS NOT NULL FOR UPDATE; + IF NOT FOUND AND NOT server_probe_storage_gc_allowed(OLD.receipt_id) THEN RAISE EXCEPTION 'probe payload is not purged'; END IF; + RETURN OLD; + END IF; + SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR a.purged_at IS NOT NULL OR a.cancelled_at IS NOT NULL OR a.claim_token IS NOT NULL OR a.claim_attempted_at IS NULL OR a.authority_expires_at<=clock_timestamp() + THEN RAISE EXCEPTION 'probe payload has no live admission'; END IF; + SELECT * INTO e FROM server_probe_executions WHERE receipt_id=NEW.receipt_id FOR UPDATE; + IF NOT FOUND OR e.state='claimed' OR e.receipt_proof<>NEW.receipt_proof OR e.actor_id<>a.owner_id OR e.binding_proof<>a.binding_proof + OR e.identity_kind<>a.identity_kind OR e.execution_id<>a.execution_id OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines q + WHERE q.receipt_id=NEW.receipt_id OR(q.identity_kind=a.identity_kind AND q.execution_id=a.execution_id)) + THEN RAISE EXCEPTION 'probe payload terminal identity mismatch or quarantine'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER server_probe_storage_payload_guard BEFORE INSERT OR UPDATE OR DELETE ON server_probe_execution_payloads + FOR EACH ROW EXECUTE FUNCTION server_probe_storage_payload_guard(); +-- Preserve every original INSERT/UPDATE rule; only the already-validated +-- lifetime predicate can admit DELETE in this isolated proposal. +CREATE OR REPLACE FUNCTION server_probe_stage_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' THEN + IF server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe stage lifetime has not elapsed'; + END IF; + PERFORM receipt_id FROM server_probe_executions WHERE receipt_id=NEW.receipt_id AND state='claimed' FOR UPDATE; + IF NOT FOUND OR EXISTS(SELECT 1 FROM server_probe_execution_quarantines WHERE receipt_id=NEW.receipt_id) THEN RAISE EXCEPTION 'probe attempt is not executable'; END IF; + IF TG_OP='INSERT' AND (NEW.stage_index<>(SELECT COALESCE(max(stage_index),0)+1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id) + OR EXISTS(SELECT 1 FROM server_probe_execution_stages WHERE receipt_id=NEW.receipt_id AND state<>'received')) + THEN RAISE EXCEPTION 'probe stages must be serial and contiguous'; END IF; + IF TG_OP='UPDATE' AND (OLD.state<>'intent' OR NEW.state<>'received' OR (to_jsonb(OLD)-ARRAY['state','outcome','received_at']) IS DISTINCT FROM (to_jsonb(NEW)-ARRAY['state','outcome','received_at'])) + THEN RAISE EXCEPTION 'probe stage terminal state is immutable'; END IF; + RETURN NEW; END $$; +CREATE OR REPLACE FUNCTION server_probe_quarantine_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN + IF TG_OP='DELETE' AND server_probe_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF; + RAISE EXCEPTION 'probe quarantine remains immutable'; END $$; +CREATE OR REPLACE FUNCTION mcp_diagnostic_cancellation_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE original_receipt uuid; +BEGIN + IF TG_OP='DELETE' THEN + SELECT a.receipt_id INTO original_receipt FROM server_probe_storage_admissions a + JOIN mcp_operations o ON o.id=a.execution_id AND o.connection_id=OLD.connection_id + WHERE a.identity_kind='mcp' AND a.execution_id=OLD.operation_id AND a.owner_id=OLD.actor_id; + IF FOUND AND server_probe_storage_gc_allowed(original_receipt) THEN RETURN OLD; END IF; + END IF; + RAISE EXCEPTION 'diagnostic cancellation lifetime has not elapsed'; +END $$; +COMMIT; diff --git a/backend/src/db/routing-reservation-schema.owned.test.ts b/backend/src/db/routing-reservation-schema.owned.test.ts new file mode 100644 index 00000000..91bf653b --- /dev/null +++ b/backend/src/db/routing-reservation-schema.owned.test.ts @@ -0,0 +1,121 @@ +/** Numbered schema acceptance only. Synthetic legacy rows are never native + * execution/checkpoint authority. No controller, router or external network. */ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { readFile } from "node:fs/promises" +import { basename, dirname, isAbsolute, resolve } from "node:path" +import { spawn } from "node:child_process" + +async function main() { + assert.equal(process.env.MMAPP_OWN_ROUTING_RESERVATION_SCHEMA, "true") + assert.equal(process.env.NODE_ENV, "test") + const endpoint = new URL(process.env.DATABASE_URL ?? "") + assert(["postgres:", "postgresql:"].includes(endpoint.protocol)) + assert.equal(endpoint.hostname, "127.0.0.1"); assert.equal(endpoint.username, "mcp_test") + assert.equal(endpoint.pathname, "/postgres"); assert(endpoint.password) + assert(Number.isInteger(Number(endpoint.port)) && Number(endpoint.port) > 0 && Number(endpoint.port) <= 65535) + assert.equal(endpoint.search, ""); assert.equal(endpoint.hash, "") + assert(process.env.MCP_STATE_DIR && isAbsolute(process.env.MCP_STATE_DIR)) + assert.equal(basename(resolve(process.env.MCP_STATE_DIR)), "state") + assert(basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) + const { Pool } = await import("pg") + const { pool, closePool } = await import("./index.js") + const { applySqlMigrations } = await import("./migrate.js") + const { MIGRATION_MANIFEST, analyzeMigration } = await import("./migration-manifest.js") + const files = await Promise.all(MIGRATION_MANIFEST.map(async pin => { + const bytes = await readFile(new URL("../../drizzle/" + pin.file, import.meta.url)) + analyzeMigration(pin.file, bytes, pin); return { pin, sql: bytes.toString("utf8") } + })) + assert.equal(files.length, 40); assert.equal(files.at(-1)!.pin.file, "0040_mcp_routing_reservations.sql") + const names: string[] = [], clients = new Set>(), groups: string[] = [] + const identifier = (name: string) => { assert(/^mm_rt_schema_[a-f0-9]{32}$/.test(name)); return '"' + name + '"' } + const legacy = async () => { + const name = "mm_rt_schema_" + randomUUID().replaceAll("-", "") + await pool.query(`CREATE DATABASE ${identifier(name)} TEMPLATE template0`); names.push(name) + const url = new URL(endpoint); url.pathname = "/" + name + const target = new Pool({ connectionString: url.toString(), max: 2 }); clients.add(target) + const owner = await target.connect() + try { + await owner.query("CREATE TABLE schema_migrations(id text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now())") + for (const file of files.slice(0, -1)) { + if (file.pin.transaction === "owner-transaction") await owner.query("BEGIN") + await owner.query(file.sql) + if (!file.pin.selfMarker) await owner.query("INSERT INTO schema_migrations(id) VALUES($1)", [file.pin.file.slice(0, -4)]) + if (file.pin.transaction === "owner-transaction") await owner.query("COMMIT") + } + } finally { owner.release(true) } + return target + } + const snapshot = async (target: InstanceType, projected = false) => { + const output: Record = {} + for (const table of ["mcp_connections", "mcp_plans", "mcp_operations", "mcp_routing_profiles", "mcp_budget_buckets", "mcp_scoped_recovery_intents", "servers", "traffic_flow_settings"]) + output[table] = (await target.query(`SELECT to_jsonb(t) ${projected && table === "mcp_scoped_recovery_intents" ? "- ARRAY['reservation_id','slot_phase','inverse_intent_sealed','inverse_intent_proof','inverse_configuration_sealed','inverse_configuration_proof']" : ""} AS value FROM ${table} t ORDER BY to_jsonb(t)::text`)).rows.map(row => row.value) + return output + } + const seed = async (target: InstanceType, reserved = false) => { + const connection = randomUUID(), profile = randomUUID(), operations: string[] = [] + await target.query("INSERT INTO mcp_connections(id,name,owner_id,owner_permissions,policy,generation) VALUES($1,'owned schema','owned-schema','[]','{}','owned-generation')", [connection]) + await target.query("INSERT INTO mcp_routing_profiles(id,connection_id,config,state,enabled,grant_expires_at) VALUES($1,$2,'{}','{}',false,now()+interval '1 hour')", [profile, connection]) + await target.query("INSERT INTO mcp_budget_buckets(connection_id,kind,bucket,used) VALUES($1,'mutations',date_trunc('hour',now()),7)", [connection]) + for (const state of ["intent", "verified", "unknown"]) { + const plan = randomUUID(), operation = randomUUID(); operations.push(operation) + await target.query("INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at) VALUES($1,$2,'routing.community.switch',7,1,'owned-generation','{\"sealed\":\"owned-plan-bytes\"}','{\"sealed\":\"owned-evidence-bytes\"}','completed',now()+interval '1 hour')", [plan, connection]) + await target.query("INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state) VALUES($1,$2,$3,$4,'partial')", [operation, plan, connection, randomUUID()]) + await target.query(`INSERT INTO mcp_scoped_recovery_intents(source_operation_id,source_step_index,source_receipt_id,source_configuration_proof,recovery_receipt_id,mode,authority_profile_id,authority_profile_version,intent_sealed,intent_proof,state,deadline,verified_sealed,verified_proof) + VALUES($1,0,$2,'owned-source-proof',$3,$4,$5,1,'owned-intent-bytes','owned-intent-proof',$6,now()+interval '1 hour',$7,$8)`, + [operation, randomUUID(), randomUUID(), reserved ? "reserved" : "manual", profile, state, state === "verified" ? "owned-verified-bytes" : null, state === "verified" ? "owned-verified-proof" : null]) + } + return { connection, profile, operations } + } + const assertRefused = async (target: InstanceType, reason: RegExp) => { + const before = await snapshot(target) + const owner = await target.connect() + try { await assert.rejects(owner.query(files.at(-1)!.sql), reason) } + finally { await owner.query("ROLLBACK"); owner.release() } + assert.deepEqual(await snapshot(target), before) + assert.equal((await target.query("SELECT count(*)::int AS n FROM schema_migrations WHERE id='0040_mcp_routing_reservations'")).rows[0].n, 0) + assert.equal((await target.query("SELECT count(*)::int AS n FROM pg_attribute WHERE attrelid='mcp_scoped_recovery_intents'::regclass AND attname='inverse_configuration_proof' AND NOT attisdropped")).rows[0].n, 0) + } + try { + await applySqlMigrations(pool) + assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, 40) + for (const table of ["mcp_routing_reservations", "mcp_routing_reservation_slots"]) + assert.equal((await pool.query(`SELECT count(*)::int AS n FROM ${table}`)).rows[0].n, 0) + groups.push("fresh full40 numbered chain creates empty inactive ledgers") + const upgraded = await legacy(); await seed(upgraded) + const before = await snapshot(upgraded) + await applySqlMigrations(upgraded) + assert.deepEqual(await snapshot(upgraded, true), before) + assert.deepEqual((await upgraded.query("SELECT DISTINCT mode,slot_phase,reservation_id,inverse_intent_sealed,inverse_configuration_sealed FROM mcp_scoped_recovery_intents")).rows, + [{ mode: "manual", slot_phase: "recovery-intent", reservation_id: null, inverse_intent_sealed: null, inverse_configuration_sealed: null }]) + const once = await snapshot(upgraded); await applySqlMigrations(upgraded); assert.deepEqual(await snapshot(upgraded), once) + groups.push("nonempty legacy39 manual intent/verified/unknown and profiles/budget/ciphertext preserved; repeated startup skips0040") + for (const update of ["source_configuration_proof=NULL", "slot_phase='reserved-attempt'", "inverse_intent_sealed='forged'", "inverse_intent_sealed='forged',inverse_intent_proof='forged'"]) + await assert.rejects(upgraded.query(`UPDATE mcp_scoped_recovery_intents SET ${update}`), (error: unknown) => (error as { code: string }).code === "23514") + assert.deepEqual(await snapshot(upgraded), once) + groups.push("real constraints reject manual mode/phase/null source proof/unpaired or reserved inverse evidence") + const oldReserved = await legacy(); await seed(oldReserved, true); await assertRefused(oldReserved, /legacy reserved recovery/) + groups.push("legacy reserved evidence refuses atomically without fabricated debit, capability or historical backfill") + for (const ddl of ["CREATE TABLE mcp_routing_reservations(id uuid)", "ALTER TABLE mcp_scoped_recovery_intents ADD COLUMN reservation_id uuid", "ALTER TABLE mcp_scoped_recovery_intents ADD CONSTRAINT reserved_inverse_evidence_pairs CHECK(true)"]) { + const collision = await legacy(); await seed(collision); await collision.query(ddl); await assertRefused(collision, /unmarked routing reservation/) + } + groups.push("unmarked proposal table/column/constraint refuses atomically; no enrollment marker or subsequent inverse columns") + // New process cannot reconstruct either capability from persisted/serialized + // kind JSON. No running operation or signed checkpoint is fabricated here. + const storeUrl = new URL("../modules/mcp/routing-reservation-store.ts", import.meta.url).href + const dbUrl = new URL("./index.ts", import.meta.url).href + const script = `import assert from 'node:assert/strict';const {createRoutingReservationStore}=await import(${JSON.stringify(storeUrl)});const store=createRoutingReservationStore();await assert.rejects(store.beginInverseAttempt(JSON.parse('{"kind":"routing-reserved-inverse-allocation"}')),/capability/);await assert.rejects(store.inspectInverseExecution(JSON.parse('{"kind":"routing-reserved-inverse-execution"}')),/capability/);const {closePool}=await import(${JSON.stringify(dbUrl)});await closePool();` + await new Promise((done, reject) => { + const child = spawn(process.execPath, ["--import", "tsx", "--input-type=module", "-e", script], { stdio: "inherit", windowsHide: true }) + child.once("error", reject); child.once("exit", code => code === 0 ? done() : reject(Error("Owned capability restart check failed"))) + }) + groups.push("fresh process refuses serialized inverse allocation/execution objects; schema does not mint capabilities") + console.log(JSON.stringify({ accepted: groups.length, groups, numberedMigration: "0040_mcp_routing_reservations", migrationCount: 40, + actualSql: true, proposalDdlExecuted: false, legacyManualDataPreserved: true, automaticBackfill: false, + readyAuthorized: false, reservedNativeDispatchEnabled: false, genuineReservedLifecycleAccepted: false })) + } finally { + try { for (const target of clients) await target.end(); for (const name of names.reverse()) await pool.query(`DROP DATABASE ${identifier(name)}`) } + finally { await closePool() } + } +} +await main() diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts index cfddaf3e..4ba9684a 100644 --- a/backend/src/db/schema.ts +++ b/backend/src/db/schema.ts @@ -2,6 +2,7 @@ import { sql } from "drizzle-orm" import { bigint, boolean, + check, date, doublePrecision, index, @@ -14,6 +15,8 @@ import { text, timestamp, uniqueIndex, + unique, + uuid, } from "drizzle-orm/pg-core" function ts(name: string) { @@ -59,6 +62,7 @@ export const servers = pgTable("servers", { password: text("password").notNull().default(""), useSsl: boolean("use_ssl").notNull().default(true), verifySsl: boolean("verify_ssl").notNull().default(false), + routingEndpointEpoch: uuid("routing_endpoint_epoch").notNull().defaultRandom(), type: text("type", { enum: ["jump-host", "exit-node", "home-router"] }) .notNull().default("home-router"), site: text("site").notNull().default(""), @@ -143,6 +147,7 @@ export const recursiveRoutes = pgTable("recursive_routes", { ]) export const trafficSettings = pgTable("traffic_settings", { + pollConfigurationEpoch: uuid("poll_configuration_epoch").notNull().defaultRandom(), id: idSingleton(), enabled: boolean("enabled").notNull().default(true), intervalSec: integer("interval_sec").notNull().default(30), @@ -342,6 +347,7 @@ export const trafficSamples = pgTable("traffic_samples", { ]) export const uptimeSettings = pgTable("uptime_settings", { + pollConfigurationEpoch: uuid("poll_configuration_epoch").notNull().defaultRandom(), id: idSingleton(), enabled: boolean("enabled").notNull().default(true), resourcesEnabled: boolean("resources_enabled").notNull().default(true), @@ -715,6 +721,7 @@ export const uptimeSpeedTestRuns = pgTable("uptime_speed_test_runs", { ]) export const internetPathSettings = pgTable("internet_path_settings", { + pollConfigurationEpoch: uuid("poll_configuration_epoch").notNull().defaultRandom(), id: idSingleton(), enabled: boolean("enabled").notNull().default(true), intervalSec: integer("interval_sec").notNull().default(300), @@ -746,7 +753,7 @@ export const userInterfaceBindings = pgTable("user_interface_bindings", { userId: text("user_id").notNull().references(() => appUsers.id, { onDelete: "cascade" }), serverId: intPkRef().references(() => servers.id, { onDelete: "cascade" }), interfaceName: text("interface_name").notNull(), - interfaceType: text("interface_type", { enum: ["ether", "gre", "wg", "ipsec", "other"] }) + interfaceType: text("interface_type", { enum: ["ether", "bridge", "vlan", "vxlan", "gre", "wg", "ipsec", "other"] }) .notNull().default("other"), peerPublicKey: text("peer_public_key").notNull().default(""), peerName: text("peer_name").notNull().default(""), @@ -813,5 +820,89 @@ export type AlertEngineStateRow = typeof alertEngineState.$inferSelect export type AlertHistoryRow = typeof alertHistory.$inferSelect export type AlertOutboxRow = typeof alertOutbox.$inferSelect export type AlertEngineCursorRow = typeof alertEngineCursor.$inferSelect +export const flowWriteReceipts = pgTable("flow_write_receipts", { + batchId: uuid("batch_id").primaryKey(), + kind: text("kind", { enum: ["ingest", "facts", "rebuild"] }).notNull(), + sourceProof: text("source_proof").notNull(), + result: jsonb("result").$type>().notNull(), + committedAt: ts("committed_at").notNull().defaultNow(), +}) + +export const flowSettingsIntents = pgTable("flow_settings_intents", { + receiptId: uuid("receipt_id").notNull(), + phase: text("phase", { enum: ["configuration", "listener"] }).notNull(), + requestProof: text("request_proof").notNull(), + state: text("state", { enum: ["claimed", "completed"] }).notNull().default("claimed"), + result: jsonb("result"), + createdAt: ts("created_at").notNull().defaultNow(), +}, table => [primaryKey({ columns: [table.receiptId, table.phase] })]) + export type AppUserRow = typeof appUsers.$inferSelect export type UserInterfaceBindingRow = typeof userInterfaceBindings.$inferSelect + +// mcp_plans/mcp_connections live in SQL migration 0015; migration 0034 +// supplies their foreign keys. This receipt never authorizes native execution. +export const vrfProductionTickets = pgTable("vrf_production_tickets", { + ticketId: uuid("ticket_id").primaryKey(), + planId: uuid("plan_id").notNull().unique(), + connectionId: uuid("connection_id").notNull(), + serverId: bigint("server_id", { mode: "number" }).notNull().references(() => servers.id), + ownerId: text("owner_id").notNull(), + actionId: text("action_id").notNull(), + actionVersion: integer("action_version").notNull(), + sourceProof: text("source_proof").notNull(), + digest: text("digest").notNull(), + ciphertext: text("ciphertext").notNull(), + state: text("state", { enum: ["issued", "claimed"] }).notNull().default("issued"), + issuedAt: ts("issued_at").notNull(), + expiresAt: ts("expires_at").notNull(), + claimedAt: ts("claimed_at"), +}) + +// Historical charge identities deliberately have no lifetime FK/cascade. +// Authority and encrypted proof are checked by routing-reservation-store. +export const mcpRoutingForwardCharges = pgTable("mcp_routing_forward_charges", { + sourceOperationId: uuid("source_operation_id").primaryKey(), + sourcePlanId: uuid("source_plan_id").notNull(), + sourceStepIndex: integer("source_step_index").notNull(), + profileId: uuid("profile_id").notNull(), + connectionId: uuid("connection_id").notNull(), + chargeUnits: integer("charge_units").notNull().default(1), + bindingSealed: text("binding_sealed").notNull(), + bindingProof: text("binding_proof").notNull(), + createdAt: ts("created_at").notNull(), +}, table => [ + index("mcp_routing_forward_charges_window").on(table.profileId, table.createdAt), + index("mcp_routing_forward_charges_retention").on(table.createdAt, table.sourceOperationId), + check("mcp_routing_forward_charges_source_step_index_check", sql`${table.sourceStepIndex}=0`), + check("mcp_routing_forward_charges_charge_units_check", sql`${table.chargeUnits}=1`), + check("mcp_routing_forward_charges_binding_sealed_check", sql`octet_length(${table.bindingSealed}) BETWEEN 1 AND 262144`), + check("mcp_routing_forward_charges_binding_proof_check", sql`${table.bindingProof} ~ '^[a-f0-9]{64}$'`), +]) + +// Immutable logical manual inverse identities intentionally outlive family FKs. +export const mcpRoutingManualInverseCharges = pgTable("mcp_routing_manual_inverse_charges", { + recoveryOperationId: uuid("recovery_operation_id").primaryKey(), + recoveryPlanId: uuid("recovery_plan_id").notNull(), + sourceOperationId: uuid("source_operation_id").notNull(), + sourcePlanId: uuid("source_plan_id").notNull(), + sourceStepIndex: integer("source_step_index").notNull(), + profileId: uuid("profile_id").notNull(), + profileVersion: integer("profile_version").notNull(), + connectionId: uuid("connection_id").notNull(), + ownerId: text("owner_id").notNull(), + chargeUnits: integer("charge_units").notNull(), + bindingSealed: text("binding_sealed").notNull(), + bindingProof: text("binding_proof").notNull(), + createdAt: ts("created_at").notNull(), +}, table => [ + unique("mcp_manual_inverse_source_step_unique").on(table.sourceOperationId,table.sourceStepIndex), + index("mcp_routing_manual_inverse_charges_window").on(table.profileId,table.createdAt), + index("mcp_routing_manual_inverse_charges_retention").on(table.createdAt,table.recoveryOperationId), + check("mcp_routing_manual_inverse_charges_source_step_index_check",sql`${table.sourceStepIndex}=0`), + check("mcp_routing_manual_inverse_charges_profile_version_check",sql`${table.profileVersion}>0`), + check("mcp_routing_manual_inverse_charges_owner_id_check",sql`octet_length(${table.ownerId}) BETWEEN 1 AND 1000`), + check("mcp_routing_manual_inverse_charges_charge_units_check",sql`${table.chargeUnits}=1`), + check("mcp_routing_manual_inverse_charges_binding_sealed_check",sql`octet_length(${table.bindingSealed}) BETWEEN 1 AND 262144`), + check("mcp_routing_manual_inverse_charges_binding_proof_check",sql`${table.bindingProof} ~ '^[a-f0-9]{64}$'`), +]) diff --git a/backend/src/db/server-poll-settings-epoch.owned-helper.ts b/backend/src/db/server-poll-settings-epoch.owned-helper.ts new file mode 100644 index 00000000..50fd4388 --- /dev/null +++ b/backend/src/db/server-poll-settings-epoch.owned-helper.ts @@ -0,0 +1,140 @@ +/** Actual 41→current upgrade (including42 epochs) and pg_dump/pg_restore in two databases owned by the + * existing disposable-cluster runner. No runtime activation or external hosts. */ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { readFile, mkdir, unlink } from "node:fs/promises" +import { execFile } from "node:child_process" +import { promisify } from "node:util" +import { basename, dirname, resolve, join } from "node:path" +import { Pool } from "pg" +import { MIGRATION_MANIFEST, analyzeMigration } from "./migration-manifest.js" +import { applySqlMigrations } from "./migrate.js" +import { captureMigrationData, compareMigrationData } from "./migration-data-inventory.js" +import { compareMigrationSchemas } from "./migration-schema-inventory.js" +import { acceptServerPollSettingsEpochSql } from "../services/server-poll-settings-epoch.sql-helper.js" +import { assertServerPollSettingsEpochCatalog, SERVER_POLL_SETTINGS_EPOCH_INVENTORY_SQL, SERVER_POLL_SETTINGS_TABLES } from "../services/server-poll-settings-epoch.js" + +export async function acceptServerPollSettingsEpochMigrationSql(admin: Pool) { + assert.equal(process.env.MMAPP_OWN_SERVER_POLL_SETTINGS_EPOCH, "true") + assert.equal(process.env.MMAPP_OWN_SERVER_POLL_STORAGE_FRESH, "true") + assert.equal(process.env.NODE_ENV, "test") + const url = new URL(process.env.DATABASE_URL ?? "") + assert.equal(url.hostname, "127.0.0.1"); assert.equal(url.pathname, "/postgres") + assert.equal(url.username, "mcp_test"); assert(url.port && url.port !== "5432") + const state = resolve(process.env.MCP_STATE_DIR ?? "") + assert(basename(dirname(state)).startsWith("mm-mcp-test-")) + const names: string[] = [], pools: Pool[] = [], groups: string[] = [] + const archive = join(state, "owned-settings-epoch-" + randomUUID() + ".dump") + const execute = promisify(execFile) + const create = async () => { + const name = "mm_poll_epoch_" + randomUUID().replaceAll("-", "") + assert(/^mm_poll_epoch_[a-f0-9]{32}$/.test(name)) + await admin.query(`CREATE DATABASE "${name}" TEMPLATE template0`); names.push(name) + const target = new URL(url); target.pathname = "/" + name + const pool = new Pool({ connectionString: target.toString(), max: 2 }); pools.push(pool) + return { name, pool } + } + const snapshot = async (pool: Pool) => { + const client = await pool.connect() + try { + await client.query("BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY") + for (const sql of ["SET LOCAL search_path=pg_catalog", "SET LOCAL TimeZone='UTC'", "SET LOCAL DateStyle='ISO, YMD'", + "SET LOCAL standard_conforming_strings=on", "SET LOCAL statement_timeout='10s'", "SET LOCAL row_security=off", + "SET LOCAL extra_float_digits=3", "SET LOCAL bytea_output=hex", "SET LOCAL IntervalStyle=postgres", + "SET LOCAL lc_monetary='C'", "SET LOCAL lc_numeric='C'"]) await client.query(sql) + return await captureMigrationData(client) + } finally { try { await client.query("ROLLBACK") } finally { client.release(true) } } + } + try { + assert.equal(MIGRATION_MANIFEST[41]?.file, "0042_server_poll_settings_epoch.sql") + const source = await create(), restored = await create() + const owner = await source.pool.connect() + try { + await owner.query("CREATE TABLE schema_migrations(id text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now())") + for (const pin of MIGRATION_MANIFEST.slice(0, 41)) { + const bytes = await readFile(new URL("../../drizzle/" + pin.file, import.meta.url)) + analyzeMigration(pin.file, bytes, pin) + if (pin.transaction === "owner-transaction") await owner.query("BEGIN") + await owner.query(bytes.toString("utf8")) + if (!pin.selfMarker) await owner.query("INSERT INTO schema_migrations(id) VALUES($1)", [pin.file.slice(0, -4)]) + if (pin.transaction === "owner-transaction") await owner.query("COMMIT") + } + } finally { owner.release(true) } + assert.equal((await source.pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, 41) + for (const table of SERVER_POLL_SETTINGS_TABLES) await source.pool.query(`INSERT INTO public.${table} + (id,enabled,interval_sec,retention_days,last_collected_at,last_duration_ms,last_error,created_at,updated_at) + VALUES(1,false,47,19,'2026-10-30 01:02:03.123456+00',29,'synthetic before upgrade', + '2026-10-29 01:02:03.123456+00','2026-10-30 01:02:03.654321+00') + ON CONFLICT(id) DO UPDATE SET enabled=excluded.enabled,interval_sec=excluded.interval_sec, + retention_days=excluded.retention_days,last_collected_at=excluded.last_collected_at, + last_duration_ms=excluded.last_duration_ms,last_error=excluded.last_error, + created_at=excluded.created_at,updated_at=excluded.updated_at`) + const rows = async (pool: Pool) => { + const values = [] + for (const table of SERVER_POLL_SETTINGS_TABLES) values.push((await pool.query(`SELECT to_jsonb(t) AS value FROM public.${table} t WHERE id=1`)).rows[0].value) + return values + } + const oldRows = await rows(source.pool) + // A partially matching unmarked proposal must never leave the first ALTER + // committed while a later duplicate column refuses the migration. + await source.pool.query("ALTER TABLE public.uptime_settings ADD COLUMN poll_configuration_epoch uuid") + const collisionRows = await rows(source.pool) + await assert.rejects(applySqlMigrations(source.pool)) + assert.deepEqual(await rows(source.pool), collisionRows) + assert.equal((await source.pool.query("SELECT count(*)::int AS n FROM schema_migrations WHERE id='0042_server_poll_settings_epoch'")).rows[0].n, 0) + assert.equal((await source.pool.query("SELECT to_regprocedure('public.server_poll_settings_epoch_guard()') IS NULL AS absent")).rows[0].absent, true) + assert.deepEqual((await source.pool.query(`SELECT table_name FROM information_schema.columns + WHERE table_schema='public' AND column_name='poll_configuration_epoch' ORDER BY table_name`)).rows, + [{ table_name: "uptime_settings" }]) + await source.pool.query("ALTER TABLE public.uptime_settings DROP COLUMN poll_configuration_epoch") + assert.deepEqual(await rows(source.pool), oldRows) + groups.push("partial-unmarked-schema-collision-refuses-without-earlier-ddl-or-marker-commit") + await applySqlMigrations(source.pool); await applySqlMigrations(source.pool) + assert.equal((await source.pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length) + const newRows = await rows(source.pool) + for (const [index, value] of newRows.entries()) { + assert.match(value.poll_configuration_epoch, /^[a-f0-9-]{36}$/) + const { poll_configuration_epoch: epoch, ...original } = value + assert.equal(typeof epoch, "string"); assert.deepEqual(original, oldRows[index]) + } + groups.push("actual-nonempty-41-to-current-upgrade-preserves-every-old-field-and-microsecond") + const afterUpgrade = await rows(source.pool) + await applySqlMigrations(source.pool) + assert.deepEqual(await rows(source.pool), afterUpgrade) + groups.push("normal-startup-repetition-preserves-issued-epochs-and-atomic-marker") + await acceptServerPollSettingsEpochSql(source.pool, true) + const before = await snapshot(source.pool) + const args = ["--host", url.hostname, "--port", url.port, "--username", "mcp_test", "--no-password"] + const env = { ...process.env, PGPASSWORD: decodeURIComponent(url.password) } + await mkdir(state, { recursive: true }) + await execute("pg_dump", [...args, "--format=custom", "--file", archive, "--dbname=" + source.name], + { env, timeout: 60000, maxBuffer: 1048576, windowsHide: true }) + await execute("pg_restore", [...args, "--single-transaction", "--exit-on-error", "--dbname=" + restored.name, archive], + { env, timeout: 60000, maxBuffer: 1048576, windowsHide: true }) + const after = await snapshot(restored.pool) + assert.equal(compareMigrationSchemas(before.schema, after.schema).equal, true) + assert.equal(compareMigrationData(before, after).equal, true) + assert.deepEqual(await rows(restored.pool), afterUpgrade) + assertServerPollSettingsEpochCatalog((await restored.pool.query(SERVER_POLL_SETTINGS_EPOCH_INVENTORY_SQL, [SERVER_POLL_SETTINGS_TABLES])).rows) + groups.push("actual-pg-dump-restore-complete-schema-data-and-exact-epoch-preserved") + const epoch = (await rows(restored.pool))[1].poll_configuration_epoch + await restored.pool.query("UPDATE public.uptime_settings SET last_error='restored collector',updated_at=clock_timestamp() WHERE id=1") + assert.equal((await rows(restored.pool))[1].poll_configuration_epoch, epoch) + await restored.pool.query("UPDATE public.uptime_settings SET retention_days=retention_days+1 WHERE id=1") + const changed = (await rows(restored.pool))[1].poll_configuration_epoch + assert.notEqual(changed, epoch) + await restored.pool.query("UPDATE public.uptime_settings SET retention_days=retention_days-1 WHERE id=1") + const back = (await rows(restored.pool))[1].poll_configuration_epoch + assert.notEqual(back, epoch); assert.notEqual(back, changed) + groups.push("restored-guard-enforces-runtime-stability-and-configuration-ABA") + console.log("Owned settings epoch migration/restore:", JSON.stringify({ groups, groupCount: groups.length, + numberedMigrations: MIGRATION_MANIFEST.length, priorFiles: 41, actualPgDumpRestore: true, completeLogicalSchemaAndDataEqual: true, + exactEpochsPreservedAcrossRestore: true, runtimeApprovalWired: false, realChrUsed: false })) + } finally { + for (const pool of pools) await pool.end() + for (const name of names.reverse()) { + assert(/^mm_poll_epoch_[a-f0-9]{32}$/.test(name)); await admin.query(`DROP DATABASE "${name}"`) + } + try { await unlink(archive) } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error } + } +} diff --git a/backend/src/db/server-poll-terminal-archives.sql.test.ts b/backend/src/db/server-poll-terminal-archives.sql.test.ts new file mode 100644 index 00000000..1212fc06 --- /dev/null +++ b/backend/src/db/server-poll-terminal-archives.sql.test.ts @@ -0,0 +1,109 @@ +/** Schema upgrade/negative guards only. Positive archive acceptance must use the + * genuine public poll runtime and private historical codec, never these rows. */ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { readFile } from "node:fs/promises" +import { basename, dirname, isAbsolute, resolve } from "node:path" + +assert.equal(process.env.MMAPP_OWN_POLL_ARCHIVE_SCHEMA, "true") +assert.equal(process.env.NODE_ENV, "test") +const endpoint = new URL(process.env.DATABASE_URL ?? "") +assert.equal(endpoint.hostname, "127.0.0.1"); assert.equal(endpoint.username, "mcp_test"); assert(endpoint.password) +assert.equal(endpoint.pathname, "/postgres"); assert.equal(endpoint.search, ""); assert.equal(endpoint.hash, "") +assert(Number.isInteger(Number(endpoint.port)) && Number(endpoint.port) > 0 && Number(endpoint.port) <= 65535) +assert(process.env.MCP_STATE_DIR && isAbsolute(process.env.MCP_STATE_DIR)) +assert.equal(basename(resolve(process.env.MCP_STATE_DIR)), "state") +assert(basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-")) +const { Pool } = await import("pg") +const { pool, closePool } = await import("./index.js") +const { MIGRATION_MANIFEST, analyzeMigration } = await import("./migration-manifest.js") +const archivePin = MIGRATION_MANIFEST.find(pin => pin.file === "0041_server_poll_terminal_archives.sql")! +const migration = await readFile(new URL("../../drizzle/0041_server_poll_terminal_archives.sql", import.meta.url)) +analyzeMigration(archivePin.file, migration, archivePin) +const databases: { name: string; pool: InstanceType }[] = [], accepted: string[] = [] +const prefix40 = async () => { + const name = "mm_poll_archive_schema_" + randomUUID().replaceAll("-", "") + assert(/^mm_poll_archive_schema_[a-f0-9]{32}$/.test(name)) + await pool.query(`CREATE DATABASE "${name}" TEMPLATE template0`) + const target = new URL(endpoint); target.pathname = "/" + name + const connection = new Pool({ connectionString: target.toString(), max: 4 }) + databases.push({ name, pool: connection }) + const pins = MIGRATION_MANIFEST.filter(pin => pin.file < "0041_") + assert.equal(pins.length, 40); assert.equal(pins.at(-1)?.file, "0040_mcp_routing_reservations.sql") + await connection.query("CREATE TABLE schema_migrations(id text PRIMARY KEY,applied_at timestamptz NOT NULL DEFAULT now())") + for (const pin of pins) { + const bytes = await readFile(new URL("../../drizzle/" + pin.file, import.meta.url)); analyzeMigration(pin.file, bytes, pin) + await connection.query(bytes.toString("utf8")) + await connection.query("INSERT INTO schema_migrations(id) VALUES($1) ON CONFLICT DO NOTHING", [pin.file.slice(0, -4)]) + } + return connection +} +const seedFullRest = async (connection: InstanceType) => { + const receipt = randomUUID(), execution = randomUUID(), token = randomUUID() + await connection.query(`INSERT INTO server_poll_storage_admissions(receipt_id,identity_kind,execution_id,owner_id,binding_proof,context_proof, + authority_issued_at,authority_expires_at,max_stages,claim_token,retention_days,safety_ms,gc_not_before) + VALUES($1,'rest',$2,'owned-archive-schema',$3,$4,clock_timestamp()-interval '1 second',clock_timestamp()+interval '3 minutes',8,$5,1,60000,clock_timestamp()+interval '2 days')`, + [receipt, execution, "a".repeat(64), "b".repeat(64), token]) + await connection.query("UPDATE server_poll_storage_admissions SET claim_token=NULL,claim_attempted_at=clock_timestamp() WHERE receipt_id=$1", [receipt]) + await connection.query("INSERT INTO server_poll_executions(receipt_id,identity_kind,execution_id,binding_proof,context_ciphertext,context_proof) VALUES($1,'rest',$2,$3,'SYNTHETIC_SCHEMA_FIXTURE_ONLY',$4)", [receipt, execution, "a".repeat(64), "b".repeat(64)]) + return { receipt, execution } +} +try { + const connection = await prefix40(), { receipt, execution } = await seedFullRest(connection) + const originalAdmission = (await connection.query("SELECT to_jsonb(a) AS value FROM server_poll_storage_admissions a WHERE receipt_id=$1", [receipt])).rows[0].value + const originalJournal = (await connection.query("SELECT to_jsonb(e) AS value FROM server_poll_executions e WHERE receipt_id=$1", [receipt])).rows[0].value + const originalUsage = (await connection.query("SELECT * FROM server_poll_storage_usage ORDER BY scope")).rows + console.log("Owned prefix40 poll constraint inventory:", JSON.stringify((await connection.query("SELECT conname,pg_get_constraintdef(oid) AS definition FROM pg_constraint WHERE conrelid='public.server_poll_storage_usage'::regclass AND contype='c' ORDER BY conname")).rows)) + await connection.query(migration.toString("utf8")) + assert.equal(Number((await connection.query("SELECT count(*) AS n FROM schema_migrations")).rows[0].n), 41) + assert.deepEqual((await connection.query("SELECT to_jsonb(a)-ARRAY['storage_state','compact_at','archive_bytes','original_activity_at','compaction_claim','compaction_claimed_at','compaction_claim_xid'] AS value FROM server_poll_storage_admissions a WHERE receipt_id=$1", [receipt])).rows[0].value, originalAdmission) + assert.deepEqual((await connection.query("SELECT to_jsonb(e) AS value FROM server_poll_executions e WHERE receipt_id=$1", [receipt])).rows[0].value, originalJournal) + assert.deepEqual((await connection.query("SELECT * FROM server_poll_storage_usage ORDER BY scope")).rows, originalUsage.map(row => ({ ...row, compacted_admissions: 0, archive_bytes: 0 }))) + assert.equal((await connection.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid, true) + assert.equal((await connection.query("SELECT server_poll_storage_source_payload($1) AS value", [receipt])).rows[0].value, null) + assert.deepEqual((await connection.query("SELECT compaction_claim,compaction_claimed_at,compaction_claim_xid FROM server_poll_storage_admissions WHERE receipt_id=$1", [receipt])).rows[0], { compaction_claim: null, compaction_claimed_at: null, compaction_claim_xid: null }) + accepted.push("nonempty40 full REST journal, original admission/proofs/counters preserved; archive source remains unavailable for REST") + await assert.rejects(connection.query("UPDATE server_poll_storage_admissions SET storage_state='compacted',compact_at=clock_timestamp(),archive_bytes=1,original_activity_at=created_at WHERE receipt_id=$1", [receipt])) + await assert.rejects(connection.query("UPDATE server_poll_storage_admissions SET compaction_claim=$2,compaction_claimed_at=clock_timestamp() WHERE receipt_id=$1", [receipt, randomUUID()])) + await assert.rejects(connection.query("UPDATE server_poll_storage_admissions SET compaction_claimed_at=clock_timestamp() WHERE receipt_id=$1", [receipt])) + await assert.rejects(connection.query("UPDATE server_poll_storage_admissions SET compaction_claim_xid=pg_current_xact_id() WHERE receipt_id=$1", [receipt])) + await assert.rejects(connection.query("UPDATE server_poll_storage_admissions SET compaction_claim=NULL,compaction_claimed_at=NULL WHERE receipt_id=$1", [receipt])) + await assert.rejects(connection.query(`INSERT INTO server_poll_storage_admissions(receipt_id,identity_kind,execution_id,owner_id,binding_proof,context_proof, + authority_issued_at,authority_expires_at,max_stages,claim_token,retention_days,safety_ms,gc_not_before,compaction_claim,compaction_claimed_at) + VALUES($1,'rest',$2,'owned-archive-schema',$3,$3,clock_timestamp()-interval '1 second',clock_timestamp()+interval '3 minutes',8,$4,1,60000,clock_timestamp()+interval '2 days',$5,clock_timestamp())`, + [randomUUID(), randomUUID(), "a".repeat(64), randomUUID(), randomUUID()])) + await assert.rejects(connection.query(`INSERT INTO server_poll_archives(receipt_id,version,execution_id,connection_id,owner_id,binding_proof,context_proof,source_digest,source_sha256,terminal_digest, + original_activity_at,original_expires_at,decoded_bytes,compressed_bytes,ciphertext,ciphertext_bytes,archive_proof) + VALUES($1,1,$2,$3,'owned-archive-schema',$4,$4,$4,$4,$4,clock_timestamp(),clock_timestamp(),1,1,$5,$6,$4)`, + [receipt, execution, randomUUID(), "a".repeat(64), Buffer.alloc(29).toString("base64"), Buffer.alloc(29).toString("base64").length])) + const leased = await connection.connect() + try { + await leased.query("BEGIN"); await leased.query("SET LOCAL mm.poll_storage_compact='on'") + await assert.rejects(leased.query("DELETE FROM server_poll_executions WHERE receipt_id=$1", [receipt])) + } finally { await leased.query("ROLLBACK"); leased.release() } + assert.deepEqual((await connection.query("SELECT to_jsonb(e) AS value FROM server_poll_executions e WHERE receipt_id=$1", [receipt])).rows[0].value, originalJournal) + assert.equal(Number((await connection.query("SELECT count(*) AS n FROM server_poll_archives")).rows[0].n), 0) + accepted.push("no archive, prefilled/reset/altered claim, direct state edit or session flag can compact an ineligible source or delete immutable journal") + + const inconsistent = await prefix40(); await seedFullRest(inconsistent) + // Deliberately valid OLD scalar quota shape but inconsistent with retained + // rows: additive enrollment must refuse rather than resetting the ledger. + await inconsistent.query("UPDATE server_poll_storage_usage SET admissions=0,reserved_rows=0,reserved_bytes=0") + const before = (await inconsistent.query("SELECT to_jsonb(a) AS value FROM server_poll_storage_admissions a")).rows + const target = await inconsistent.connect() + try { await assert.rejects(target.query(migration.toString("utf8"))) } + finally { await target.query("ROLLBACK"); target.release() } + assert.deepEqual((await inconsistent.query("SELECT to_jsonb(a) AS value FROM server_poll_storage_admissions a")).rows, before) + assert.equal((await inconsistent.query("SELECT to_regclass('public.server_poll_archives') AS name")).rows[0].name, null) + assert.equal(Number((await inconsistent.query("SELECT count(*) AS n FROM schema_migrations")).rows[0].n), 40) + accepted.push("inconsistent nonempty40 ledger fails before archive DDL and marker with original source untouched") + console.log(JSON.stringify({ accepted: accepted.length, cases: accepted, fullHistoryPreserved: true, + positiveArchiveAcceptance: false, nativeOrPublicActivation: false })) +} finally { + for (const item of databases) { + await item.pool.end() + assert(/^mm_poll_archive_schema_[a-f0-9]{32}$/.test(item.name)) + await pool.query(`DROP DATABASE "${item.name}"`) + } + await closePool() +} diff --git a/backend/src/db/sqlite-import-control-writes.sql.test.ts b/backend/src/db/sqlite-import-control-writes.sql.test.ts new file mode 100644 index 00000000..f47d27a3 --- /dev/null +++ b/backend/src/db/sqlite-import-control-writes.sql.test.ts @@ -0,0 +1,167 @@ +/** Permanent owned PostgreSQL regression candidate. No source-capture TMP + * module, app config, application pool or dotenv import is needed. */ +import assert from "node:assert/strict" +import { randomUUID } from "node:crypto" +import { basename, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { Pool, type PoolClient } from "pg" +assert.equal(process.env.MMAPP_OWN_IMPORT_CONTROL, "true") +assert(process.env.MMAPP_OWN_IMPORT_DIRECTORY && basename(process.env.MMAPP_OWN_IMPORT_DIRECTORY).startsWith("mm-import-control-")) +const adminUrl = new URL(process.env.MMAPP_OWN_IMPORT_ADMIN_URL ?? "") +assert.equal(adminUrl.hostname, "127.0.0.1"); assert.equal(adminUrl.pathname, "/postgres") +assert.equal(process.env.DATABASE_URL, undefined) +// The candidate runner can point at the real repository while this test stays +// in TMP. Once integrated, relative production source is the default. +const sourceRoot = process.env.MMAPP_OWN_IMPORT_SOURCE_ROOT +const source = (relative: string, ownRelative: string) => sourceRoot ? pathToFileURL(resolve(sourceRoot, "backend/src", relative)).href : new URL(ownRelative, import.meta.url).href +const { withPartitionWrite, PARTITION_WRITER_ADVISORY_KEY, isPartitionWriteOutcomeUnknown } = await import(source("services/partition-write-transaction.ts", "../services/partition-write-transaction.js")) +const { writeSqliteImportMarker, resetSqliteImportSequence } = await import(source("db/sqlite-import-control-writes.ts", "./sqlite-import-control-writes.js")) +const admin = new Pool({ connectionString: adminUrl.toString(), max: 2 }) +const name = "import_control_" + randomUUID().replaceAll("-", "") +assert(/^import_control_[a-f0-9]{32}$/.test(name)) +await admin.query('CREATE DATABASE "' + name + '" TEMPLATE template0') +const target = new URL(adminUrl); target.pathname = "/" + name +const pool = new Pool({ connectionString: target.toString(), max: 8, connectionTimeoutMillis: 3000 }) +const ownerRole = `sequence_owner_${randomUUID().replaceAll("-", "")}` +let ownerPool: Pool | undefined +const held: PoolClient[] = [], releases: (() => void)[] = [], cases: string[] = [] +async function until(check: () => Promise) { + const end = performance.now() + 4000 + while (!await check()) { if (performance.now() >= end) throw Error("Owned lock state not observed"); await new Promise(r => setTimeout(r, 10)) } +} +function fault(mode: "lost-commit" | "aborted-commit" | "rollback-after-setval", database = pool) { + const commands: string[] = [], destroyed: boolean[] = [] + let commitTag: string | undefined + const adapted = { async connect() { const client = await database.connect(); return { + release(destroy = false) { destroyed.push(destroy); client.release(destroy) }, async query(text: string, values?: unknown[]) { + commands.push(text) + if (text === "COMMIT" && mode === "aborted-commit") await assert.rejects(client.query("SELECT 1/0"), (e: unknown) => !!e && typeof e === "object" && "code" in e && e.code === "22012") + const result = await client.query(text, values) + if (text === "COMMIT") commitTag = result.command + if (text === "COMMIT" && mode === "lost-commit") throw Error("Owned fixture loses actual COMMIT ACK") + if (text.startsWith("SELECT pg_catalog.setval") && mode === "rollback-after-setval") { + assert.equal((await client.query("ROLLBACK")).command, "ROLLBACK") + throw Error("Owned fixture reports failure after actual setval and acknowledged ROLLBACK") + } + return result + }, + } } } as unknown as Pool + return { pool: adapted, commands, destroyed, tag: () => commitTag } +} +async function lastSequence() { return (await pool.query("SELECT last_value::text AS value FROM public.import_rows_id_seq")).rows[0].value } +async function identityState() { return (await pool.query("SELECT last_value::text AS value,is_called FROM public.identity_rows_id_seq")).rows[0] } +async function markerPath() { return (await pool.query("SELECT sqlite_path FROM public.data_migration WHERE id=1")).rows[0].sqlite_path } +try { + await pool.query("CREATE TABLE public.data_migration(id integer PRIMARY KEY,sqlite_imported_at timestamptz,sqlite_path text,sqlite_sha256 text,report_json jsonb); INSERT INTO public.data_migration(id) VALUES(1); CREATE TABLE public.import_rows(id bigserial PRIMARY KEY); INSERT INTO public.import_rows(id) VALUES(42); CREATE TABLE public.no_sequence(id integer PRIMARY KEY)") + const exclusive = await pool.connect(); held.push(exclusive) + await exclusive.query("SELECT pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + const marker = writeSqliteImportMarker(pool, "owned.sqlite", "a".repeat(64), JSON.stringify({ import_rows: { copied: 1 } })) + const sequence = resetSqliteImportSequence(pool, "import_rows") + await until(async () => (await pool.query("SELECT count(*)::int AS n FROM pg_locks WHERE locktype='advisory' AND objid=$1::oid AND NOT granted", [PARTITION_WRITER_ADVISORY_KEY])).rows[0].n === 2) + assert.equal(await markerPath(), null); assert.equal(await lastSequence(), "1") + assert.equal((await exclusive.query("SELECT pg_advisory_unlock($1::bigint) AS unlocked", [PARTITION_WRITER_ADVISORY_KEY])).rows[0].unlocked, true) + await marker; assert.equal(await sequence, "sequence-set"); assert.equal(await lastSequence(), "42") + const saved = (await pool.query("SELECT sqlite_path,sqlite_sha256,report_json FROM public.data_migration WHERE id=1")).rows[0] + assert.equal(saved.sqlite_path, "owned.sqlite"); assert.equal(saved.sqlite_sha256, "a".repeat(64)); assert.deepEqual(saved.report_json, { import_rows: { copied: 1 } }) + cases.push("marker and setval use same real shared admission before SQL, wait for exclusive drain and preserve values") + assert.equal(await resetSqliteImportSequence(pool, "no_sequence"), "sequence-absent") + cases.push("absent serial identity is explicit compatibility no-op") + + let entered!: () => void, release!: () => void + const ready = new Promise(r => { entered = r }), waiting = new Promise(r => { release = r }); releases.push(release) + const writer = withPartitionWrite(async ({ client }: { client: PoolClient }) => { await client.query("UPDATE public.data_migration SET sqlite_path='already-admitted' WHERE id=1"); entered(); await waiting }, { connect: () => pool.connect() }) + await ready + const drain = exclusive.query("SELECT pg_advisory_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + await until(async () => (await pool.query("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND objid=$1::oid AND NOT granted) AS waiting", [PARTITION_WRITER_ADVISORY_KEY])).rows[0].waiting === true) + release(); await writer; await drain; assert.equal(await markerPath(), "already-admitted") + await exclusive.query("SELECT pg_advisory_unlock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY]) + cases.push("exclusive drain waits for already-admitted writer COMMIT") + + const lostMarker = fault("lost-commit") + await assert.rejects(writeSqliteImportMarker(lostMarker.pool, "lost-ack.sqlite", "b".repeat(64), "{}"), isPartitionWriteOutcomeUnknown) + assert.equal(lostMarker.tag(), "COMMIT"); assert.equal(await markerPath(), "lost-ack.sqlite") + assert.equal(lostMarker.commands.filter(q => q.startsWith("UPDATE data_migration")).length, 1); assert.equal(lostMarker.commands.filter(q => q === "COMMIT").length, 1) + assert(!lostMarker.commands.includes("ROLLBACK")); assert.deepEqual(lostMarker.destroyed, [true]) + cases.push("lost actual marker COMMIT ACK is UNKNOWN, one update and no second transition/replay") + + await pool.query("INSERT INTO public.import_rows(id) VALUES(77)") + const lostSequence = fault("lost-commit") + await assert.rejects(resetSqliteImportSequence(lostSequence.pool, "import_rows"), isPartitionWriteOutcomeUnknown) + assert.equal(await lastSequence(), "77"); assert.equal(lostSequence.commands.filter(q => q.startsWith("SELECT pg_catalog.setval")).length, 1) + cases.push("lost actual sequence COMMIT ACK propagates UNKNOWN with setval already visible and no replay") + + await pool.query("INSERT INTO public.import_rows(id) VALUES(88)") + const rolledSequence = fault("rollback-after-setval") + await assert.rejects(resetSqliteImportSequence(rolledSequence.pool, "import_rows"), isPartitionWriteOutcomeUnknown) + assert.equal(await lastSequence(), "88"); assert.equal(rolledSequence.commands.filter(q => q.startsWith("SELECT pg_catalog.setval")).length, 1) + cases.push("acknowledged actual ROLLBACK does not undo setval; post-sequence failure remains UNKNOWN") + + const abortedDml = fault("aborted-commit") + await assert.rejects(withPartitionWrite(async ({ client }: { client: PoolClient }) => { await client.query("INSERT INTO public.no_sequence VALUES(999)") }, { connect: () => abortedDml.pool.connect() }), isPartitionWriteOutcomeUnknown) + assert.equal(abortedDml.tag(), "ROLLBACK"); assert.equal((await pool.query("SELECT count(*)::int AS n FROM public.no_sequence")).rows[0].n, 0) + assert.equal(abortedDml.commands.filter(q => q === "COMMIT").length, 1); assert(!abortedDml.commands.includes("ROLLBACK")); assert.deepEqual(abortedDml.destroyed, [true]) + cases.push("real aborted DML COMMIT resolves ROLLBACK and is refused without success/retry/second transition") + const abortedMarker = fault("aborted-commit") + await assert.rejects(writeSqliteImportMarker(abortedMarker.pool, "must-not-publish.sqlite", "c".repeat(64), "{}"), isPartitionWriteOutcomeUnknown) + assert.equal(abortedMarker.tag(), "ROLLBACK"); assert.equal(await markerPath(), "lost-ack.sqlite") + cases.push("real aborted marker COMMIT=>ROLLBACK cannot publish success") + await pool.query("INSERT INTO public.import_rows(id) VALUES(99)") + const abortedSequence = fault("aborted-commit") + await assert.rejects(resetSqliteImportSequence(abortedSequence.pool, "import_rows"), isPartitionWriteOutcomeUnknown) + assert.equal(abortedSequence.tag(), "ROLLBACK"); assert.equal(await lastSequence(), "99") + cases.push("real aborted setval COMMIT=>ROLLBACK refuses success while sequence effect honestly remains") + await assert.rejects(resetSqliteImportSequence(pool, "missing_import_table")) + cases.push("non-compatibility SQL errors propagate; blanket catch and replay absent") + + const ownerPassword = randomUUID() + await pool.query(`CREATE ROLE ${ownerRole} LOGIN PASSWORD '${ownerPassword}'; GRANT USAGE ON SCHEMA public TO ${ownerRole}; + CREATE TABLE public.identity_rows(id bigint GENERATED BY DEFAULT AS IDENTITY, observed int DEFAULT 1) PARTITION BY RANGE (observed); + CREATE TABLE public.identity_rows_p1 PARTITION OF public.identity_rows FOR VALUES FROM (1) TO (2); + INSERT INTO public.identity_rows(id) VALUES (500); + ALTER TABLE public.identity_rows OWNER TO ${ownerRole}; + ALTER TABLE public.identity_rows_p1 OWNER TO ${ownerRole}; + REVOKE USAGE,UPDATE ON SEQUENCE public.identity_rows_id_seq FROM ${ownerRole}; + CREATE FUNCTION public.owned_check_sequence_ddl() RETURNS event_trigger LANGUAGE plpgsql SET search_path=pg_catalog AS $$ + BEGIN + IF SESSION_USER='${ownerRole}' AND NOT EXISTS (SELECT 1 FROM pg_locks + WHERE locktype='advisory' AND pid=pg_backend_pid() AND classid=0 AND objid=${PARTITION_WRITER_ADVISORY_KEY} + AND objsubid=1 AND granted AND mode='ShareLock') THEN + RAISE EXCEPTION 'operator DDL requires admission' USING ERRCODE='55000'; + END IF; + END $$; + CREATE EVENT TRIGGER owned_sequence_ddl_guard ON ddl_command_start EXECUTE FUNCTION public.owned_check_sequence_ddl()`) + const ownerUrl = new URL(target) + ownerUrl.username = ownerRole; ownerUrl.password = ownerPassword + ownerPool = new Pool({ connectionString: ownerUrl.toString(), max: 2 }) + await assert.rejects(ownerPool.query("SELECT pg_catalog.setval('public.identity_rows_id_seq', 900)"), { code: "42501" }) + await assert.rejects(ownerPool.query("ALTER TABLE public.identity_rows ALTER COLUMN id RESTART WITH 900"), { code: "55000" }) + assert.equal(await resetSqliteImportSequence(ownerPool, "identity_rows"), "sequence-set") + assert.deepEqual(await identityState(), { value: "501", is_called: false }) + assert.equal((await ownerPool.query("INSERT INTO public.identity_rows DEFAULT VALUES RETURNING id::text")).rows[0].id, "501") + cases.push("admitted native IDENTITY reset works for a partitioned table owner without setval rights") + + await pool.query("INSERT INTO public.identity_rows(id) VALUES (600)") + const abortedIdentity = fault("aborted-commit", ownerPool) + await assert.rejects(resetSqliteImportSequence(abortedIdentity.pool, "identity_rows"), isPartitionWriteOutcomeUnknown) + assert.equal(abortedIdentity.tag(), "ROLLBACK") + assert.deepEqual(await identityState(), { value: "501", is_called: true }) + assert.equal(abortedIdentity.commands.filter(q => q.includes("RESTART WITH 601")).length, 1) + cases.push("actual aborted COMMIT rolls back the native identity reset") + + const lostIdentity = fault("lost-commit", ownerPool) + await assert.rejects(resetSqliteImportSequence(lostIdentity.pool, "identity_rows"), isPartitionWriteOutcomeUnknown) + assert.equal(lostIdentity.tag(), "COMMIT") + assert.deepEqual(await identityState(), { value: "601", is_called: false }) + assert.equal(lostIdentity.commands.filter(q => q.includes("RESTART WITH 601")).length, 1) + assert(!lostIdentity.commands.includes("ROLLBACK")); assert.deepEqual(lostIdentity.destroyed, [true]) + cases.push("lost identity reset COMMIT ACK stays UNKNOWN; restart happens once") + + console.log(JSON.stringify({ cases: cases.length, names: cases, actualSql: true, genuineCommitRollbackCases: 4, + actualCurrentProductionPartitionWriter: true, actualCurrentProductionImporterHelpers: true, sourceCaptureTmpDependency: false, + allWriterDrainAuthenticated: false, readyAuthorized: false, productionDatabaseTouched: false, hostUsersGroupsServicesChanged: false }, null, 2)) +} finally { + for (const release of releases) release() + for (const c of held) c.release(true) + await ownerPool?.end(); await pool.end(); await admin.query('DROP DATABASE "' + name + '" WITH (FORCE)') + await admin.query(`DROP ROLE IF EXISTS ${ownerRole}`); await admin.end() +} diff --git a/backend/src/db/sqlite-import-control-writes.ts b/backend/src/db/sqlite-import-control-writes.ts new file mode 100644 index 00000000..29e1a844 --- /dev/null +++ b/backend/src/db/sqlite-import-control-writes.ts @@ -0,0 +1,61 @@ +import type { Pool } from "pg" +import { withPartitionWrite, PartitionWriteError, isPartitionWriteOutcomeUnknown } from "../services/partition-write-transaction.js" +/** Uses the same + * admission already used by import batches, on the marker's actual backend. */ +export async function writeSqliteImportMarker(pool: Pool, sqlitePath: string, sqliteSha256: string, reportJson: string): Promise { + await withPartitionWrite(async ({ client }) => { + await client.query(`UPDATE data_migration + SET sqlite_imported_at = now(), sqlite_path = $1, sqlite_sha256 = $2, report_json = $3::jsonb + WHERE id = 1`, [sqlitePath, sqliteSha256, reportJson]) + }, { connect: () => pool.connect() }) +} + +const identitySettings = `attribute.attidentity IN ('a', 'd') AND NOT attribute.attisdropped + AND settings.seqincrement = 1 AND settings.seqcache = 1 AND NOT settings.seqcycle` + +export async function resetSqliteImportSequence(pool: Pool, table: string): Promise<"sequence-absent" | "sequence-set"> { + if (typeof table !== "string" || !/^[a-z_][a-z0-9_]{0,62}$/.test(table)) throw Error("Fixed importer relation identifier required") + const relation = `"public"."${table}"` + let sequenceMutationSent = false + try { return await withPartitionWrite(async ({ client }) => { + const { rows } = await client.query<{ sequence_oid: string | null; restartable: boolean }>(`SELECT + sequence_oid::text, (pg_catalog.pg_has_role(tbl.relowner, 'USAGE') AND ${identitySettings}) IS TRUE AS restartable + FROM pg_catalog.pg_class tbl + CROSS JOIN LATERAL (SELECT pg_catalog.pg_get_serial_sequence($1, 'id')::pg_catalog.regclass::oid AS sequence_oid) seq + JOIN pg_catalog.pg_attribute attribute ON attribute.attrelid = tbl.oid AND attribute.attname = 'id' + LEFT JOIN pg_catalog.pg_sequence settings ON settings.seqrelid = seq.sequence_oid + WHERE tbl.oid = $1::pg_catalog.regclass`, [relation]) + if (rows.length !== 1) throw Error("Importer identity metadata unavailable") + const { sequence_oid: sequenceOid, restartable } = rows[0] + if (sequenceOid === null) return "sequence-absent" as const + if (!/^[1-9][0-9]{0,9}$/.test(sequenceOid)) throw Error("Importer sequence identity unavailable") + if (restartable) { + // Freeze the ID high-water mark, including direct writes to descendants. + await client.query(`LOCK TABLE ${relation} IN ACCESS EXCLUSIVE MODE`) + const { rows } = await client.query<{ restart: string }>(`SELECT (used.last_id::numeric + 1)::text AS restart + FROM pg_catalog.pg_attribute attribute + JOIN pg_catalog.pg_sequence settings + ON settings.seqrelid = pg_catalog.pg_get_serial_sequence($1, 'id')::pg_catalog.regclass + CROSS JOIN (SELECT GREATEST(COALESCE(MAX(id), 1), 1) AS last_id FROM ${relation}) used + WHERE attribute.attrelid = $1::pg_catalog.regclass AND attribute.attname = 'id' + AND settings.seqrelid = $2::pg_catalog.oid AND ${identitySettings} + AND used.last_id >= settings.seqmin AND used.last_id < settings.seqmax`, [relation, sequenceOid]) + if (rows.length !== 1 || !/^[1-9][0-9]{0,18}$/.test(rows[0].restart)) { + throw Error("Unsupported or exhausted importer identity sequence") + } + await client.query(`ALTER TABLE ${relation} ALTER COLUMN id RESTART WITH ${rows[0].restart}`) + return "sequence-set" as const + } + // Legacy serial and grant-only callers can use setval without owning the table. + sequenceMutationSent = true + await client.query(`SELECT pg_catalog.setval($1::pg_catalog.oid::pg_catalog.regclass, + GREATEST(COALESCE((SELECT MAX(id) FROM ${relation}),1),1),true)`, [sequenceOid]) + return "sequence-set" as const + }, { connect: () => pool.connect() }) } + catch (error) { + // The surrounding SQL transaction may be known rolled back, while setval + // was already visible. Never feed that case into known-failure/retry lanes. + if (sequenceMutationSent && !isPartitionWriteOutcomeUnknown(error)) throw new PartitionWriteError(true) + throw error + } +} diff --git a/backend/src/db/sqlite-import-partition-admission.test.ts b/backend/src/db/sqlite-import-partition-admission.test.ts new file mode 100644 index 00000000..7370335e --- /dev/null +++ b/backend/src/db/sqlite-import-partition-admission.test.ts @@ -0,0 +1,52 @@ +import assert from "node:assert/strict" +import { readFile } from "node:fs/promises" +import { runInNewContext } from "node:vm" +import ts from "typescript" + +// Execute the actual private copyTable implementation with inert SQLite/SQL +// providers. No native SQLite file, configured pool or external DB is opened. +const source = await readFile(new URL("./sqlite-import.ts", import.meta.url), "utf8") +const compiled = ts.transpileModule(source + "\nexport { copyTable };\n", { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText +class UnknownCommit extends Error { readonly outcomeUnknown = true } +async function fixture(count: number, unknownCommit = false) { + let consumed = 0, connects = 0 + const events: string[] = [], inserted: unknown[][] = [] + const rows = Array.from({ length: count }, (_, index) => ({ server_id: 1, sampled_at: "2026-10-25T00:00:00Z", interface_name: `synthetic-${index}` })) + const sqlite = { prepare(text: string) { return { + get() { if (text.includes("sqlite_master")) return { name: "traffic_samples" }; if (text.includes("MIN(")) return { a: rows[0]?.sampled_at, b: rows.at(-1)?.sampled_at }; return { c: count } }, + *iterate() { for (const row of rows) { consumed++; yield row } }, + } } } + const client = { async query(text: string, values: unknown[]) { assert(text.startsWith("INSERT INTO traffic_samples")); events.push("INSERT"); inserted.push(values); return { rows: [] } } } + const pool = { async connect() { connects++; return client }, query: async () => { assert.fail("Copy SQL escaped owned client") } } + const modules: Record = { + "node:crypto": {}, "node:fs": {}, "better-sqlite3": {}, "../config.js": { env: {} }, "./traffic-flags.js": { encodeTrafficFlags: () => 0 }, + "./sqlite-import-control-writes.js": { writeSqliteImportMarker: () => assert.fail("Marker not reached in copyTable-only fixture"), resetSqliteImportSequence: () => assert.fail("Non-identity fixture must not reset a sequence") }, + "./partitions.js": { specForParent: () => ({ kind: "week" }), ensurePartitionsBetween: async () => { events.push("admitted DDL helper") } }, + "../services/partition-write-transaction.js": { + withPartitionWrite: async (work: (owner: { client: typeof client }) => Promise, options: { connect: () => Promise }) => { + assert.equal(await options.connect(), client) + events.push("BEGIN", "SHARED 8723110"); await work({ client }); events.push("COMMIT") + if (unknownCommit) throw new UnknownCommit("Synthetic lost COMMIT ACK") + }, + isPartitionWriteOutcomeUnknown: (error: unknown) => error instanceof UnknownCommit, + }, + } + const exports = {} as { copyTable: (...args: unknown[]) => Promise<{ copied: number; skipped: number }> } + runInNewContext(compiled, { exports, require: (name: string) => { assert(name in modules, name); return modules[name] }, console: { log: () => {} }, Date, Error, Promise, Set, Map, String, Number, Object, JSON, Buffer }, { filename: "sqlite-import.ts" }) + const spec = { table: "traffic_samples", columns: [["server_id", "int"], ["sampled_at", "ts"], ["interface_name", "text"]], timeCol: "sampled_at" } + return { run: (strict: boolean) => exports.copyTable(sqlite, pool, spec, { strict, fullHistory: true, rejects: [] }), events, inserted, consumed: () => consumed, connects: () => connects } +} +const confirmed = await fixture(1001) +const report = await confirmed.run(false) +assert.equal(report.copied, 1001); assert.equal(report.skipped, 0); assert.equal(confirmed.connects(), 2) +assert.equal(confirmed.events[0], "admitted DDL helper") +assert.equal(confirmed.events.filter(event => event === "SHARED 8723110").length, 2) +assert.equal(confirmed.inserted[0].length, 3000); assert.equal(confirmed.inserted[1].length, 3) +for (const strict of [true, false]) { + const f = await fixture(1001, true) + await assert.rejects(f.run(strict), UnknownCommit) + assert.equal(f.consumed(), 1000, "Unknown commit escapes the source-row catch even in non-strict import") + assert.equal(f.connects(), 1); assert.equal(f.inserted.length, 1) + assert.equal(f.events.filter(event => event === "COMMIT").length, 1) +} +console.log("SQLite import partition admission: 3 actual-source inert chunk/count/no-replay scenarios passed") diff --git a/backend/src/db/sqlite-import.ts b/backend/src/db/sqlite-import.ts index c0944572..c81d30cd 100644 --- a/backend/src/db/sqlite-import.ts +++ b/backend/src/db/sqlite-import.ts @@ -5,6 +5,8 @@ import type { Pool } from "pg" import { env } from "../config.js" import { ensurePartitionsBetween, specForParent } from "./partitions.js" import { encodeTrafficFlags } from "./traffic-flags.js" +import { withPartitionWrite, isPartitionWriteOutcomeUnknown } from "../services/partition-write-transaction.js" +import { writeSqliteImportMarker, resetSqliteImportSequence } from "./sqlite-import-control-writes.js" export interface ImportReport { sqlitePath: string @@ -342,17 +344,6 @@ function sqliteTableExists(sqlite: Database.Database, name: string): boolean { return Boolean(row) } -async function setval(pool: Pool, table: string): Promise { - await pool.query( - `SELECT setval( - pg_get_serial_sequence($1, 'id'), - GREATEST(COALESCE((SELECT MAX(id) FROM ${table}), 1), 1), - true - )`, - [table], - ) -} - function placeholderFor(kind: ColKind, index: number): string { if (kind === "json" || kind === "json-null") return `$${index}::jsonb` if (kind === "inet") return `$${index}::inet` @@ -413,18 +404,18 @@ async function copyTable( `(${spec.columns.map(([, kind], j) => placeholderFor(kind, i * n + j + 1)).join(", ")})`, ).join(", ") const insertSql = `INSERT INTO ${spec.table} (${cols.join(", ")}) VALUES ${valuesSql} ${conflictSql}` - const client = await pool.connect() try { - await client.query("BEGIN") - await client.query(insertSql, batch.flat()) + await withPartitionWrite(async ({ client }) => { + await client.query(insertSql, batch.flat()) + }, { connect: () => pool.connect() }) + // Count only confirmed commits. Unknown COMMIT must escape the ETL row + // rejection/fallback lane and must never replay the source batch. copied += batch.length - await client.query("COMMIT") } catch (err) { - await client.query("ROLLBACK") + if (isPartitionWriteOutcomeUnknown(err)) throw err const detail = err instanceof Error ? err.message : String(err) throw new Error(`${spec.table}: ${detail}`) } finally { - client.release() batch.length = 0 } } @@ -440,6 +431,7 @@ async function copyTable( batch.push(values) if (batch.length >= INSERT_CHUNK) await flush() } catch (err) { + if (isPartitionWriteOutcomeUnknown(err)) throw err skipped += 1 const msg = `${spec.table}: ${err instanceof Error ? err.message : String(err)}` opts.rejects.push(msg) @@ -447,13 +439,7 @@ async function copyTable( } } await flush() - if (spec.identity) { - try { - await setval(pool, spec.table) - } catch { - /* partitioned identity sequence name may differ */ - } - } + if (spec.identity) await resetSqliteImportSequence(pool, spec.table) return { sqlite: total, copied, skipped } } @@ -518,20 +504,18 @@ export async function importSqliteToPostgres( report.durationMs = Date.now() - started return report } + try { for (const spec of TABLES) { console.log(`SQLite → PostgreSQL: таблица ${spec.table}`) report.tables[spec.table] = await copyTable(sqlite, pool, spec, { strict, fullHistory, rejects }) } - sqlite.close() - await pool.query( - `UPDATE data_migration - SET sqlite_imported_at = now(), sqlite_path = $1, sqlite_sha256 = $2, report_json = $3::jsonb - WHERE id = 1`, - [sqlitePath, sha, JSON.stringify(report.tables)], - ) + await writeSqliteImportMarker(pool, sqlitePath, sha, JSON.stringify(report.tables)) report.durationMs = Date.now() - started if (strict && rejects.length > 0) { throw new Error(`SQLite import strict: ${rejects.length} rejects\n${rejects.slice(0, 20).join("\n")}`) } return report + } finally { + sqlite.close() + } } diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 0e6e4aed..83a6bf95 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -10,6 +10,18 @@ "build": "tsc -p tsconfig.json" }, "exports": { + "./bridge-vlan": { + "types": "./dist/bridge-vlan.d.ts", + "default": "./dist/bridge-vlan.js" + }, + "./mcp-routing": { + "types": "./dist/mcp-routing.d.ts", + "default": "./dist/mcp-routing.js" + }, + "./mcp": { + "types": "./dist/mcp.d.ts", + "default": "./dist/mcp.js" + }, ".": { "types": "./dist/index.d.ts", "default": "./dist/index.js" diff --git a/packages/contracts/src/bridge-vlan.ts b/packages/contracts/src/bridge-vlan.ts new file mode 100644 index 00000000..6f44995c --- /dev/null +++ b/packages/contracts/src/bridge-vlan.ts @@ -0,0 +1,23 @@ +import { z } from "zod" + +export const bridgeVlanRowIdSchema = z.string().regex(/^\*[0-9a-f]+$/i) +export const bridgeVlanInterfaceReferenceSchema = z.object({ resource: z.string().min(1).max(64), id: bridgeVlanRowIdSchema }).strict() +export const bridgeVlanIdSchema = z.number().int().min(1).max(4094) +export const bridgeVlanFrameTypesSchema = z.enum(["admit-all", "admit-only-vlan-tagged", "admit-only-untagged-and-priority-tagged"]) +/** Complete desired static state. Omissions never authorize deletion. */ +export const bridgeVlanDesiredSchema = z.object({ + filtering: z.boolean(), pvid: bridgeVlanIdSchema, + frameTypes: bridgeVlanFrameTypesSchema.optional(), ingressFiltering: z.boolean().optional(), + ports: z.array(z.object({ id: bridgeVlanRowIdSchema.optional(), interface: bridgeVlanInterfaceReferenceSchema, pvid: bridgeVlanIdSchema, ingressFiltering: z.boolean(), frameTypes: bridgeVlanFrameTypesSchema, disabled: z.boolean(), comment: z.string().max(1000).optional() }).strict()).max(1024), + vlans: z.array(z.object({ id: bridgeVlanRowIdSchema.optional(), vlanIds: z.string().min(1).max(16384), tagged: z.array(bridgeVlanInterfaceReferenceSchema).max(1024), untagged: z.array(bridgeVlanInterfaceReferenceSchema).max(1024), disabled: z.boolean(), comment: z.string().max(1000).optional() }).strict()).max(4094), + removePortIds: z.array(bridgeVlanRowIdSchema).max(1024), removeVlanIds: z.array(bridgeVlanRowIdSchema).max(4094), +}).strict() +export const bridgeVlanConfigurationRequestSchema = z.object({ serverId: z.number().int().positive(), bridgeId: bridgeVlanRowIdSchema, desired: bridgeVlanDesiredSchema }).strict() +export type BridgeVlanDesired = z.infer +export type BridgeVlanConfigurationRequest = z.infer +const rows = z.array(z.record(z.string(), z.unknown())) +export const bridgeVlanStateSchema = z.object({ + bridges: rows, ports: rows, vlans: rows, vlanInterfaces: rows, ipv4Addresses: rows, ipv6Addresses: rows, + interfaces: z.array(z.object({ resource: z.string(), id: z.string(), name: z.string(), disabled: z.unknown().optional() }).strict()), +}).strict() +export type BridgeVlanState = z.infer diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 432c70b4..7e23821c 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1,4 +1,5 @@ export * from "./servers.js" +export * from "./mcp.js" export * from "./logs.js" export * from "./alerts.js" export * from "./events.js" diff --git a/packages/contracts/src/ipsec.ts b/packages/contracts/src/ipsec.ts index e34c6b0a..851e9179 100644 --- a/packages/contracts/src/ipsec.ts +++ b/packages/contracts/src/ipsec.ts @@ -191,17 +191,23 @@ export const ipsecUserPatchSchema = z.object({ disabled: z.boolean().optional(), }) -export const ipsecCertExportRequestSchema = z.object({ +export const ipsecExistingCertificateExportIntentSchema = z.object({ + privateKeyExport: z.literal(true), + caPrivateKeyExport: z.boolean().default(false), +}).strict() +export const ipsecExistingCertificateExportRequestKeySchema = z.string().uuid() +export const ipsecExistingCertificateExportBodySchema = ipsecExistingCertificateExportIntentSchema.extend({ + passphrase: z.string().min(IPSEC_MIN_PASSPHRASE).max(16384), +}).strict() +export const ipsecCertExportRequestSchema = ipsecExistingCertificateExportBodySchema.extend({ serverId: z.union([z.string(), z.number()]), - clientId: z.string().min(1), - passphrase: z.string().min(IPSEC_MIN_PASSPHRASE), -}) + clientId: z.string().min(1).max(256), +}).strict() /** Экспорт .p12 существующего клиентского сертификата по имени (client1/anakondra и т.п.). */ -export const ipsecCertExportByNameRequestSchema = z.object({ - name: z.string().min(1), - passphrase: z.string().min(IPSEC_MIN_PASSPHRASE), -}) +export const ipsecCertExportByNameRequestSchema = ipsecExistingCertificateExportBodySchema.extend({ + name: z.string().min(1).max(256), +}).strict() /** Бандл для авторизации клиента: .p12 (+ strongSwan .sswan + инструкция). */ export const ipsecCertBundleSchema = z.object({ diff --git a/packages/contracts/src/logs.ts b/packages/contracts/src/logs.ts index 41390711..ea2f06cb 100644 --- a/packages/contracts/src/logs.ts +++ b/packages/contracts/src/logs.ts @@ -4,6 +4,13 @@ export const logLevels = ["critical", "error", "warning", "info", "debug", "unkn export const logLevelSchema = z.enum(logLevels) export type LogLevel = z.infer const id = z.string().regex(/^\d+$/) +const maxLogSequenceId = "9223372036854775807" +export function isLogSequenceId(value: unknown): value is string { + if (typeof value !== "string" || value.length === 0 || /[^0-9]/.test(value)) return false + const digits = value.replace(/^0+/, "") + return digits.length < maxLogSequenceId.length || (digits.length === maxLogSequenceId.length && digits <= maxLogSequenceId) +} +export const logSequenceInputSchema = z.string().regex(/^\d{1,20}$/).refine(isLogSequenceId, "ID журнала выходит за допустимый диапазон") const instant = z.string().datetime({ offset: true }) export const logEntrySchema = z.object({ id, serverId: z.number().int(), serverName: z.string(), host: z.string(), @@ -20,7 +27,7 @@ export const logFiltersSchema = z.object({ }).refine(v => !v.from || !v.to || Date.parse(v.from) <= Date.parse(v.to), "Некорректный период") export type LogFilters = z.infer export const logPageQuerySchema = logFiltersSchema.safeExtend({ - cursor: z.string().max(200).optional(), watermark: id.optional(), + cursor: z.string().max(200).optional(), watermark: id.refine(isLogSequenceId, "Отметка журнала выходит за допустимый диапазон").optional(), limit: z.coerce.number().int().min(1).max(200).default(200), }) export const logStateSchema = z.object({ serverId: z.number().int(), serverName: z.string(), host: z.string(), diff --git a/packages/contracts/src/mcp-routing.test.ts b/packages/contracts/src/mcp-routing.test.ts new file mode 100644 index 00000000..d4a67645 --- /dev/null +++ b/packages/contracts/src/mcp-routing.test.ts @@ -0,0 +1,16 @@ +import assert from "node:assert/strict" +import { routeAiControllerBlockReason, routeAiProfileSchema } from "./mcp-routing.js" +const raw = { name: "context", serverId: 1, community: "65000:10", family: "ipv4", table: "policy-only", candidates: [{ id: "a", wanInterface: "wan", downstreamInterface: "down", jumpHostId: 2, exitNodeId: 3, gateway: "192.0.2.1", gatewayRef: { resource: "ipv4_route", id: "*a" } }] } +assert.equal(routeAiProfileSchema.parse(raw).probeVrf, "main") +assert.deepEqual(routeAiProfileSchema.parse({ ...raw, probeVrf: "tenant-vrf", bgpContext: { connectionId: "*c", instance: "tenant-bgp", inputChain: "tenant-in" } }).bgpContext, { connectionId: "*c", instance: "tenant-bgp", inputChain: "tenant-in", vrf: "main" }) +assert.equal(routeAiProfileSchema.parse({ ...raw, probeVrf: "probe-vrf", bgpContext: { connectionId: "*c", instance: "tenant-bgp", inputChain: "tenant-in", vrf: "bgp-vrf" } }).bgpContext?.vrf, "bgp-vrf") +for (const bgpContext of [{ connectionId: "name", instance: "i", inputChain: "in" }, { connectionId: "*c", instance: "i", inputChain: "in", direction: "output" }, { connectionId: "*c", instance: "i\n", inputChain: "in\u0000" }]) assert.equal(routeAiProfileSchema.safeParse({ ...raw, bgpContext }).success, false) +console.log("Route AI profile: installation table/probe VRF/input binding remain separate; strict native identity and control-character rejection") + +const scoped = routeAiProfileSchema.parse({ ...raw, bgpContext: { connectionId: "*c", instance: "tenant-bgp", inputChain: "tenant-in" } }) +assert.match(routeAiControllerBlockReason(routeAiProfileSchema.parse(raw))!, /BGP-привязку/) +assert.match(routeAiControllerBlockReason(scoped)!, /исходным адресом/) +const withProbe = { ...scoped, candidates: [{ ...scoped.candidates[0], controlProbe: { sourceAddress: "192.0.2.2", targetInterface: "down", expectedHops: [{ serverId: 2, interface: "wan" }] } }] } +assert.equal(routeAiControllerBlockReason(withProbe), null) +assert.match(routeAiControllerBlockReason({ ...withProbe, autoRecovery: true })!, /ROUTE_RECOVERY_EXECUTOR_UNAVAILABLE.*непрерывность событий RouterOS.*REST \/log/) +assert.equal(routeAiControllerBlockReason(routeAiProfileSchema.parse({ ...raw, table: "main" })), null) diff --git a/packages/contracts/src/mcp-routing.ts b/packages/contracts/src/mcp-routing.ts new file mode 100644 index 00000000..dc5b1cfa --- /dev/null +++ b/packages/contracts/src/mcp-routing.ts @@ -0,0 +1,63 @@ +import { z } from "zod" +const sid = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const name = z.string().trim().min(1).max(100).refine(value => !/[\u0000-\u001f\u007f-\u009f\u2028\u2029]/.test(value)) +export const routeAiWeightsSchema = z.object({ ping: z.number().min(0).max(100).default(60), speed: z.number().min(0).max(100).default(30), loss: z.number().min(0).max(100).default(10) }).strict().refine(weights => Math.abs(weights.ping + weights.speed + weights.loss - 100) < 0.0001, "Сумма весов должна быть 100") +export const routeAiCandidateSchema = z.object({ + id: name, wanInterface: name, jumpHostId: sid, exitNodeId: sid, + gateway: name, gatewayRef: z.object({ resource: z.enum(["gre", "ipv4_route", "ipv6_route"]), id: name }).strict(), + downstreamInterface: name, + wanRemoteInterface: name.optional(), downstreamRemoteInterface: name.optional(), + controlProbe: z.object({ targetInterface: name, targetAddress: name.optional(), expectedHops: z.array(z.object({ serverId: sid, interface: name, address: name.optional() }).strict()).min(1).max(8), sourceAddress: name.optional() }).strict().optional(), +}).strict() +export const routeAiProfileSchema = z.object({ + name, serverId: sid, community: z.string().regex(/^\d{1,10}:\d{1,10}$/), family: z.enum(["ipv4", "ipv6"]), table: name.default("main"), + // A route installation table is not a diagnostic VRF. Never derive one from the other. + probeVrf: name.default("main"), + // BGP transport VRF is independent of both installation table and probe VRF. + bgpContext: z.object({ connectionId: z.string().regex(/^\*[A-Za-z0-9]+$/).max(200), instance: name, inputChain: name, vrf: name.default("main") }).strict().optional(), + candidates: z.array(routeAiCandidateSchema).min(1).max(100).refine(candidates => new Set(candidates.map(candidate => candidate.id)).size === candidates.length, "Повторяющиеся пути"), + pinnedCandidateId: name.nullable().default(null), excludedCandidateIds: z.array(name).max(100).default([]), + weights: routeAiWeightsSchema.default(() => ({ ping: 60, speed: 30, loss: 10 })), + maxPingMs: z.number().positive().nullable().default(null), maxLossPct: z.number().min(0).max(100).nullable().default(null), minBandwidthMbps: z.number().nonnegative().nullable().default(null), + pingFreshnessSec: z.number().int().min(10).max(3600).default(300), bandwidthFreshnessSec: z.number().int().min(60).max(86400).default(3600), + switchThreshold: z.number().min(0).max(100).default(15), hysteresis: z.number().min(0).max(100).default(10), + confirmationSamples: z.number().int().min(1).max(10).default(3), cooldownSec: z.number().int().min(60).max(86400).default(600), maxSwitchesPerHour: z.number().int().min(1).max(10).default(3), + autoRecovery: z.boolean().default(false), +}).strict().superRefine((profile, context) => { + const ids = new Set(profile.candidates.map(candidate => candidate.id)) + if (profile.autoRecovery && profile.candidates.some(candidate => !candidate.controlProbe)) context.addIssue({ code: "custom", message: "Автовосстановление требует контрольных проверок всех путей" }) + for (const candidate of profile.candidates) if (candidate.controlProbe?.expectedHops.some(hop => ![profile.serverId, candidate.jumpHostId, candidate.exitNodeId].includes(hop.serverId))) context.addIssue({ code: "custom", message: "Контрольный hop должен принадлежать устройству выбранного пути" }) + if (profile.pinnedCandidateId && !ids.has(profile.pinnedCandidateId)) context.addIssue({ code: "custom", message: "Закреплён неизвестный путь" }) + if (profile.excludedCandidateIds.some(id => !ids.has(id))) context.addIssue({ code: "custom", message: "Исключён неизвестный путь" }) +}) +export const routeMeasurementSchema = z.object({ + srcServerId: sid, dstServerId: sid, srcInterface: name, dstInterface: name, family: z.enum(["ipv4", "ipv6"]), vrf: name.default("main"), + dstAddress: z.union([z.ipv4(), z.ipv6()]).optional(), + bandwidth: z.boolean().default(false), durationSec: z.number().int().min(3).max(30).default(5), maxMbps: z.number().positive().max(10000).optional(), +}).strict().refine(input => !input.bandwidth || input.maxMbps !== undefined, "Bandwidth-тест требует ограничения скорости") +export type RouteAiProfile = z.infer +type RouteAiContext = Pick + +export function usesScopedRouteAiContext({ table, probeVrf, bgpContext }: RouteAiContext): boolean { + return table.trim() !== "main" || probeVrf.trim() !== "main" || + !!bgpContext && (bgpContext.inputChain.trim() !== "bgp-in" || bgpContext.vrf.trim() !== "main") +} + +export function routeAiControllerBlockReason(profile: RouteAiContext & Pick & { + candidates: readonly { controlProbe?: { sourceAddress?: string } }[] +}): string | null { + if (profile.autoRecovery) return "ROUTE_RECOVERY_EXECUTOR_UNAVAILABLE: для автоматического восстановления нужны проверка состояния после переключения и подтверждённая непрерывность событий RouterOS. REST /log показывает только текущее содержимое буфера." + if (!usesScopedRouteAiContext(profile)) return null + if (!profile.bgpContext) return "Укажите явную BGP-привязку для контроллера отдельной таблицы или VRF" + if (profile.candidates.some(candidate => !candidate.controlProbe?.sourceAddress?.trim())) return "Укажите контрольную пробу с исходным адресом для каждого пути" + return null +} +export interface RouteObservation { + id: string; srcServerId: number; dstServerId: number; srcInterface: string; dstInterface: string; + family: "ipv4" | "ipv6"; table: string; pingMs: number | null; lossPct: number | null; dlMbps: number | null; ulMbps: number | null; + pingAt: string | null; bandwidthAt: string | null; contextVerified: boolean; + /** False for legacy/tampered data or measurements of a previous endpoint epoch. */ + sourceVerified?: boolean; + /** Publication ordering is distinct from independent ping/bandwidth times. */ + recordedAt?: string; +} diff --git a/packages/contracts/src/mcp.ts b/packages/contracts/src/mcp.ts new file mode 100644 index 00000000..32fdb615 --- /dev/null +++ b/packages/contracts/src/mcp.ts @@ -0,0 +1,174 @@ +import { z } from "zod" + +const identifier = z.string().trim().min(1).max(200) +const serverId = z.number().int().positive().max(Number.MAX_SAFE_INTEGER) +const instant = z.string().datetime({ offset: true }) +const uniqueStrings = z.array(identifier).max(1000).refine(values => new Set(values).size === values.length, "Повторяющиеся значения") +const serverIds = z.array(serverId).max(1000).refine(values => new Set(values).size === values.length, "Повторяющиеся серверы") + +export const mcpModeSchema = z.enum(["observer", "approval", "autonomous"]) +export const mcpLimitsSchema = z.object({ + callsPerMinute: z.number().int().min(1).max(10000).default(60), + concurrentReads: z.number().int().min(1).max(100).default(4), + mutationsPerHour: z.number().int().min(1).max(10000).default(10), + serversPerPlan: z.number().int().min(1).max(1000).default(5), + objectsPerPlan: z.number().int().min(1).max(10000).default(100), +}).strict() +export const mcpScopeSchema = z.object({ + serverIds, + operationIds: uniqueStrings, + allowGlobal: z.boolean().default(false), +}).strict() +export const mcpTemporaryGrantSchema = z.object({ + id: identifier, + startsAt: instant, + expiresAt: instant, + scope: mcpScopeSchema, +}).strict().refine(grant => { + const duration = Date.parse(grant.expiresAt) - Date.parse(grant.startsAt) + return duration > 0 && duration <= 24 * 60 * 60 * 1000 +}, "Временный доступ должен действовать не более 24 часов") +export const mcpConnectionPolicySchema = z.object({ + id: identifier, + version: z.number().int().positive(), + status: z.enum(["active", "paused", "revoked"]), + expiresAt: instant, + mode: mcpModeSchema, + permissions: uniqueStrings, + scope: mcpScopeSchema, + dangerousPolicy: z.enum(["approval", "explicit-allow"]).default("approval"), + autonomousDangerousOperationIds: uniqueStrings.default([]), + limits: mcpLimitsSchema.default(() => mcpLimitsSchema.parse({})), +}).strict() + +export const mcpTargetSchema = z.discriminatedUnion("scope", [ + z.object({ scope: z.literal("servers"), serverIds: serverIds.refine(ids => ids.length > 0, "Выберите серверы") }).strict(), + z.object({ scope: z.literal("global") }).strict(), + z.object({ scope: z.literal("connection") }).strict(), +]) +export const mcpObjectRefSchema = z.object({ + kind: identifier, + id: identifier, + serverId: serverId.optional(), +}).strict() +export const mcpProtectedObjectSchema = mcpObjectRefSchema.extend({ + protection: z.enum(["deny", "human-approval-required"]), +}).strict() +export type McpObjectRef = z.infer +export const mcpOperationDescriptorSchema = z.object({ + id: identifier, + toolName: z.string().regex(/^mm_[a-z0-9_]+$/), + version: z.number().int().min(1).max(2147483647), + title: identifier, + section: identifier, + effect: z.enum(["read", "write", "diagnostic"]), + risk: z.enum(["ordinary", "dangerous"]), + scope: z.enum(["servers", "global", "connection"]), + requiredPermissions: uniqueStrings, + implementation: z.enum(["planned", "available"]), + annotations: z.object({ + readOnlyHint: z.boolean(), + destructiveHint: z.boolean(), + idempotentHint: z.boolean(), + openWorldHint: z.boolean(), + }).strict(), +}).strict() + +// This payload is hypothetical evidence for the human-only simulator, never authorization. +// Future executors must construct runtime evidence from trusted repositories and locks. +export const mcpPolicyPreviewRequestSchema = z.object({ + operationId: identifier, + policy: mcpConnectionPolicySchema, + ownerPermissions: uniqueStrings, + target: mcpTargetSchema, + temporaryGrant: mcpTemporaryGrantSchema.optional(), + objectCount: z.number().int().nonnegative().max(1000000).default(0), + objects: z.array(mcpObjectRefSchema).max(1000).default([]), + protections: z.array(mcpProtectedObjectSchema).max(1000).default([]), + usage: z.object({ + callsInMinute: z.number().int().nonnegative().default(0), + concurrentReads: z.number().int().nonnegative().default(0), + mutationsInHour: z.number().int().nonnegative().default(0), + }).strict().default(() => ({ callsInMinute: 0, concurrentReads: 0, mutationsInHour: 0 })), + auditAvailable: z.boolean().default(true), +}).strict() + +export const mcpPolicyReasonCodeSchema = z.enum([ + "UNKNOWN_OPERATION", "CONNECTION_INACTIVE", "CONNECTION_EXPIRED", "OPERATION_DENIED", + "OWNER_PERMISSION_MISSING", "CONNECTION_PERMISSION_MISSING", "TARGET_SCOPE_MISMATCH", + "SERVER_DENIED", "GLOBAL_DENIED", "GRANT_INACTIVE", "GRANT_SCOPE_DENIED", + "INVALID_OBJECT_SCOPE", "OBSERVER_WRITE_DENIED", "RATE_LIMITED", "BUDGET_EXCEEDED", + "PLAN_LIMIT_EXCEEDED", "PROTECTED_OBJECT", "AUDIT_UNAVAILABLE", "APPROVAL_REQUIRED", +]) +export const mcpPolicyDecisionSchema = z.object({ + decision: z.enum(["denied", "approval-required", "allowed"]), + reasons: z.array(z.object({ code: mcpPolicyReasonCodeSchema, message: z.string() }).strict()), + limits: mcpLimitsSchema, +}).strict() +export const mcpPolicyPreviewResponseSchema = z.object({ + simulation: z.literal(true), + executionAvailable: z.literal(false), + evaluatedAt: instant, + policyVersion: z.number().int().positive(), + result: mcpPolicyDecisionSchema, +}).strict() +export const mcpCatalogResponseSchema = z.object({ + stage: z.enum(["foundation", "runtime"]), + executionAvailable: z.boolean(), + operations: z.array(mcpOperationDescriptorSchema), +}).strict() + +export type McpLimits = z.infer +export type McpConnectionPolicy = z.infer +export type McpOperationDescriptor = z.infer +export type McpPolicyPreviewRequest = z.infer +export type McpPolicyDecision = z.infer +export type McpPolicyReasonCode = z.infer + +export const mcpConnectionCreateSchema = z.object({ + name: z.string().trim().min(1).max(100), + expiresAt: instant, + mode: mcpModeSchema.default("observer"), + permissions: uniqueStrings, + scope: mcpScopeSchema, + dangerousPolicy: z.enum(["approval", "explicit-allow"]).default("approval"), + autonomousDangerousOperationIds: uniqueStrings.default([]), + limits: mcpLimitsSchema.default(() => mcpLimitsSchema.parse({})), +}).strict() +export const mcpConnectionSchema = z.object({ + id: z.string().uuid(), name: z.string(), ownerId: z.string(), + policy: mcpConnectionPolicySchema, createdAt: instant, updatedAt: instant, + lastUsedAt: instant.nullable(), generationValid: z.boolean(), + ownerValidUntil: instant.optional(), +}).strict() +export const mcpConnectionsSchema = z.object({ connections: z.array(mcpConnectionSchema) }).strict() +export const mcpSettingsSchema = z.object({ + enabled: z.boolean(), requestsRetentionDays: z.number().int().min(1).max(365), + auditRetentionDays: z.number().int().min(30).max(3650), +}).strict() +export const mcpSettingsPatchSchema = mcpSettingsSchema.partial().strict().refine(value => Object.keys(value).length > 0) +export const mcpPageQuerySchema = z.object({ + limit: z.coerce.number().int().min(1).max(100).default(50), + operationId: z.string().uuid().optional(), + cursor: z.union([z.string().uuid(), z.string().regex(/^[1-9]\d{0,18}$/)]).optional(), + connectionId: z.string().uuid().optional(), + status: z.string().max(50).optional(), + tool: z.string().max(200).optional(), + userId: z.string().trim().min(1).max(200).optional(), + serverId: z.coerce.number().int().positive().max(Number.MAX_SAFE_INTEGER).optional(), from: instant.optional(), to: instant.optional(), +}).strict() +export type McpConnection = z.infer +export type McpConnectionCreate = z.infer + +const pollOwnerCapacitySchema=z.object({used:z.number().int().min(0).max(8),limit:z.literal(8),remaining:z.number().int().min(0).max(8)}).strict().refine(value=>value.used+value.remaining<=value.limit) +const pollGlobalCapacitySchema=z.object({used:z.number().int().min(0).max(128),limit:z.literal(128),remaining:z.number().int().min(0).max(128)}).strict().refine(value=>value.used+value.remaining<=value.limit) +/** Heavy reservations. Retained archive bytes/rows can further reduce the + * remaining capacity; this observation never grants an execution admission. */ +export const mcpPollStorageCapacitySchema=z.discriminatedUnion("status",[ + z.object({status:z.literal("unavailable")}).strict(), + z.object({status:z.literal("observed"),owner:pollOwnerCapacitySchema, + global:z.object({exhausted:z.boolean(),usage:pollGlobalCapacitySchema.nullable()}).strict() + .refine(value=>value.usage===null||value.exhausted===(value.usage.remaining===0)), + retentionDays:z.number().int().min(1).max(365)}).strict(), +]) +export type McpPollStorageCapacity=z.infer diff --git a/packages/contracts/src/servers.test.ts b/packages/contracts/src/servers.test.ts new file mode 100644 index 00000000..bc8f0f3f --- /dev/null +++ b/packages/contracts/src/servers.test.ts @@ -0,0 +1,8 @@ +import assert from "node:assert/strict" +import { serverCreateSchema, serverUpdateSchema } from "./servers.js" +assert.deepEqual(serverUpdateSchema.parse({ name: "Renamed" }), { name: "Renamed" }) +assert.deepEqual(serverUpdateSchema.parse({ enabled: false }), { enabled: false }) +assert.deepEqual(serverUpdateSchema.parse({}), {}) +assert.equal(serverCreateSchema.parse({ host: "192.0.2.1" }).port, 443) +assert.equal(serverCreateSchema.parse({ host: "192.0.2.1" }).username, "admin") +console.log("Server PATCH preserves omitted credentials, endpoint, TLS and metadata; create defaults remain valid") diff --git a/packages/contracts/src/servers.ts b/packages/contracts/src/servers.ts index d1f2a7f5..9a104acd 100644 --- a/packages/contracts/src/servers.ts +++ b/packages/contracts/src/servers.ts @@ -31,8 +31,15 @@ export const serverCreateSchema = z.object({ wanUplinks: z.array(wanUplinkSchema).default([]), }) -export const serverUpdateSchema = serverCreateSchema.partial().omit({ host: true }).extend({ - host: z.string().min(1).optional(), +// Updates must preserve omitted connection parameters and credentials. Creation +// defaults inside optional fields would otherwise reset them under Zod 4. +export const serverUpdateSchema = z.object({ + host: z.string().min(1).optional(), port: z.number().int().positive().optional(), + username: z.string().optional(), password: z.string().optional(), + useSsl: z.boolean().optional(), verifySsl: z.boolean().optional(), + name: z.string().optional(), type: serverTypeSchema.optional(), site: z.string().optional(), + country: z.string().optional(), asn: z.string().optional(), comment: z.string().optional(), + enabled: z.boolean().optional(), lanSubnet: z.string().optional(), wanUplinks: z.array(wanUplinkSchema).optional(), }) export const serverIdParamSchema = z.object({ diff --git a/packages/contracts/src/traffic-flow.ts b/packages/contracts/src/traffic-flow.ts index 78be2160..55cc8458 100644 --- a/packages/contracts/src/traffic-flow.ts +++ b/packages/contracts/src/traffic-flow.ts @@ -26,7 +26,39 @@ export const trafficFlowHostStatusSchema = z.object({ }) export type TrafficFlowHostStatus = z.infer +const flowLossU64Maximum = "18446744073709551615" +const flowLossCountSchema = z.string().refine(value => value.length <= 20 && /^(?:0|[1-9][0-9]*)$/.test(value) && BigInt(value) <= 18446744073709551615n, "Invalid canonical U64 counter") +const flowLossTimestampSchema = z.string().refine(value => value.length === 24 && Number.isFinite(Date.parse(value)) && new Date(value).toISOString() === value, "Invalid canonical loss timestamp") +export const flowIngestQualitySchema = z.object({ + version: z.literal(1), collectorEpoch: z.string().uuid(), scope: z.literal("collector-global"), exportCoverage: z.literal("unverified"), + localAdmission: z.enum(["no-local-loss-observed", "partial", "loss-quantity-unknown"]), + records: flowLossCountSchema.nullable(), bytes: flowLossCountSchema.nullable(), packets: flowLossCountSchema.nullable(), + knownLowerBounds: z.object({ records: flowLossCountSchema, bytes: flowLossCountSchema, packets: flowLossCountSchema }), + firstAt: flowLossTimestampSchema.nullable(), lastAt: flowLossTimestampSchema.nullable(), intervalUnknown: z.boolean(), + reasons: z.object({ "aggregate-capacity": z.boolean(), "ring-capacity": z.boolean(), "key-width": z.boolean(), "unsafe-numeric": z.boolean() }), + counterOverflow: z.boolean(), rawRowsPolicy: z.literal("topN-and-pending-cap-not-complete"), lifetimeScope: z.literal("current-collector-isolate"), +}).superRefine((value, context) => { + const fail = () => context.addIssue({ code: "custom", message: "Inconsistent collector loss accounting" }) + const keys = ["records", "bytes", "packets"] as const + if (keys.some(key => value[key] !== null && value[key] !== value.knownLowerBounds[key])) fail() + if (value.records === null && (!value.counterOverflow || value.knownLowerBounds.records !== flowLossU64Maximum)) fail() + if (value.counterOverflow && !keys.some(key => value[key] === null && value.knownLowerBounds[key] === flowLossU64Maximum)) fail() + const lost = value.knownLowerBounds.records !== "0" + if (Object.values(value.reasons).some(Boolean) !== lost) fail() + if (!lost) { + if (value.localAdmission !== "no-local-loss-observed" || value.counterOverflow || value.intervalUnknown + || keys.some(key => value[key] !== "0" || value.knownLowerBounds[key] !== "0") || value.firstAt !== null || value.lastAt !== null) fail() + return + } + const unknownQuantity = value.counterOverflow || keys.some(key => value[key] === null) + if (value.localAdmission !== (unknownQuantity ? "loss-quantity-unknown" : "partial")) fail() + if (value.intervalUnknown) { + if (value.firstAt !== null || value.lastAt !== null) fail() + } else if (value.firstAt === null || value.lastAt === null || value.firstAt > value.lastAt) fail() +}) + export const trafficFlowSettingsDtoSchema = z.object({ + dataQuality:flowIngestQualitySchema.nullable().optional(), enabled: z.boolean(), collectorIp: z.string().min(1), flowListenPort: z.number().int().positive(), @@ -132,6 +164,7 @@ export const flowTalkerDtoSchema = z.object({ }) export const flowStatsDtoSchema = z.object({ + dataQuality:flowIngestQualitySchema.nullable().optional(), exportersOnline: z.number().int().nonnegative(), bytesPerMin: z.number().nonnegative(), uniqueSrc: z.number().int().nonnegative(), diff --git a/packages/contracts/src/uptime-speed.ts b/packages/contracts/src/uptime-speed.ts index b21f38a9..22aa5f50 100644 --- a/packages/contracts/src/uptime-speed.ts +++ b/packages/contracts/src/uptime-speed.ts @@ -54,6 +54,7 @@ export const speedTestCommandSchema = z.object({ protocol: z.enum(["tcp", "udp"]).default("tcp"), direction: z.enum(["transmit", "receive", "both"]).default("both"), durationSec: speedPositiveInteger.pipe(z.number().min(3).max(2147483647)).default(10), + maxMbps: z.number().int().min(1).max(1000).optional(), }).refine(row => row.srcServerId !== row.dstServerId, "Источник и назначение должны отличаться") export type SpeedTestCommand = z.infer diff --git a/packages/contracts/src/users.test.ts b/packages/contracts/src/users.test.ts new file mode 100644 index 00000000..b8cdcfa6 --- /dev/null +++ b/packages/contracts/src/users.test.ts @@ -0,0 +1,9 @@ +import assert from "node:assert/strict" +import { appUserCreateSchema, appUserUpdateSchema, interfaceTypeSchema } from "./users.js" +assert.deepEqual(appUserUpdateSchema.parse({ name: "Renamed" }), { name: "Renamed" }) +assert.deepEqual(appUserUpdateSchema.parse({}), {}) +assert.deepEqual(appUserUpdateSchema.parse({ active: false }), { active: false }) +const created = appUserCreateSchema.parse({ name: "A", login: "a", email: "a@example.invalid" }) +assert.equal(created.role, "viewer"); assert.equal(created.active, true); assert.deepEqual(created.sections, []); assert.deepEqual(created.servers, []) +for (const type of ["bridge", "vlan", "vxlan", "ipsec", "other"]) assert.equal(interfaceTypeSchema.parse(type), type) +console.log("User PATCH preserves omitted role, activity and permissions; creation defaults and native interface types remain valid") diff --git a/packages/contracts/src/users.ts b/packages/contracts/src/users.ts index 91cf2cf0..03493a73 100644 --- a/packages/contracts/src/users.ts +++ b/packages/contracts/src/users.ts @@ -2,7 +2,7 @@ import { z } from "zod" export const appUserRoleSchema = z.enum(["admin", "operator", "viewer"]) export const permLevelSchema = z.enum(["none", "read", "write"]) -export const interfaceTypeSchema = z.enum(["ether", "gre", "wg", "ipsec", "other"]) +export const interfaceTypeSchema = z.enum(["ether", "bridge", "vlan", "vxlan", "gre", "wg", "ipsec", "other"]) export const sectionPermSchema = z.object({ section: z.string().min(1), @@ -58,7 +58,12 @@ export const appUserCreateSchema = z.object({ servers: z.array(serverPermSchema).default([]), }) -export const appUserUpdateSchema = appUserCreateSchema.partial() +// Zod applies inner defaults even through optional fields. PATCH must not turn +// an omitted role/permissions into the creation defaults. +export const appUserUpdateSchema = appUserCreateSchema.extend({ + role: appUserRoleSchema, active: z.boolean(), + sections: z.array(sectionPermSchema), servers: z.array(serverPermSchema), +}).partial() export const appUserIdParamSchema = z.object({ id: z.string().min(1),