fix(mcp): обновить проверенный снимок маршрутизации и хранения операций
This commit is contained in:
Denozordec committed
1 parent
f1af1a85e5
commit
5ce56b5168
117 files changed
+9960
-182
No files matched your search
@@ -0,0 +1,13 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.mcp_operation_replay_tombstones (
|
||||
operation_id uuid PRIMARY KEY,
|
||||
plan_id uuid NOT NULL UNIQUE,
|
||||
connection_id uuid NOT NULL REFERENCES public.mcp_connections(id),
|
||||
idempotency_digest text NOT NULL CHECK (idempotency_digest ~ '^[0-9a-f]{64}$'),
|
||||
action_id text NOT NULL CHECK (length(action_id) BETWEEN 1 AND 200),
|
||||
target_server_ids bigint[] NOT NULL CHECK (array_position(target_server_ids, NULL) IS NULL),
|
||||
target_global boolean NOT NULL,
|
||||
UNIQUE (connection_id,idempotency_digest)
|
||||
);
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0045_mcp_operation_replay_tombstones') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,186 @@
|
||||
-- Inactive poll-history V1-to-V2 retirement. Application owner authenticates
|
||||
-- both ciphertext versions; SQL independently fences identity and lifetime.
|
||||
BEGIN;
|
||||
LOCK TABLE public.server_poll_archives,public.server_poll_storage_admissions IN ACCESS EXCLUSIVE MODE;
|
||||
ALTER TABLE public.server_poll_archives DROP CONSTRAINT server_poll_archives_version_check;
|
||||
ALTER TABLE public.server_poll_archives ADD CONSTRAINT server_poll_archives_version_check
|
||||
CHECK(version=1 OR (version=2 AND decoded_bytes<=24576 AND compressed_bytes<=24576 AND ciphertext_bytes<=33000));
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.server_poll_archive_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='DELETE' AND public.server_poll_storage_gc_allowed(OLD.receipt_id) THEN RETURN OLD; END IF;
|
||||
IF TG_OP='UPDATE' AND OLD.version=1 AND NEW.version=2
|
||||
AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text
|
||||
AND NEW.ciphertext_bytes<=OLD.ciphertext_bytes
|
||||
AND NEW.ciphertext ~ '^[A-Za-z0-9+/]+={0,2}$'
|
||||
AND replace(encode(decode(NEW.ciphertext,'base64'),'base64'),E'\n','')=NEW.ciphertext
|
||||
AND octet_length(decode(NEW.ciphertext,'base64'))=NEW.compressed_bytes+28
|
||||
AND (to_jsonb(OLD)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof'])=
|
||||
(to_jsonb(NEW)-ARRAY['version','source_digest','source_sha256','decoded_bytes','compressed_bytes','ciphertext','ciphertext_bytes','archive_proof'])
|
||||
AND EXISTS(
|
||||
SELECT 1 FROM public.server_poll_storage_admissions a JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=o.id AND t.plan_id=p.id AND t.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id
|
||||
WHERE a.receipt_id=OLD.receipt_id AND a.identity_kind='mcp' AND a.storage_state='compacted'
|
||||
AND a.execution_id=OLD.execution_id AND a.connection_id=OLD.connection_id AND a.owner_id=OLD.owner_id
|
||||
AND a.binding_proof=OLD.binding_proof AND a.context_proof=OLD.context_proof AND a.archive_bytes=OLD.ciphertext_bytes
|
||||
AND a.original_activity_at=OLD.original_activity_at AND a.authority_expires_at=OLD.original_expires_at
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at)
|
||||
AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.state='completed' AND p.operation_id='servers.poll' AND p.operation_version=1 AND p.approved_by IS NOT NULL
|
||||
AND s.state='verified' AND s.finished_at IS NOT NULL AND t.action_id=p.operation_id
|
||||
AND t.target_global AND cardinality(t.target_server_ids)>0
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'poll archive immutable outside exact retirement or original lifetime GC';
|
||||
END $$;
|
||||
|
||||
CREATE FUNCTION public.server_poll_retirement_final_guard() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF NOT EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones t
|
||||
WHERE t.operation_id=NEW.execution_id AND t.connection_id=NEW.connection_id AND t.action_id='servers.poll'
|
||||
AND t.target_global AND NOT EXISTS(SELECT 1 FROM public.mcp_plans p WHERE p.id=t.plan_id))
|
||||
OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=NEW.execution_id)
|
||||
OR NOT public.server_poll_storage_accounting_valid()
|
||||
THEN RAISE EXCEPTION 'poll retirement is incomplete'; END IF;
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE CONSTRAINT TRIGGER server_poll_retirement_final_guard AFTER UPDATE ON public.server_poll_archives
|
||||
DEFERRABLE INITIALLY DEFERRED FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2)
|
||||
EXECUTE FUNCTION public.server_poll_retirement_final_guard();
|
||||
|
||||
CREATE OR REPLACE FUNCTION server_poll_storage_admission_guard() RETURNS trigger LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
DECLARE retention integer; charged integer; r public.server_poll_archives%ROWTYPE;
|
||||
BEGIN
|
||||
PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0));
|
||||
IF TG_OP='DELETE' THEN
|
||||
IF NOT public.server_poll_storage_gc_allowed(OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id)
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id)
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id)
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id) THEN RAISE EXCEPTION 'poll lifetime or children prevent admission deletion'; END IF;
|
||||
UPDATE public.server_poll_storage_usage SET admissions=admissions-CASE WHEN OLD.storage_state='full' THEN 1 ELSE 0 END,
|
||||
compacted_admissions=compacted_admissions-CASE WHEN OLD.storage_state='compacted' THEN 1 ELSE 0 END,
|
||||
archive_bytes=archive_bytes-COALESCE(OLD.archive_bytes,0),
|
||||
reserved_rows=reserved_rows-CASE WHEN OLD.storage_state='full' THEN 4019 ELSE 2 END,
|
||||
reserved_bytes=reserved_bytes-CASE WHEN OLD.storage_state='full' THEN 8421376 ELSE 32768+OLD.archive_bytes END
|
||||
WHERE scope IN('global','owner:'||OLD.owner_id);
|
||||
GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll quota refund mismatch'; END IF; RETURN OLD;
|
||||
END IF;
|
||||
IF TG_OP='UPDATE' THEN
|
||||
IF OLD.storage_state='compacted' AND NEW.storage_state='compacted'
|
||||
AND current_setting('mm.poll_retire_receipt',true)=OLD.receipt_id::text
|
||||
AND (to_jsonb(OLD)-'archive_bytes')=(to_jsonb(NEW)-'archive_bytes') THEN
|
||||
SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND version=2;
|
||||
IF NOT FOUND OR r.ciphertext_bytes<>NEW.archive_bytes OR NEW.archive_bytes>OLD.archive_bytes
|
||||
OR r.execution_id<>OLD.execution_id OR r.connection_id<>OLD.connection_id OR r.owner_id<>OLD.owner_id
|
||||
THEN RAISE EXCEPTION 'retired poll charge source mismatch'; END IF;
|
||||
UPDATE public.server_poll_storage_usage
|
||||
SET archive_bytes=archive_bytes-OLD.archive_bytes+NEW.archive_bytes,
|
||||
reserved_bytes=reserved_bytes-OLD.archive_bytes+NEW.archive_bytes
|
||||
WHERE scope IN ('global','owner:'||OLD.owner_id) AND compacted_admissions>=1 AND archive_bytes>=OLD.archive_bytes;
|
||||
GET DIAGNOSTICS charged=ROW_COUNT;
|
||||
IF charged<>2 THEN RAISE EXCEPTION 'retired poll charge adjustment mismatch'; END IF;
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
-- This durable one-shot marker is written and ACKed in its OWN transaction
|
||||
-- before archive construction. A committed/unknown claim never resets and
|
||||
-- the worker never retries destructive work against an already claimed full
|
||||
-- source. Claim time is maintenance metadata, not renewed execution activity.
|
||||
IF OLD.storage_state='full' AND NEW.storage_state='full' AND OLD.compaction_claim IS NULL AND OLD.compaction_claimed_at IS NULL AND OLD.compaction_claim_xid IS NULL
|
||||
AND NEW.compaction_claim IS NOT NULL AND NEW.compaction_claim_xid IS NULL
|
||||
AND(to_jsonb(OLD)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid'])=(to_jsonb(NEW)-ARRAY['compaction_claim','compaction_claimed_at','compaction_claim_xid'])
|
||||
AND public.server_poll_storage_compact_allowed(OLD.receipt_id)
|
||||
THEN NEW.compaction_claimed_at:=clock_timestamp(); NEW.compaction_claim_xid:=pg_current_xact_id(); RETURN NEW; END IF;
|
||||
IF(to_jsonb(OLD)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) IS DISTINCT FROM
|
||||
(to_jsonb(NEW)-ARRAY['claim_token','claim_attempted_at','used_rows','used_bytes','storage_state','compact_at','archive_bytes','original_activity_at']) THEN RAISE EXCEPTION 'poll admission identity is immutable'; END IF;
|
||||
IF OLD.storage_state<>'full' THEN RAISE EXCEPTION 'compacted poll identity cannot reopen or change'; END IF;
|
||||
IF NEW.storage_state='compacted' THEN
|
||||
SELECT * INTO r FROM public.server_poll_archives WHERE receipt_id=OLD.receipt_id AND compaction_xid=pg_current_xact_id();
|
||||
IF NOT FOUND OR NEW.used_rows<>OLD.used_rows OR NEW.used_bytes<>OLD.used_bytes OR NEW.claim_token IS DISTINCT FROM OLD.claim_token
|
||||
OR NEW.claim_attempted_at IS DISTINCT FROM OLD.claim_attempted_at OR NEW.archive_bytes<>r.ciphertext_bytes OR NEW.original_activity_at<>r.original_activity_at
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_executions WHERE receipt_id=OLD.receipt_id)
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_intents WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_acks WHERE receipt_id=OLD.receipt_id)
|
||||
OR EXISTS(SELECT 1 FROM public.server_poll_receipts WHERE receipt_id=OLD.receipt_id) OR EXISTS(SELECT 1 FROM public.server_poll_quarantines WHERE receipt_id=OLD.receipt_id)
|
||||
THEN RAISE EXCEPTION 'poll compaction requires the complete current-transaction archive'; END IF;
|
||||
NEW.compact_at:=r.created_at;
|
||||
UPDATE public.server_poll_storage_usage SET admissions=admissions-1,compacted_admissions=compacted_admissions+1,
|
||||
archive_bytes=archive_bytes+r.ciphertext_bytes,reserved_rows=reserved_rows-4019+2,reserved_bytes=reserved_bytes-8421376+32768+r.ciphertext_bytes
|
||||
WHERE scope IN('global','owner:'||OLD.owner_id) AND admissions>=1;
|
||||
GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll compaction charge mismatch'; END IF;
|
||||
RETURN NEW;
|
||||
END IF;
|
||||
IF NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL THEN RAISE EXCEPTION 'full poll cannot carry archive metadata'; END IF;
|
||||
IF OLD.claim_token IS NOT NULL AND NEW.claim_token IS NULL AND OLD.claim_attempted_at IS NULL AND NEW.claim_attempted_at IS NOT NULL
|
||||
AND NEW.used_rows=OLD.used_rows AND NEW.used_bytes=OLD.used_bytes AND OLD.authority_expires_at>clock_timestamp()
|
||||
THEN NEW.claim_attempted_at:=clock_timestamp(); RETURN NEW; END IF;
|
||||
IF OLD.claim_token IS NOT DISTINCT FROM NEW.claim_token AND OLD.claim_attempted_at IS NOT DISTINCT FROM NEW.claim_attempted_at
|
||||
AND NEW.used_rows=OLD.used_rows+1 AND NEW.used_bytes>OLD.used_bytes THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'poll admission cannot refresh, reopen or refund itself';
|
||||
END IF;
|
||||
IF NEW.storage_state<>'full' OR NEW.compact_at IS NOT NULL OR NEW.archive_bytes IS NOT NULL OR NEW.original_activity_at IS NOT NULL
|
||||
OR NEW.compaction_claim IS NOT NULL OR NEW.compaction_claimed_at IS NOT NULL OR NEW.compaction_claim_xid IS NOT NULL
|
||||
OR NOT public.server_poll_storage_accounting_valid() THEN RAISE EXCEPTION 'new poll requires full reservation and exact ledger'; END IF;
|
||||
SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE id=1;
|
||||
IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 OR NEW.authority_issued_at>clock_timestamp() OR NEW.authority_expires_at<=clock_timestamp()
|
||||
OR NEW.claim_token IS NULL OR NEW.claim_attempted_at IS NOT NULL OR NEW.used_rows<>0 OR NEW.used_bytes<>0 THEN RAISE EXCEPTION 'new poll admission needs original live authority'; END IF;
|
||||
IF EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE receipt_id=NEW.receipt_id OR(identity_kind=NEW.identity_kind AND execution_id=NEW.execution_id)) THEN RAISE EXCEPTION 'poll identity already spent'; END IF;
|
||||
NEW.created_at:=clock_timestamp(); NEW.retention_days:=retention; NEW.safety_ms:=60000;
|
||||
NEW.gc_not_before:=NEW.authority_expires_at+make_interval(days=>retention)+interval '60 seconds';
|
||||
INSERT INTO public.server_poll_storage_usage(scope) VALUES('owner:'||NEW.owner_id) ON CONFLICT DO NOTHING;
|
||||
IF(SELECT count(*) FROM public.server_poll_storage_usage WHERE scope IN('global','owner:'||NEW.owner_id)
|
||||
AND admissions<CASE WHEN scope='global' THEN 128 ELSE 8 END
|
||||
AND reserved_rows<=CASE WHEN scope='global' THEN 514432 ELSE 32152 END-4019
|
||||
AND reserved_bytes<=CASE WHEN scope='global' THEN 1077936128 ELSE 67371008 END-8421376)<>2 THEN RAISE EXCEPTION 'poll admission quota exhausted'; END IF;
|
||||
UPDATE public.server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope IN('global','owner:'||NEW.owner_id);
|
||||
GET DIAGNOSTICS charged=ROW_COUNT; IF charged<>2 THEN RAISE EXCEPTION 'poll admission quota unavailable'; END IF;
|
||||
RETURN NEW;
|
||||
END $$;
|
||||
CREATE OR REPLACE FUNCTION server_poll_storage_gc_allowed(id uuid) RETURNS boolean LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
DECLARE a public.server_poll_storage_admissions%ROWTYPE; retention integer; activity timestamptz; terminal_at timestamptz;
|
||||
BEGIN
|
||||
IF current_setting('mm.poll_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF;
|
||||
PERFORM pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0));
|
||||
SELECT * INTO a FROM public.server_poll_storage_admissions WHERE receipt_id=id FOR UPDATE;
|
||||
IF NOT FOUND OR a.authority_expires_at>=clock_timestamp() OR a.gc_not_before>=clock_timestamp() THEN RETURN false; END IF;
|
||||
SELECT requests_retention_days INTO retention FROM public.mcp_settings WHERE mcp_settings.id=1 FOR SHARE;
|
||||
IF retention IS NULL OR retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF;
|
||||
terminal_at:=a.created_at;
|
||||
IF a.identity_kind='mcp' THEN
|
||||
IF NOT pg_try_advisory_xact_lock(hashtextextended('mcp-operation:'||a.execution_id::text,0)) THEN RETURN false; END IF;
|
||||
BEGIN
|
||||
SELECT GREATEST(o.finished_at,p.expires_at) INTO terminal_at FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN public.mcp_connections c ON c.id=o.connection_id
|
||||
WHERE o.id=a.execution_id AND o.connection_id=a.connection_id AND c.owner_id=a.owner_id AND p.operation_id='servers.poll'
|
||||
AND o.state IN('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL FOR UPDATE OF o,p NOWAIT;
|
||||
IF NOT FOUND THEN
|
||||
IF a.storage_state<>'compacted' OR EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=a.execution_id) OR NOT EXISTS (
|
||||
SELECT 1 FROM public.mcp_operation_replay_tombstones t JOIN public.mcp_connections c ON c.id=t.connection_id
|
||||
WHERE t.operation_id=a.execution_id AND t.connection_id=a.connection_id AND c.owner_id=a.owner_id
|
||||
AND t.action_id='servers.poll' AND t.target_global AND cardinality(t.target_server_ids)>0
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE mcp_plans.id=t.plan_id)
|
||||
) OR EXISTS(SELECT 1 FROM public.server_poll_archives r WHERE r.receipt_id=id AND
|
||||
(r.version<>2 OR r.execution_id<>a.execution_id OR r.connection_id<>a.connection_id OR r.owner_id<>a.owner_id
|
||||
OR r.original_activity_at IS DISTINCT FROM a.original_activity_at OR r.original_expires_at<>a.authority_expires_at
|
||||
OR r.binding_proof<>a.binding_proof OR r.context_proof<>a.context_proof OR r.ciphertext_bytes<>a.archive_bytes))
|
||||
THEN RETURN false; END IF;
|
||||
terminal_at:=a.original_activity_at;
|
||||
END IF;
|
||||
IF terminal_at IS NULL THEN RETURN false; END IF;
|
||||
EXCEPTION WHEN lock_not_available THEN RETURN false; END;
|
||||
END IF;
|
||||
IF a.storage_state='compacted' THEN activity:=GREATEST(a.original_activity_at,a.authority_expires_at,terminal_at);
|
||||
ELSE
|
||||
SELECT GREATEST(a.authority_expires_at,a.created_at,COALESCE(a.claim_attempted_at,a.created_at),terminal_at,
|
||||
COALESCE((SELECT created_at FROM public.server_poll_executions WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT max(created_at) FROM public.server_poll_intents WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT max(created_at) FROM public.server_poll_acks WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT created_at FROM public.server_poll_receipts WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT created_at FROM public.server_poll_quarantines WHERE receipt_id=id),a.created_at)) INTO activity;
|
||||
END IF;
|
||||
RETURN isfinite(activity) AND activity+make_interval(days=>GREATEST(a.retention_days,retention))+a.safety_ms*interval '1 millisecond'<clock_timestamp();
|
||||
END $$;
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0046_server_poll_operation_retirement') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,40 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.server_poll_retirement_claims (
|
||||
receipt_id uuid PRIMARY KEY REFERENCES public.server_poll_storage_admissions(receipt_id) ON DELETE CASCADE,
|
||||
old_archive_proof text NOT NULL CHECK(old_archive_proof ~ '^[0-9a-f]{64}$'),
|
||||
claim_token uuid NOT NULL,
|
||||
claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id()
|
||||
);
|
||||
CREATE FUNCTION public.server_poll_retirement_claim_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||
SELECT 1 FROM public.server_poll_archives r WHERE r.receipt_id=NEW.receipt_id
|
||||
AND r.version=1 AND r.archive_proof=NEW.old_archive_proof
|
||||
) THEN RETURN NEW; END IF;
|
||||
IF TG_OP='DELETE' AND pg_trigger_depth()>1 AND NOT EXISTS(
|
||||
SELECT 1 FROM public.server_poll_storage_admissions a WHERE a.receipt_id=OLD.receipt_id
|
||||
) THEN RETURN OLD; END IF;
|
||||
RAISE EXCEPTION 'poll retirement claim cannot be replaced or reopened';
|
||||
END $$;
|
||||
CREATE TRIGGER server_poll_retirement_claim_immutable
|
||||
BEFORE INSERT OR UPDATE OR DELETE ON public.server_poll_retirement_claims
|
||||
FOR EACH ROW EXECUTE FUNCTION public.server_poll_retirement_claim_immutable();
|
||||
CREATE TRIGGER server_poll_retirement_claim_no_truncate
|
||||
BEFORE TRUNCATE ON public.server_poll_retirement_claims
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.server_poll_retirement_claim_immutable();
|
||||
CREATE FUNCTION public.server_poll_retirement_requires_claim() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c
|
||||
WHERE c.receipt_id=OLD.receipt_id AND c.old_archive_proof=OLD.archive_proof
|
||||
AND c.claim_token::text=current_setting('mm.poll_retire_claim',true)
|
||||
AND c.claim_xid<>pg_current_xact_id())
|
||||
THEN RAISE EXCEPTION 'poll retirement requires prior exact claim'; END IF;
|
||||
RETURN NEW;
|
||||
END $$;
|
||||
CREATE TRIGGER server_poll_retirement_requires_claim BEFORE UPDATE ON public.server_poll_archives
|
||||
FOR EACH ROW WHEN(OLD.version=1 AND NEW.version=2)
|
||||
EXECUTE FUNCTION public.server_poll_retirement_requires_claim();
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0047_server_poll_retirement_claims') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,27 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.mcp_ordinary_retirement_claims (
|
||||
operation_id uuid PRIMARY KEY,
|
||||
plan_id uuid NOT NULL UNIQUE,
|
||||
connection_id uuid NOT NULL REFERENCES public.mcp_connections(id),
|
||||
source_proof text NOT NULL CHECK(source_proof ~ '^[0-9a-f]{64}$'),
|
||||
claim_token uuid NOT NULL,
|
||||
claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id()
|
||||
);
|
||||
CREATE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.operation_id='logs.settings.update' AND p.operation_version=1 AND p.state='completed'
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
|
||||
END $$;
|
||||
CREATE TRIGGER mcp_ordinary_retirement_claim_immutable BEFORE INSERT OR UPDATE OR DELETE
|
||||
ON public.mcp_ordinary_retirement_claims FOR EACH ROW EXECUTE FUNCTION public.mcp_ordinary_retirement_claim_immutable();
|
||||
CREATE TRIGGER mcp_ordinary_retirement_claim_no_truncate BEFORE TRUNCATE
|
||||
ON public.mcp_ordinary_retirement_claims FOR EACH STATEMENT EXECUTE FUNCTION public.mcp_ordinary_retirement_claim_immutable();
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0048_mcp_ordinary_retirement_claims') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,18 @@
|
||||
BEGIN;
|
||||
CREATE OR REPLACE FUNCTION public.mcp_ordinary_retirement_claim_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||
SELECT 1 FROM public.mcp_operations o JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
WHERE o.id=NEW.operation_id AND o.plan_id=NEW.plan_id AND o.connection_id=NEW.connection_id
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND (p.operation_id,p.operation_version) IN (
|
||||
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
|
||||
) AND p.state='completed'
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'ordinary operation retirement claim cannot be replaced or reopened';
|
||||
END $$;
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0049_mcp_ordinary_telemetry_retirement') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,22 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.server_probe_retired_sources (
|
||||
receipt_id uuid PRIMARY KEY REFERENCES public.server_probe_storage_admissions(receipt_id),
|
||||
operation_id uuid NOT NULL UNIQUE REFERENCES public.mcp_operation_replay_tombstones(operation_id),
|
||||
source jsonb NOT NULL CHECK(jsonb_typeof(source)='object' AND octet_length(source::text) BETWEEN 1 AND 8192),
|
||||
proof text NOT NULL CHECK(proof ~ '^[0-9a-f]{64}$'),
|
||||
CHECK((source->>'version') IS NOT DISTINCT FROM '1'
|
||||
AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text
|
||||
AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text)
|
||||
);
|
||||
CREATE FUNCTION public.server_probe_retired_source_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
RAISE EXCEPTION 'retired diagnostic source is immutable';
|
||||
END $$;
|
||||
CREATE TRIGGER server_probe_retired_source_immutable
|
||||
BEFORE UPDATE OR DELETE ON public.server_probe_retired_sources
|
||||
FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_immutable();
|
||||
CREATE TRIGGER server_probe_retired_source_no_truncate BEFORE TRUNCATE ON public.server_probe_retired_sources
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_retired_source_immutable();
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0050_server_probe_retired_sources') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,52 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.server_probe_retirement_claims (
|
||||
receipt_id uuid PRIMARY KEY,
|
||||
operation_id uuid NOT NULL UNIQUE,
|
||||
plan_id uuid NOT NULL UNIQUE,
|
||||
connection_id uuid NOT NULL,
|
||||
source_proof text NOT NULL CHECK(source_proof ~ '^[a-f0-9]{64}$'),
|
||||
claim_token uuid NOT NULL,
|
||||
claim_xid xid8 NOT NULL DEFAULT pg_current_xact_id()
|
||||
);
|
||||
CREATE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||
SELECT 1 FROM public.server_probe_storage_admissions a
|
||||
JOIN public.mcp_operations o ON o.id=a.execution_id
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id
|
||||
WHERE a.receipt_id=NEW.receipt_id AND a.identity_kind='mcp' AND a.claim_token IS NULL
|
||||
AND a.claim_attempted_at IS NOT NULL AND a.cancelled_at IS NULL AND a.purged_at IS NULL
|
||||
AND o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id
|
||||
AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'diagnostic retirement claim cannot be replaced or reopened';
|
||||
END $$;
|
||||
CREATE TRIGGER server_probe_retirement_claim_guard BEFORE INSERT OR UPDATE OR DELETE
|
||||
ON public.server_probe_retirement_claims FOR EACH ROW EXECUTE FUNCTION public.server_probe_retirement_claim_guard();
|
||||
CREATE TRIGGER server_probe_retirement_claim_no_truncate BEFORE TRUNCATE
|
||||
ON public.server_probe_retirement_claims FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_retirement_claim_guard();
|
||||
CREATE FUNCTION public.server_probe_retired_source_insert_guard() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF EXISTS(
|
||||
SELECT 1 FROM public.server_probe_retirement_claims r
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.plan_id=r.plan_id
|
||||
AND t.connection_id=r.connection_id AND t.action_id='servers.probes.run' AND NOT t.target_global
|
||||
JOIN public.mcp_operations o ON o.id=r.operation_id AND o.plan_id=r.plan_id AND o.connection_id=r.connection_id
|
||||
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id
|
||||
WHERE r.receipt_id=NEW.receipt_id AND r.operation_id=NEW.operation_id AND r.source_proof=NEW.proof
|
||||
AND r.claim_xid<>pg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true)
|
||||
AND NEW.source->>'planId'=r.plan_id::text AND NEW.source->>'connectionId'=r.connection_id::text
|
||||
AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof
|
||||
AND NEW.source->'serverIds'=to_jsonb(t.target_server_ids)
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'diagnostic archive requires an exact committed retirement claim';
|
||||
END $$;
|
||||
CREATE TRIGGER server_probe_retired_source_insert_guard BEFORE INSERT ON public.server_probe_retired_sources
|
||||
FOR EACH ROW EXECUTE FUNCTION public.server_probe_retired_source_insert_guard();
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0051_server_probe_retirement_claims') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,96 @@
|
||||
BEGIN;
|
||||
ALTER TABLE public.server_probe_retired_sources
|
||||
DROP CONSTRAINT server_probe_retired_sources_receipt_id_fkey,
|
||||
ADD CONSTRAINT server_probe_retired_sources_receipt_id_fkey
|
||||
FOREIGN KEY(receipt_id) REFERENCES public.server_probe_storage_admissions(receipt_id) ON DELETE CASCADE;
|
||||
|
||||
CREATE FUNCTION public.server_probe_retired_gc_allowed(target_receipt uuid,current_retention integer) RETURNS boolean
|
||||
LANGUAGE plpgsql SET TimeZone='UTC' SET search_path=pg_catalog,public AS $$
|
||||
DECLARE activity timestamptz; retained_days integer; safety integer;
|
||||
BEGIN
|
||||
IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on'
|
||||
OR current_setting('mm.probe_retired_gc_receipt',true) IS DISTINCT FROM target_receipt::text THEN RETURN false; END IF;
|
||||
SELECT (r.source->>'activityAt')::timestamptz,a.retention_days,a.safety_ms
|
||||
INTO activity,retained_days,safety
|
||||
FROM public.server_probe_retired_sources r
|
||||
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
|
||||
JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id
|
||||
WHERE r.receipt_id=target_receipt AND r.proof=current_setting('mm.probe_retired_gc_proof',true)
|
||||
AND a.identity_kind='mcp' AND a.cancelled_at IS NULL AND a.claim_token IS NULL AND isfinite(a.claim_attempted_at)
|
||||
AND t.action_id='servers.probes.run' AND NOT t.target_global
|
||||
AND r.source->>'planId'=t.plan_id::text AND r.source->>'connectionId'=c.id::text
|
||||
AND r.source->'serverIds'=to_jsonb(t.target_server_ids) AND r.source->>'ownerId'=a.owner_id
|
||||
AND r.source->>'bindingProof'=a.binding_proof
|
||||
AND (r.source->>'authorityExpiresAt')::timestamptz=a.authority_expires_at
|
||||
AND (r.source->>'gcNotBefore')::timestamptz=a.gc_not_before
|
||||
AND (r.source->>'retentionDays')::int=a.retention_days AND (r.source->>'safetyMs')::int=a.safety_ms
|
||||
AND isfinite(a.authority_expires_at) AND isfinite(a.gc_not_before)
|
||||
AND a.authority_expires_at<clock_timestamp() AND a.gc_not_before<clock_timestamp()
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE id=t.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE id=t.plan_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=t.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=t.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=t.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines q
|
||||
WHERE q.receipt_id=r.receipt_id OR(q.identity_kind='mcp' AND q.execution_id=r.operation_id));
|
||||
IF NOT FOUND OR activity IS NULL OR NOT isfinite(activity)
|
||||
OR current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF;
|
||||
IF EXISTS(SELECT 1 FROM public.server_probe_executions j WHERE j.receipt_id=target_receipt
|
||||
AND (j.finished_at IS NULL OR j.finished_at>activity))
|
||||
OR EXISTS(SELECT 1 FROM public.server_probe_execution_stages s WHERE s.receipt_id=target_receipt
|
||||
AND GREATEST(s.created_at,COALESCE(s.received_at,s.created_at))>activity)
|
||||
THEN RETURN false; END IF;
|
||||
RETURN activity+make_interval(days=>GREATEST(retained_days,current_retention))
|
||||
+GREATEST(safety,60000)*interval '1 millisecond'<clock_timestamp();
|
||||
END $$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.server_probe_retired_source_immutable() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='DELETE' THEN
|
||||
IF current_setting('mm.probe_storage_gc',true)='on'
|
||||
AND current_setting('mm.probe_retired_gc_receipt',true)=OLD.receipt_id::text
|
||||
AND current_setting('mm.probe_retired_gc_proof',true)=OLD.proof
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE receipt_id=OLD.receipt_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_executions WHERE receipt_id=OLD.receipt_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_payloads WHERE receipt_id=OLD.receipt_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_stages WHERE receipt_id=OLD.receipt_id)
|
||||
THEN RETURN OLD; END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION 'retired diagnostic source is immutable';
|
||||
END $$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.server_probe_storage_gc_allowed(id uuid) RETURNS boolean
|
||||
LANGUAGE plpgsql SET TimeZone='UTC' AS $$
|
||||
DECLARE a server_probe_storage_admissions%ROWTYPE; activity timestamptz; current_retention integer; terminal_at timestamptz;
|
||||
BEGIN
|
||||
IF current_setting('mm.probe_storage_gc',true) IS DISTINCT FROM 'on' THEN RETURN false; END IF;
|
||||
PERFORM pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0));
|
||||
SELECT * INTO a FROM server_probe_storage_admissions WHERE receipt_id=id FOR UPDATE;
|
||||
IF NOT FOUND OR a.gc_not_before>=clock_timestamp() OR a.authority_expires_at>=clock_timestamp() THEN RETURN false; END IF;
|
||||
SELECT settings.requests_retention_days INTO current_retention FROM mcp_settings settings WHERE settings.id=1;
|
||||
IF current_retention IS NULL OR current_retention NOT BETWEEN 1 AND 365 THEN RETURN false; END IF;
|
||||
IF EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=id) THEN
|
||||
RETURN public.server_probe_retired_gc_allowed(id,current_retention);
|
||||
END IF;
|
||||
terminal_at:=a.created_at;
|
||||
IF a.identity_kind='mcp' THEN
|
||||
SELECT GREATEST(o.finished_at,p.expires_at,COALESCE(cancellation.created_at,o.finished_at)) INTO terminal_at
|
||||
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN mcp_connections owner_connection ON owner_connection.id=o.connection_id
|
||||
LEFT JOIN mcp_diagnostic_cancellations cancellation ON cancellation.operation_id=o.id
|
||||
WHERE o.id=a.execution_id AND owner_connection.owner_id=a.owner_id AND p.operation_id='servers.probes.run'
|
||||
AND o.state IN ('applied','failed','unknown','rejected','partial','cancelled') AND o.finished_at IS NOT NULL
|
||||
AND (cancellation.operation_id IS NULL OR(cancellation.connection_id=o.connection_id AND cancellation.actor_id=a.owner_id))
|
||||
FOR UPDATE OF o,p,owner_connection;
|
||||
IF NOT FOUND OR terminal_at IS NULL THEN RETURN false; END IF;
|
||||
END IF;
|
||||
SELECT GREATEST(a.created_at,a.authority_expires_at,terminal_at,COALESCE(a.cancelled_at,a.created_at),
|
||||
COALESCE((SELECT finished_at FROM server_probe_executions WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT created_at FROM server_probe_execution_quarantines WHERE receipt_id=id),a.created_at),
|
||||
COALESCE((SELECT max(GREATEST(created_at,COALESCE(received_at,created_at))) FROM server_probe_execution_stages WHERE receipt_id=id),a.created_at)) INTO activity;
|
||||
RETURN activity+make_interval(days=>GREATEST(a.retention_days,current_retention))+a.safety_ms*interval '1 millisecond'<clock_timestamp();
|
||||
END $$;
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0052_server_probe_retired_gc') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -0,0 +1,74 @@
|
||||
BEGIN;
|
||||
CREATE TABLE public.server_probe_terminal_proofs (
|
||||
operation_id uuid PRIMARY KEY REFERENCES public.mcp_operations(id),
|
||||
receipt_id uuid NOT NULL UNIQUE,
|
||||
source jsonb NOT NULL CHECK(jsonb_typeof(source)='object' AND octet_length(source::text) BETWEEN 1 AND 8192),
|
||||
source_proof text NOT NULL CHECK(source_proof ~ '^[a-f0-9]{64}$'),
|
||||
idempotency_digest text NOT NULL CHECK(idempotency_digest ~ '^[a-f0-9]{64}$'),
|
||||
capture_proof text NOT NULL CHECK(capture_proof ~ '^[a-f0-9]{64}$'),
|
||||
CHECK((source->>'version') IS NOT DISTINCT FROM '1'
|
||||
AND (source->>'executionId') IS NOT DISTINCT FROM operation_id::text
|
||||
AND (source->>'receiptId') IS NOT DISTINCT FROM receipt_id::text)
|
||||
);
|
||||
CREATE FUNCTION public.server_probe_terminal_proof_guard() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' THEN
|
||||
IF EXISTS(
|
||||
SELECT 1 FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id
|
||||
JOIN public.server_probe_storage_admissions a ON a.execution_id=o.id AND a.identity_kind='mcp' AND a.owner_id=c.owner_id
|
||||
JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id
|
||||
JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id AND b.receipt_proof=j.receipt_proof
|
||||
WHERE o.id=NEW.operation_id AND a.receipt_id=NEW.receipt_id AND a.purged_at IS NULL AND a.cancelled_at IS NULL
|
||||
AND a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL
|
||||
AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND p.state='completed' AND p.operation_id='servers.probes.run' AND p.operation_version=1
|
||||
AND j.state='completed' AND j.execution_id=o.id AND j.mcp_connection_id=c.id AND j.actor_id=c.owner_id
|
||||
AND NEW.source->>'connectionId'=c.id::text AND NEW.source->>'planId'=p.id::text
|
||||
AND NEW.source->>'ownerId'=a.owner_id AND NEW.source->>'bindingProof'=a.binding_proof
|
||||
AND NEW.source->>'receiptProof'=b.receipt_proof
|
||||
AND NEW.source->>'ciphertextSha256'=encode(sha256(convert_to(b.receipt_ciphertext,'UTF8')),'hex')
|
||||
) THEN RETURN NEW; END IF;
|
||||
ELSIF TG_OP='DELETE' THEN
|
||||
IF EXISTS(
|
||||
SELECT 1 FROM public.server_probe_retirement_claims r
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.plan_id=r.plan_id AND t.connection_id=r.connection_id
|
||||
WHERE r.operation_id=OLD.operation_id AND r.receipt_id=OLD.receipt_id AND r.source_proof=OLD.source_proof
|
||||
AND r.claim_xid<>pg_current_xact_id() AND r.claim_token::text=current_setting('mm.probe_retirement_claim',true)
|
||||
AND t.action_id='servers.probes.run' AND NOT t.target_global AND t.idempotency_digest=OLD.idempotency_digest
|
||||
AND OLD.source->'serverIds'=to_jsonb(t.target_server_ids)
|
||||
) THEN RETURN OLD; END IF;
|
||||
END IF;
|
||||
RAISE EXCEPTION 'diagnostic terminal proof must match its original source or exact retirement claim';
|
||||
END $$;
|
||||
CREATE TRIGGER server_probe_terminal_proof_guard BEFORE INSERT OR UPDATE OR DELETE
|
||||
ON public.server_probe_terminal_proofs FOR EACH ROW EXECUTE FUNCTION public.server_probe_terminal_proof_guard();
|
||||
CREATE TRIGGER server_probe_terminal_proof_no_truncate BEFORE TRUNCATE
|
||||
ON public.server_probe_terminal_proofs FOR EACH STATEMENT EXECUTE FUNCTION public.server_probe_terminal_proof_guard();
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.server_probe_retirement_claim_guard() RETURNS trigger
|
||||
LANGUAGE plpgsql SET search_path=pg_catalog,public SET TimeZone='UTC' AS $$
|
||||
BEGIN
|
||||
IF TG_OP='INSERT' AND NEW.claim_xid=pg_current_xact_id() AND EXISTS(
|
||||
SELECT 1 FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id
|
||||
WHERE o.id=NEW.operation_id AND p.id=NEW.plan_id AND c.id=NEW.connection_id
|
||||
AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
||||
AND (EXISTS(SELECT 1 FROM public.server_probe_storage_admissions a
|
||||
WHERE a.receipt_id=NEW.receipt_id AND a.execution_id=o.id AND a.identity_kind='mcp' AND a.owner_id=c.owner_id
|
||||
AND a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL AND a.cancelled_at IS NULL AND a.purged_at IS NULL)
|
||||
OR(EXISTS(SELECT 1 FROM public.server_probe_terminal_proofs f
|
||||
WHERE f.operation_id=o.id AND f.receipt_id=NEW.receipt_id AND f.source_proof=NEW.source_proof
|
||||
AND f.source->>'ownerId'=c.owner_id AND f.source->>'connectionId'=c.id::text AND f.source->>'planId'=p.id::text)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions a
|
||||
WHERE a.receipt_id=NEW.receipt_id OR(a.identity_kind='mcp' AND a.execution_id=o.id))))
|
||||
) THEN RETURN NEW; END IF;
|
||||
RAISE EXCEPTION 'diagnostic retirement claim cannot be replaced or reopened';
|
||||
END $$;
|
||||
INSERT INTO public.schema_migrations(id) VALUES ('0053_server_probe_terminal_proofs') ON CONFLICT(id) DO NOTHING;
|
||||
COMMIT;
|
||||
@@ -66,7 +66,7 @@
|
||||
"test:mcp:ipsec-export-journal:pg": "node scripts/test-mcp-postgres.mjs --ipsec-export-journal --restore",
|
||||
"test:mcp:ipsec-export:http:pg": "node scripts/test-mcp-postgres.mjs --ipsec-export-http --restore",
|
||||
"test:mcp:diagnostic-cancel-host:pg": "node scripts/test-mcp-postgres.mjs --diagnostic-cancel-host --restore",
|
||||
"test:server-probe-management": "tsx src/services/server-probe-authority.test.ts && tsx src/services/server-probe-public-result.test.ts && tsx src/services/server-probe-execution.test.ts && tsx src/services/server-probe-rest-contract.test.ts && tsx src/services/server-probe-rest-execution.test.ts && tsx src/services/server-probe-journal.test.ts && tsx src/services/server-probe-journal-history.test.ts && tsx src/modules/mcp/server-probe-actions.test.ts && tsx src/modules/mcp/diagnostic-cancellation.test.ts && tsx src/modules/mcp/service-diagnostic-progress.test.ts && tsx src/services/mikrotik-get-admission.test.ts && tsx src/services/server-probe-storage-journal-adapter.test.ts && tsx --test src/services/server-probe-storage-history.test.ts && npm run test:server-probe-storage-models",
|
||||
"test:server-probe-management": "tsx src/services/server-probe-authority.test.ts && tsx src/services/server-probe-public-result.test.ts && tsx src/services/server-probe-execution.test.ts && tsx src/services/server-probe-rest-contract.test.ts && tsx src/services/server-probe-rest-execution.test.ts && tsx src/services/server-probe-journal.test.ts && tsx src/services/server-probe-journal-history.test.ts && tsx src/modules/mcp/server-probe-actions.test.ts && tsx src/modules/mcp/diagnostic-cancellation.test.ts && tsx src/modules/mcp/service-diagnostic-progress.test.ts && tsx src/services/mikrotik-get-admission.test.ts && tsx src/services/server-probe-storage-journal-adapter.test.ts && npm run test:server-probe-history && npm run test:server-probe-storage-models",
|
||||
"test:server-poll-http:pg": "node scripts/test-server-poll-http-owned.mjs",
|
||||
"test:server-poll-approval:pg": "node scripts/test-server-poll-approval-owned.mjs",
|
||||
"test:server-poll-atomic:pg": "node scripts/test-server-poll-atomic-owned.mjs",
|
||||
@@ -155,8 +155,9 @@
|
||||
"test:server-poll-durable:pg": "node scripts/test-mcp-postgres.mjs --server-poll-durable --restore",
|
||||
"test:server-probe-legacy:pg": "node scripts/test-mcp-postgres.mjs --probe-legacy-orphans",
|
||||
"test:server-probe-storage:pg": "node scripts/test-mcp-postgres.mjs --probe-storage-journal",
|
||||
"test:server-probe-history": "tsx --test src/services/server-probe-storage-history.test.ts",
|
||||
"test:server-probe-history": "tsx --test src/services/server-probe-storage-history.test.ts src/services/server-probe-retired-source.test.ts",
|
||||
"test:server-probe-history:pg": "node scripts/test-mcp-postgres.mjs --probe-storage-history",
|
||||
"test:server-probe-retirement:pg": "node scripts/test-mcp-postgres.mjs --probe-operation-retirement",
|
||||
"test:server-probe-storage-models": "tsx src/services/server-probe-journal-maintenance.test.ts && tsx src/modules/mcp/diagnostic-cancellation-source.test.ts && tsx src/services/server-probe-storage-source.test.ts && tsx src/services/server-probe-storage-connection.test.ts && tsx src/modules/mcp/server-probe-storage-host.test.ts && tsx src/services/server-probe-storage-rest.test.ts && tsx src/services/server-probe-storage-metadata.test.ts && tsx src/services/server-probe-legacy-verifier.test.ts && tsx src/services/server-probe-legacy-snapshot.test.ts && tsx src/services/server-probe-legacy-inspection.test.ts && tsx src/db/database-source-observation.test.ts && tsx src/services/server-probe-retention.test.ts && tsx src/modules/mcp/server-probe-registration.test.ts",
|
||||
"test:mcp:migration-data:pg": "node scripts/test-mcp-postgres.mjs --migration-data-inventory --restore",
|
||||
"test:mcp:poll-memory:pg": "npm run build -w @mmapp/contracts && node scripts/acceptance/build-poll-memory.mjs && node scripts/test-mcp-postgres.mjs --poll-memory",
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import assert from 'node:assert/strict'
|
||||
import { execFileSync } from 'node:child_process'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { lstat, mkdtemp, rm } from 'node:fs/promises'
|
||||
import { createServer } from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { isAbsolute, join, resolve } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import EmbeddedPostgres from 'embedded-postgres'
|
||||
import pg from 'pg'
|
||||
import { applySqlMigrations } from '../../src/db/migrate.ts'
|
||||
import { MIGRATION_MANIFEST } from '../../src/db/migration-manifest.ts'
|
||||
|
||||
assert.deepEqual(process.argv.slice(2), ['--owned-embedded'])
|
||||
const bin = process.env.MMAPP_PG_CLIENT_BIN
|
||||
assert(bin && isAbsolute(bin) && resolve(bin) === bin, 'Set MMAPP_PG_CLIENT_BIN to the exact local PostgreSQL client directory')
|
||||
const psql = join(bin, process.platform === 'win32' ? 'psql.exe' : 'psql')
|
||||
const executable = await lstat(psql)
|
||||
assert(executable.isFile() && !executable.isSymbolicLink(), 'PostgreSQL client must be a regular file')
|
||||
const clientEnv = Object.fromEntries(Object.entries(process.env).filter(([name]) => !/^PG/i.test(name)))
|
||||
const run = (command, args, options = {}) => execFileSync(command, args, {
|
||||
timeout: 10000, maxBuffer: 16384, encoding: 'utf8', windowsHide: true, ...options,
|
||||
})
|
||||
assert.match(run(psql, ['--version'], { env: clientEnv }), /^psql \(PostgreSQL\) (?:1[6-9]|[2-9][0-9])\./)
|
||||
const witness = fileURLToPath(new URL('../../../tmp/mcp-netflow-linux-topology-20261104/collector-drain-witness.py', import.meta.url))
|
||||
const { IDENTITY_SQL, SQL } = JSON.parse(run('python', ['-I', '-B', '-c',
|
||||
'import json,runpy,sys; m=runpy.run_path(sys.argv[1]); print(json.dumps({k:m[k] for k in ("IDENTITY_SQL","SQL")}))', witness]))
|
||||
assert(typeof SQL === 'string' && typeof IDENTITY_SQL === 'string')
|
||||
|
||||
const directory = await mkdtemp(join(tmpdir(), 'mm-netflow-drain-pg-'))
|
||||
const password = randomBytes(24).toString('hex')
|
||||
const host = '127.0.0.1', database = 'postgres', user = 'mcp_test'
|
||||
const listener = createServer()
|
||||
let embedded, pool, started = false
|
||||
try {
|
||||
await new Promise((done, reject) => { listener.once('error', reject); listener.listen(0, host, done) })
|
||||
const { port } = listener.address()
|
||||
await new Promise((done, reject) => listener.close(error => error ? reject(error) : done()))
|
||||
embedded = new EmbeddedPostgres({
|
||||
databaseDir: join(directory, 'pg'), port, user, password, persistent: true, createPostgresUser: false,
|
||||
initdbFlags: ['--encoding=UTF8'], postgresFlags: ['-h', host, '-c', 'unix_socket_directories='],
|
||||
onLog: () => {}, onError: () => {},
|
||||
})
|
||||
await embedded.initialise()
|
||||
await embedded.start()
|
||||
started = true
|
||||
pool = new pg.Pool({ host, port, database, user, password, max: 1, connectionTimeoutMillis: 5000 })
|
||||
await applySqlMigrations(pool)
|
||||
const { rows: [version] } = await pool.query("SELECT current_setting('server_version_num')::int AS number, (SELECT count(*)::int FROM schema_migrations) AS migrations")
|
||||
assert(version.number >= 180000 && version.number <= 189999)
|
||||
assert.equal(version.migrations, MIGRATION_MANIFEST.length)
|
||||
|
||||
const runId = 'a012bcde', owner = 'netflow-operator-' + runId, collectorPort = 51000
|
||||
const read = (sql, probePort = collectorPort) => JSON.parse(run(psql, ['-X', '-w', '-qAt',
|
||||
'-h', host, '-p', String(port), '-U', user, '-d', database, '-v', 'ON_ERROR_STOP=1',
|
||||
'-v', 'database=' + database, '-v', 'socket_dir=/tmp/mcp-netflow-' + runId + '/pg-socket',
|
||||
'-v', 'run_id=' + runId, '-v', 'port=' + probePort],
|
||||
{ env: { ...clientEnv, PGCONNECT_TIMEOUT: '2', PGPASSWORD: password }, input: sql }).trim())
|
||||
const identity = read(IDENTITY_SQL)
|
||||
assert.deepEqual(Object.keys(identity).sort(), ['oid', 'source'])
|
||||
assert.match(identity.oid, /^[1-9][0-9]{0,9}$/)
|
||||
// The production source predicate deliberately rejects this TCP fixture.
|
||||
assert.equal(identity.source, false)
|
||||
const expect = (planCount, enabled, verified, ordered = true, sqlPort = collectorPort) =>
|
||||
assert.deepEqual(read(SQL, sqlPort), { ...identity, connectionCount: 1, planCount, enabled, verified, ordered })
|
||||
|
||||
const uuid = index => 'a012bcde-0000-4000-8000-' + String(index).padStart(12, '0')
|
||||
const connection = uuid(1)
|
||||
await pool.query(`INSERT INTO mcp_connections(id,name,owner_id,owner_permissions,policy,generation)
|
||||
VALUES($1,$2,$3,'[]','{}','inert-witness')`, [connection, 'Owned NetFlow ' + runId, owner])
|
||||
await pool.query("UPDATE traffic_flow_settings SET collector_ip='198.18.242.3', flow_listen_port=$1, enabled=false WHERE id=1", [collectorPort])
|
||||
expect(0, false, [])
|
||||
|
||||
async function seed(index, enabled) {
|
||||
const plan = uuid(index), operation = uuid(index + 1), receipt = uuid(index + 2)
|
||||
const base = Date.UTC(2026, 0, 1) + index * 1000
|
||||
const at = offset => new Date(base + offset)
|
||||
const result = { receiptId: receipt, configurationConfirmed: true, receiptConfirmed: true,
|
||||
listener: 'request-acknowledged', automaticReplay: false }
|
||||
await pool.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at)
|
||||
VALUES($1,$2,'netflow.settings.update',3,1,'inert-witness','{}','{}','completed',$3,$4,$5,$6)`,
|
||||
[plan, connection, at(9000), owner, at(100), at(0)])
|
||||
await pool.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,created_at,finished_at)
|
||||
VALUES($1,$2,$3,$4,'applied','passed',$5,$6)`, [operation, plan, connection, 'inert-' + index, at(200), at(400)])
|
||||
await pool.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at)
|
||||
VALUES($1,0,'verified',$2,$3,$4,$5)`,
|
||||
[operation, { service: 'netflow.settings.update', input: { receiptId: receipt, enabled } }, result, at(300), at(400)])
|
||||
await pool.query(`INSERT INTO mcp_audit(actor_id,action,connection_id,payload,created_at)
|
||||
VALUES($1,'plan.approve',$2,$3,$4)`, [owner, connection, { planId: plan }, at(100)])
|
||||
await pool.query(`INSERT INTO flow_settings_intents(receipt_id,phase,request_proof,state,result,created_at)
|
||||
VALUES($1,'configuration','inert-proof','completed',$2,$3),($1,'listener','inert-proof','completed',$2,$3)`, [receipt, result, at(300)])
|
||||
await pool.query('UPDATE traffic_flow_settings SET enabled=$1 WHERE id=1', [enabled])
|
||||
return [plan, operation, receipt]
|
||||
}
|
||||
const enabled = await seed(10, true)
|
||||
expect(1, true, [enabled])
|
||||
const disabled = await seed(20, false)
|
||||
expect(2, false, [enabled, disabled])
|
||||
expect(2, null, [enabled, disabled], true, collectorPort + 1)
|
||||
|
||||
await pool.query("UPDATE flow_settings_intents SET request_proof='foreign' WHERE receipt_id=$1 AND phase='listener'", [disabled[2]])
|
||||
expect(2, false, [enabled])
|
||||
await pool.query("UPDATE flow_settings_intents SET request_proof='inert-proof' WHERE receipt_id=$1 AND phase='listener'", [disabled[2]])
|
||||
const { rows: [duplicate] } = await pool.query(`INSERT INTO mcp_audit(actor_id,action,connection_id,payload)
|
||||
VALUES($1,'plan.approve',$2,$3) RETURNING id`, [owner, connection, { planId: disabled[0] }])
|
||||
expect(2, false, [enabled])
|
||||
await pool.query('DELETE FROM mcp_audit WHERE id=$1', [duplicate.id])
|
||||
const { rows: [original] } = await pool.query(`UPDATE mcp_operations SET finished_at=(SELECT created_at + interval '1 second' FROM mcp_plans WHERE id=$2)
|
||||
WHERE id=$1 RETURNING created_at`, [enabled[1], disabled[0]])
|
||||
expect(2, false, [enabled, disabled], false)
|
||||
await pool.query("UPDATE mcp_operations SET finished_at=$2::timestamptz + interval '200 milliseconds' WHERE id=$1", [enabled[1], original.created_at])
|
||||
expect(2, false, [enabled, disabled])
|
||||
console.log(JSON.stringify({ migrations: version.migrations, exactPsqlQueries: true, inertEnableDisableAndRejections: true,
|
||||
linuxSocketSourceAccepted: false, realMcpExecutionClaimed: false }))
|
||||
} finally {
|
||||
try {
|
||||
await pool?.end()
|
||||
} finally {
|
||||
if (started) await embedded.stop()
|
||||
if (listener.listening) await new Promise(done => listener.close(done))
|
||||
await rm(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
|
||||
}
|
||||
}
|
||||
@@ -94,8 +94,11 @@ const operatorPeerMetadata = process.argv.includes('--operator-peer-metadata')
|
||||
if (operatorPeerMetadata && process.argv.slice(2).some(arg => arg !== '--operator-peer-metadata')) throw new Error('Operator peer metadata requires a separate disposable cluster without restore')
|
||||
const externalPeerMetadata = process.argv.includes('--external-peer-metadata')
|
||||
if (externalPeerMetadata && process.argv.slice(2).some(arg => arg !== '--external-peer-metadata')) throw new Error('Guarded external peer metadata requires a separate disposable cluster without restore')
|
||||
const mcpOperationOrigin = process.argv.includes('--mcp-operation-origin')
|
||||
if (mcpOperationOrigin && process.argv.slice(2).some(arg => arg !== '--mcp-operation-origin')) throw new Error('MCP operation root requires a separate disposable cluster without restore')
|
||||
const pollOperationRetirement = process.argv.includes('--poll-operation-retirement')
|
||||
const ordinaryOperationRetirement = process.argv.includes('--ordinary-operation-retirement')
|
||||
const probeOperationRetirement = process.argv.includes('--probe-operation-retirement')
|
||||
const mcpOperationOrigin = process.argv.includes('--mcp-operation-origin') || pollOperationRetirement || ordinaryOperationRetirement || probeOperationRetirement
|
||||
if (mcpOperationOrigin && process.argv.slice(2).some(arg => arg !== (probeOperationRetirement ? '--probe-operation-retirement' : ordinaryOperationRetirement ? '--ordinary-operation-retirement' : pollOperationRetirement ? '--poll-operation-retirement' : '--mcp-operation-origin'))) throw new Error('MCP operation root requires a separate disposable cluster without restore')
|
||||
const mcpSpeedOriginal = process.argv.includes('--mcp-speed-original')
|
||||
if (mcpSpeedOriginal && process.argv.slice(2).some(arg => arg !== '--mcp-speed-original')) throw new Error('MCP speed component requires a separate disposable cluster without restore')
|
||||
const routerConfigurationGuard = process.argv.includes('--router-configuration-guard')
|
||||
@@ -243,13 +246,13 @@ try {
|
||||
if (regressionOnly) console.log('Selected regression only: primary integration and terminal restore are not run')
|
||||
if (profilesOnly) console.log('Profiles-only regression matrix: primary integration NOT RUN; terminal restore NOT RUN; full acceptance is not claimed')
|
||||
const result = regressionOnly || profilesOnly ? 0 : await new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, pollMemory ? ['--max-old-space-size=192', '--max-semi-space-size=16', fileURLToPath(new URL('../dist-mcp-memory/services/server-poll-memory.owned.test.js', import.meta.url))] : ['--import', 'tsx', principalSecuritySource ? 'src/modules/mcp/principal-security-source.owned.test.ts' : pollJournalSnapshot ? 'src/services/server-poll-journal-snapshot.owned.test.ts' : pollTerminalArchive ? 'src/services/server-poll-terminal-archive.owned.test.ts' : pollArchiveSchema ? 'src/db/server-poll-terminal-archives.sql.test.ts' : telemetrySettingsConfiguration ? 'src/modules/mcp/telemetry-settings-configuration.owned.test.ts' : routingReservationSchema ? 'src/db/routing-reservation-schema.owned.test.ts' : routingForwardRounds ? 'src/modules/mcp/routing-network-forward-rounds.owned.test.ts' : flowActiveServices ? 'src/services/traffic-flow-active-services.owned.test.ts' : flowActiveRebuild ? 'src/services/traffic-flow-active-rebuild.owned.test.ts' : flowQuality ? 'src/services/traffic-flow-quality.owned.test.ts' : flowWorkerIpc ? 'src/services/traffic-flow-worker-ipc.owned.test.ts' : manualUptime ? 'src/modules/mcp/manual-uptime-http.test.ts' : manualPingGroup ? 'src/modules/mcp/manual-ping-group-http.test.ts' : manualTraffic ? 'src/modules/mcp/manual-traffic-http.test.ts' : manualApiPing ? 'src/modules/mcp/manual-api-ping-http.test.ts' : simpleSpeedChr ? 'src/modules/mcp/simple-speed-chr.test.ts' : simpleSpeed ? 'src/modules/mcp/simple-speed-http.test.ts' : externalPeerMetadata ? 'src/services/bandwidth-external-peer-metadata.sql.test.ts' : operatorPeerMetadata ? 'src/services/bandwidth-operator-metadata.sql.test.ts' : publicRegistryMetadata ? 'src/services/bandwidth-public-registry.sql.test.ts' : operatorDockerSql ? 'src/services/bandwidth-operator-docker-stop.sql.test.ts' : mcpOperationOrigin ? 'src/modules/mcp/operation-execution-origin.sql.test.ts' : mcpSpeedOriginal ? 'src/modules/mcp/speed-bandwidth-original.sql.test.ts' : routerConfigurationGuard ? 'src/services/router-configuration-guard.sql.test.ts' : scheduledSpeedOriginal ? 'src/services/bandwidth-scheduled-speed-original.sql.test.ts' : bandwidthSpeedWorkflow ? 'src/services/bandwidth-speed-workflow.sql.test.ts' : restSpeedOriginal ? 'src/services/bandwidth-rest-speed-original.sql.test.ts' : operationLocks ? 'src/services/operation-config-lock-lease.sql.test.ts' : serviceExecutionOrigin ? 'src/modules/mcp/service-execution-origin.sql.test.ts' : bandwidthNodeSource ? 'src/services/bandwidth-node-source.sql.test.ts' : bandwidthExternalRegistry ? 'src/services/bandwidth-external-node-registry.sql.test.ts' : bandwidthNodeRegistry ? 'src/services/bandwidth-node-registry.sql.test.ts' : bandwidthControlSource ? 'src/services/bandwidth-control-source.sql.test.ts' : bandwidthAdmission ? 'src/services/bandwidth-admission.sql.test.ts' : serverPollStorageFresh ? 'src/services/server-poll-storage-fresh.sql.test.ts' : serverPollStorage ? 'src/services/server-poll-storage.sql.test.ts' : probeStorageFresh ? 'src/services/server-probe-storage-fresh.sql.test.ts' : probeLegacyOrphans ? 'src/services/server-probe-legacy-orphans.sql.test.ts' : diagnosticCancelHost ? 'src/modules/mcp/diagnostic-cancellation-storage.sql.test.ts' : ipsecExportHttp ? 'src/modules/mcp/ipsec-existing-certificate-export-http.sql.test.ts' : ipsecExportJournal ? 'src/services/ipsec-existing-certificate-export-rest-journal.sql.test.ts' : migrationDataInventory ? 'src/db/migration-data-inventory.sql.test.ts' : probeStorageHistory ? 'src/services/server-probe-storage-history.sql.test.ts' : probeStorageJournal ? 'src/services/server-probe-storage-journal.sql.test.ts' : controlledSchemaReference ? 'src/db/controlled-schema-reference.sql.test.ts' : controlledMigrations ? 'src/db/controlled-migrations.sql.test.ts' : databaseSchemaControl ? 'src/db/database-schema-control.sql.test.ts' : migrationSchemaInventory ? 'src/db/migration-schema-inventory.sql.test.ts' : routingNetworkNative ? 'src/modules/mcp/routing-network-native.sql.test.ts' : routingReservationRuntime ? 'src/modules/mcp/routing-reservation-runtime.sql.test.ts' : routingReservations ? 'src/modules/mcp/routing-reservation.sql.test.ts' : collectorPartitionWriters ? 'src/services/collector-partition-writes.sql.test.ts' : serverPollDurable ? 'src/services/server-poll-durable-session.sql.test.ts' : routingRecoveryAuthority ? 'src/modules/mcp/routing-recovery-authority.sql.test.ts' : legacyFlowRebuild ? 'src/test/legacy-flow-rebuild-sql.ts' : probeJournal ? 'src/services/server-probe-journal.sql.test.ts' : flowHostKeys ? 'src/modules/mcp/flow-host-keys-http.test.ts' : flowRebuildMcp ? 'src/modules/mcp/flow-facts-rebuild-http.test.ts' : flowRebuild ? 'src/test/flow-facts-rebuild-sql.ts' : flowWrites ? 'src/services/traffic-flow-write-transaction.pg.test.ts' : nativeConservation ? 'src/modules/mcp/native-conservation-sql.test.ts' : nativeCreateId ? 'src/modules/mcp/native-create-id-http.test.ts' : ipsecAdapters ? "src/mLine truncated
|
||||
const child = spawn(process.execPath, pollMemory ? ['--max-old-space-size=192', '--max-semi-space-size=16', fileURLToPath(new URL('../dist-mcp-memory/services/server-poll-memory.owned.test.js', import.meta.url))] : ['--import', 'tsx', principalSecuritySource ? 'src/modules/mcp/principal-security-source.owned.test.ts' : pollJournalSnapshot ? 'src/services/server-poll-journal-snapshot.owned.test.ts' : pollTerminalArchive ? 'src/services/server-poll-terminal-archive.owned.test.ts' : pollArchiveSchema ? 'src/db/server-poll-terminal-archives.sql.test.ts' : telemetrySettingsConfiguration ? 'src/modules/mcp/telemetry-settings-configuration.owned.test.ts' : routingReservationSchema ? 'src/db/routing-reservation-schema.owned.test.ts' : routingForwardRounds ? 'src/modules/mcp/routing-network-forward-rounds.owned.test.ts' : flowActiveServices ? 'src/services/traffic-flow-active-services.owned.test.ts' : flowActiveRebuild ? 'src/services/traffic-flow-active-rebuild.owned.test.ts' : flowQuality ? 'src/services/traffic-flow-quality.owned.test.ts' : flowWorkerIpc ? 'src/services/traffic-flow-worker-ipc.owned.test.ts' : manualUptime ? 'src/modules/mcp/manual-uptime-http.test.ts' : manualPingGroup ? 'src/modules/mcp/manual-ping-group-http.test.ts' : manualTraffic ? 'src/modules/mcp/manual-traffic-http.test.ts' : manualApiPing ? 'src/modules/mcp/manual-api-ping-http.test.ts' : simpleSpeedChr ? 'src/modules/mcp/simple-speed-chr.test.ts' : simpleSpeed ? 'src/modules/mcp/simple-speed-http.test.ts' : externalPeerMetadata ? 'src/services/bandwidth-external-peer-metadata.sql.test.ts' : operatorPeerMetadata ? 'src/services/bandwidth-operator-metadata.sql.test.ts' : publicRegistryMetadata ? 'src/services/bandwidth-public-registry.sql.test.ts' : operatorDockerSql ? 'src/services/bandwidth-operator-docker-stop.sql.test.ts' : probeOperationRetirement ? 'src/services/server-probe-operation-retirement.owned.test.ts' : ordinaryOperationRetirement ? 'src/modules/mcp/ordinary-operation-retirement.owned.test.ts' : pollOperationRetirement ? 'src/services/server-poll-operation-retirement.owned.test.ts' : mcpOperationOrigin ? 'src/modules/mcp/operation-execution-origin.sql.test.ts' : mcpSpeedOriginal ? 'src/modules/mcp/speed-bandwidth-original.sql.test.ts' : routerConfigurationGuard ? 'src/services/router-configuration-guard.sql.test.ts' : scheduledSpeedOriginal ? 'src/services/bandwidth-scheduled-speed-original.sql.test.ts' : bandwidthSpeedWorkflow ? 'src/services/bandwidth-speed-workflow.sql.test.ts' : restSpeedOriginal ? 'src/services/bandwidth-rest-speed-original.sql.test.ts' : operationLocks ? 'src/services/operation-config-lock-lease.sql.test.ts' : serviceExecutionOrigin ? 'src/modules/mcp/service-execution-origin.sql.test.ts' : bandwidthNodeSource ? 'src/services/bandwidth-node-source.sql.test.ts' : bandwidthExternalRegistry ? 'src/services/bandwidth-external-node-registry.sql.test.ts' : bandwidthNodeRegistry ? 'src/services/bandwidth-node-registry.sql.test.ts' : bandwidthControlSource ? 'src/services/bandwidth-control-source.sql.test.ts' : bandwidthAdmission ? 'src/services/bandwidth-admission.sql.test.ts' : serverPollStorageFresh ? 'src/services/server-poll-storage-fresh.sql.test.ts' : serverPollStorage ? 'src/services/server-poll-storage.sql.test.ts' : probeStorageFresh ? 'src/services/server-probe-storage-fresh.sql.test.ts' : probeLegacyOrphans ? 'src/services/server-probe-legacy-orphans.sql.test.ts' : diagnosticCancelHost ? 'src/modules/mcp/diagnostic-cancellation-storage.sql.test.ts' : ipsecExportHttp ? 'src/modules/mcp/ipsec-existing-certificate-export-http.sql.test.ts' : ipsecExportJournal ? 'src/services/ipsec-existing-certificate-export-rest-journal.sql.test.ts' : migrationDataInventory ? 'src/db/migration-data-inventory.sql.test.ts' : probeStorageHistory ? 'src/services/server-probe-storage-history.sql.test.ts' : probeStorageJournal ? 'src/services/server-probe-storage-journal.sql.test.ts' : controlledSchemaReference ? 'src/db/controlled-schema-reference.sql.test.ts' : controlledMigrations ? 'src/db/controlled-migrations.sql.test.ts' : databaseSchemaControl ? 'src/db/database-schema-control.sql.test.ts' : migrationSchemaInventory ? 'src/db/migration-schema-inventory.sql.test.ts' : routingNetworkNative ? 'src/modules/mcp/routing-network-native.sql.test.ts' : routingReservationRuntime ? 'src/modules/mcp/routing-reservation-runtime.sql.test.ts' : routingReservations ? 'src/modules/mcp/routing-reservation.sql.test.ts' : collectorPartitionWriters ? 'src/services/collector-partition-writes.sql.test.ts' : serverPollDurable ? 'src/services/server-poll-durable-session.sql.test.ts' : routingRecoveryAuthority ? 'src/modules/mcp/routing-recovery-authority.sql.test.ts' : legacyFlowRebuild ? 'src/test/legacy-flow-rebuild-sql.ts' : probeJournal ? 'src/services/server-probe-journal.sql.test.ts' : flowHostKeys ? 'src/modules/mcp/flow-host-keys-http.test.ts' : flowRebuildMcp ? 'src/modules/mcp/flow-facts-rebuild-http.test.ts' : flowRebuildLine truncated
|
||||
cwd: pollMemory ? directory : new URL('../', import.meta.url), stdio: 'inherit', env: { ...(pollMemory ? pollMemoryEnvironment : process.env),
|
||||
...(routingOriginalEffects ? { MMAPP_OWN_ROUTING_ORIGINAL_EFFECTS: 'true' } : {}),
|
||||
DATABASE_URL: `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres`,
|
||||
DATABASE_PATH: join(directory, 'no-legacy-database.sqlite'), SQLITE_IMPORT_FULL_HISTORY: 'false',
|
||||
MCP_STATE_DIR: join(directory, 'state'), AUTH_REQUIRED: 'true', AUTH_JWT_SECRET: 'mcp-integration-test-only', MMAPP_TEST_WG_EXPORT_ONLY: String(wireguardExport),
|
||||
MCP_PUBLIC_URL: 'http://localhost:8000/mcp', NODE_ENV: 'test', PATH: testPath, NODE_OPTIONS: pollMemory ? '' : process.env.NODE_OPTIONS, MMAPP_OWN_PRINCIPAL_SECURITY_SOURCE: String(principalSecuritySource), MMAPP_OWN_POLL_JOURNAL_SNAPSHOT: String(pollJournalSnapshot), MMAPP_OWN_POLL_MEMORY: String(pollMemory), MMAPP_OWN_POLL_BOOTSTRAP: String(pollMemoryBootstrap), MMAPP_OWN_POLL_COST_DIAGNOSTICS: String(pollMemoryCost), MMAPP_OWN_POLL_ARCHIVE_CLAIM_LOSS: String(pollArchiveClaimLoss), MMAPP_OWN_POLL_ARCHIVE_HISTORY: String(pollArchiveHistory), MMAPP_OWN_POLL_ARCHIVE_COLD: String(pollArchiveCold), MMAPP_OWN_POLL_ARCHIVE_LOST_ACK: String(pollArchiveLostAck), MMAPP_OWN_POLL_TERMINAL_ARCHIVE: String(pollTerminalArchive), MMAPP_OWN_POLL_ARCHIVE_SCHEMA: String(pollArchiveSchema), MMAPP_OWN_TELEMETRY_SETTINGS_CONFIGURATION: String(telemetrySettingsConfiguration), MMAPP_OWN_ROUTING_RESERVATION_SCHEMA: String(routingReservationSchema), MMAPP_OWN_ROUTING_FORWARD_ROUNDS: String(routingForwardRounds), MMAPP_OWN_FLOW_ACTIVE_SERVICES: String(flowActiveServices), MMAPP_OWN_FLOW_ACTIVE_REBUILD: String(flowActiveRebuild), MMAPP_OWN_FLOW_QUALITY: String(flowQuality), MMAPP_OWN_FLOW_WORKER_IPC: String(flowWorkerIpc), MMAPP_OWN_MANUAL_UPTIME: String(manualUptime), MMAPP_OWN_MANUAL_PING_GROUP: String(manualPingGroup), MMAPP_OWN_MANUAL_TRAFFIC: String(manualTraffic), MMAPP_OWN_MANUAL_API_PING: String(manualApiPing), MMAPP_OWN_SIMPLE_SPEED: String(simpleSpeed), MMAPP_OWN_SIMPLE_SPEED_CHR: String(simpleSpeedChr), MMAPP_OWN_DIAGNOSTIC_CANCEL_HOST: String(diagnosticCancelHost), MMAPP_OWN_OPERATOR_DOCKER_SQL: String(operatorDockerSql), MMAPP_OWN_PUBLIC_REGISTRY_METADATA: String(publicRegistryMetadata), MMAPP_OWN_OPERATOR_PEER_METADATA: String(operatorPeerMetadata), MMAPP_OWN_EXTERNAL_PEER_METADATA: String(externalPeerMetadata), MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpSpeedOriginal || bandwidthSpeedWorkflow || bandwidthExternalRegistry), MMAPP_OWN_BANDWIDTH_SPEED_WORKFLOW: String(bandwidthSpeedWorkflow), MMAPP_OWN_REST_SPEED_ORIGINAL: String(restSpeedOriginal), MMAPP_OWN_MCP_OPERATION_ORIGIN: String(mcpOperationOrigin), MMAPP_OWN_MCP_SPEED_ORIGINAL: String(mcpSpeedOriginal), MMAPP_OWN_SCHEDULED_SPEED_ORIGINAL: String(scheduledSpeedOriginal), MMAPP_TEST_ROUTER_GUARD_OWNED: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_CLUSTER_DISPOSABLE: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_ADMIN_URL: routerConfigurationGuard ? `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres` : '', MMAPP_OWN_BANDWIDTH_ADMISSION: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpOperationOrigin || mcpSpeedOriginal || scheduledSpeedOriginal || bandwidthSpeedWorkflow || restSpeedOriginal || bandwidthExternalRegistry || operationLocks || serviceExecutionOrigin || bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource), MMAPP_OWN_BANDWIDTH_RESTORE: String((bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource) && process.argv.includes('--restore')), MMAPP_OWN_IPSEC_EXPORT_HTTP: String(ipsecExportHttp), MMAPP_OWN_IPSEC_EXPORT_JOURNAL: String(ipsecExportJournal), MMAPP_OWN_SQL_DB: 'postgres', MMAPP_OWN_PROBE_LEGACY_ORPHANS: String(probeLegacyOrphans), MMAPP_OWN_PROBE_STORAGE_HISTORY: String(probeStorageHistory), MMAPP_OWN_PROBE_STORAGE_FRESH: String(probeStorageFresh), MMAPP_OWN_PROBE_STORAGE_JOURNAL: String(probeStorageJournal || probeStorageFresh), MMAPP_OWN_CONTROLLED_SCHEMA_REFERENCE: String(controlledSchemaReference), MMAPP_OWN_CONTROLLED_MIGRATIONS: String(controlledMigrations), MMAPP_OWN_DATABASE_SCHEMA_CONTROL: String(databaseSchemaControl), MMAPP_OWN_MIGRATION_DATA_INVENTORY: String(migrationDataInventory), MMAPP_OWN_MIGRATION_SCHEMA_INVENTORY: String(migrationSchemaInventory), MMAPP_OWN_ROUTING_NETWORK_NATIVE: String(routingNetworkNative), MMAPP_OWN_ROUTING_RESERVATIONS: String(routingReservations || routingReservationRuntime), MMAPP_OWN_COLLECTOR_PARTITION_WRITERS: String(collectorPartitionWriters), MMAPP_OWN_SERVER_POLL_STORAGE_FRESH: String(serverPollStorageFresh), MMAPP_OWN_SERVER_POLL_STORAGE: String(serverPollStorage), MMAPP_OWN_SERVER_POLL_DURABLE: String(serverPollDurable), MMAPP_OWN_ROUTING_RECOVERY_AUTHORITY: String(routingRecoveryAuthority), MMAPP_OWN_LEGACY_FLOW_REBUILD: String(legacyFlowRebuild), MMAPP_OWN_PROBE_JOURNAL: String(probeJournal), MMAPP_MODULE_COMPLETION_ONLY: String(moduleCompletion), MMAPP_MODULE_COMPLETION_TARGET: completionTarget, MMAPP_OWN_FLOW_HOST_KEYS: String(flowHostKeys), MMAPP_OWN_MCP_FLOW_REBUILD: String(flowRebuildMcp), MMAPP_OWN_FLOW_WRITES: String(flowWrites), MMAPP_OWN_FLOW_REBUILD: String(flowRebuild) },
|
||||
MCP_PUBLIC_URL: 'http://localhost:8000/mcp', NODE_ENV: 'test', PATH: testPath, NODE_OPTIONS: pollMemory ? '' : process.env.NODE_OPTIONS, MMAPP_OWN_PRINCIPAL_SECURITY_SOURCE: String(principalSecuritySource), MMAPP_OWN_POLL_JOURNAL_SNAPSHOT: String(pollJournalSnapshot), MMAPP_OWN_POLL_MEMORY: String(pollMemory), MMAPP_OWN_POLL_BOOTSTRAP: String(pollMemoryBootstrap), MMAPP_OWN_POLL_COST_DIAGNOSTICS: String(pollMemoryCost), MMAPP_OWN_POLL_ARCHIVE_CLAIM_LOSS: String(pollArchiveClaimLoss), MMAPP_OWN_POLL_ARCHIVE_HISTORY: String(pollArchiveHistory), MMAPP_OWN_POLL_ARCHIVE_COLD: String(pollArchiveCold), MMAPP_OWN_POLL_ARCHIVE_LOST_ACK: String(pollArchiveLostAck), MMAPP_OWN_POLL_TERMINAL_ARCHIVE: String(pollTerminalArchive), MMAPP_OWN_POLL_ARCHIVE_SCHEMA: String(pollArchiveSchema), MMAPP_OWN_TELEMETRY_SETTINGS_CONFIGURATION: String(telemetrySettingsConfiguration), MMAPP_OWN_ROUTING_RESERVATION_SCHEMA: String(routingReservationSchema), MMAPP_OWN_ROUTING_FORWARD_ROUNDS: String(routingForwardRounds), MMAPP_OWN_FLOW_ACTIVE_SERVICES: String(flowActiveServices), MMAPP_OWN_FLOW_ACTIVE_REBUILD: String(flowActiveRebuild), MMAPP_OWN_FLOW_QUALITY: String(flowQuality), MMAPP_OWN_FLOW_WORKER_IPC: String(flowWorkerIpc), MMAPP_OWN_MANUAL_UPTIME: String(manualUptime), MMAPP_OWN_MANUAL_PING_GROUP: String(manualPingGroup), MMAPP_OWN_MANUAL_TRAFFIC: String(manualTraffic), MMAPP_OWN_MANUAL_API_PING: String(manualApiPing), MMAPP_OWN_SIMPLE_SPEED: String(simpleSpeed), MMAPP_OWN_SIMPLE_SPEED_CHR: String(simpleSpeedChr), MMAPP_OWN_DIAGNOSTIC_CANCEL_HOST: String(diagnosticCancelHost), MMAPP_OWN_OPERATOR_DOCKER_SQL: String(operatorDockerSql), MMAPP_OWN_PUBLIC_REGISTRY_METADATA: String(publicRegistryMetadata), MMAPP_OWN_OPERATOR_PEER_METADATA: String(operatorPeerMetadata), MMAPP_OWN_EXTERNAL_PEER_METADATA: String(externalPeerMetadata), MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpSpeedOriginal || bandwidthSpeedWorkflow || bandwidthExternalRegistry), MMAPP_OWN_BANDWIDTH_SPEED_WORKFLOW: String(bandwidthSpeedWorkflow), MMAPP_OWN_REST_SPEED_ORIGINAL: String(restSpeedOriginal), MMAPP_OWN_MCP_OPERATION_ORIGIN: String(mcpOperationOrigin), MMAPP_OWN_PROBE_OPERATION_RETIREMENT: String(probeOperationRetirement), MMAPP_OWN_MCP_SPEED_ORIGINAL: String(mcpSpeedOriginal), MMAPP_OWN_SCHEDULED_SPEED_ORIGINAL: String(scheduledSpeedOriginal), MMAPP_TEST_ROUTER_GUARD_OWNED: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_CLUSTER_DISPOSABLE: String(routerConfigurationGuard), MMAPP_TEST_ROUTER_GUARD_ADMIN_URL: routerConfigurationGuard ? `postgresql://mcp_test:${password}@127.0.0.1:${port}/postgres` : '', MMAPP_OWN_BANDWIDTH_ADMISSION: String(externalPeerMetadata || operatorPeerMetadata || publicRegistryMetadata || operatorDockerSql || mcpOperationOrigin || mcpSpeedOriginal || scheduledSpeedOriginal || bandwidthSpeedWorkflow || restSpeedOriginal || bandwidthExternalRegistry || operationLocks || serviceExecutionOrigin || bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource), MMAPP_OWN_BANDWIDTH_RESTORE: String((bandwidthAdmission || bandwidthControlSource || bandwidthNodeRegistry || bandwidthNodeSource) && process.argv.includes('--restore')), MMAPP_OWN_IPSEC_EXPORT_HTTP: String(ipsecExportHttp), MMAPP_OWN_IPSEC_EXPORT_JOURNAL: String(ipsecExportJournal), MMAPP_OWN_SQL_DB: 'postgres', MMAPP_OWN_PROBE_LEGACY_ORPHANS: String(probeLegacyOrphans), MMAPP_OWN_PROBE_STORAGE_HISTORY: String(probeStorageHistory), MMAPP_OWN_PROBE_STORAGE_FRESH: String(probeStorageFresh), MMAPP_OWN_PROBE_STORAGE_JOURNAL: String(probeStorageJournal || probeStorageFresh), MMAPP_OWN_CONTROLLED_SCHEMA_REFERENCE: String(controlledSchemaReference), MMAPP_OWN_CONTROLLED_MIGRATIONS: String(controlledMigrations), MMAPP_OWN_DATABASE_SCHEMA_CONTROL: String(databaseSchemaControl), MMAPP_OWN_MIGRATION_DATA_INVENTORY: String(migrationDataInventory), MMAPP_OWN_MIGRATION_SCHEMA_INVENTORY: String(migrationSchemaInventory), MMAPP_OWN_ROUTING_NETWORK_NATIVE: String(routingNetworkNative), MMAPP_OWN_ROUTING_RESERVATIONS: String(routingReservations || routingReservationRuntime), MMAPP_OWN_COLLECTOR_PARTITION_WRITERS: String(collectorPartitionWriters), MMAPP_OWN_SERVER_POLL_STORAGE_FRESH: String(serverPollStorageFresh), MMAPP_OWN_SERVER_POLL_STORAGE: String(serverPollStorage), MMAPP_OWN_SERVER_POLL_DURABLE: String(serverPollDurable), MMAPP_OWN_ROUTING_RECOVERY_AUTHORITY: String(routingRecoveryAuthority), MMAPP_OWN_LEGACY_FLOW_REBUILD: String(legacyFlowRebuild), MMAPP_OWN_PROBE_JOURNAL: String(probeJournal), MMAPP_MODULE_COMPLETION_ONLY: String(moduleCompletion), MMAPP_MODULE_COMPLETION_TARGET: completionTarget, MMAPP_OWN_FLOW_HOST_KEYS: String(flowHostKeys), MMAPP_OWN_MCP_FLOW_REBUILD: String(flowRebuildMcp), MMAPP_OWN_FLOW_WRITES: String(flowWrites), MMAPP_OWN_FLOW_REBUILD: String(flowRebuild) },
|
||||
})
|
||||
child.once('error', reject); child.once('exit', code => resolve(code ?? 1))
|
||||
})
|
||||
@@ -286,6 +289,9 @@ try {
|
||||
else if (result === 0 && operatorPeerMetadata) console.log('Operator peer metadata: actual managed endpoint SQL and genuine localhost license origin; synthetic Q, external metadata and full operator window remain open')
|
||||
else if (result === 0 && publicRegistryMetadata) console.log('Public registry metadata: actual owned PostgreSQL and same physical lease; no secret/Q/native/write authority or backend-window acceptance')
|
||||
else if (result === 0 && operatorDockerSql) console.log('Operator Docker SQL: actual owned PostgreSQL with mocked Docker HTTP only; not a real Docker/backend-window acceptance')
|
||||
else if (result === 0 && ordinaryOperationRetirement) console.log('Ordinary operation retirement: owned actual logs-settings apply and worker')
|
||||
else if (result === 0 && pollOperationRetirement) console.log('Poll operation retirement: owned authenticated historical fixtures and current reader/GC')
|
||||
else if (result === 0 && probeOperationRetirement) console.log('Diagnostic retirement: actual owned worker tick, PostgreSQL and product read/replay; no live scheduling')
|
||||
else if (result === 0 && mcpOperationOrigin) console.log('MCP operation root: owned PostgreSQL ordinary runner provenance; no restore/recovery, global writer activation, CHR or deployment')
|
||||
else if (result === 0 && mcpSpeedOriginal) console.log('MCP speed: genuine trusted service context with real connection/grant/SQL, host receipt and localhost native; synthetic Q/mapping; not a public MCP runner/action acceptance')
|
||||
else if (result === 0 && routerConfigurationGuard) console.log('Router configuration guard: dedicated owned databases/roles and localhost native fixture; no application migration, production activation or real CHR')
|
||||
|
||||
@@ -12,10 +12,10 @@ const reviewed = (source: string, extra: Partial<MigrationReview> = {}) => {
|
||||
const bytes = Buffer.from(source), file = "9000_fixture.sql"
|
||||
return () => analyzeMigration(file, bytes, { file, sha256: createHash("sha256").update(bytes).digest("hex"), transaction: "owner-transaction", selfMarker: false, opaqueBodies: 0, bodyReview: [], effects: ["Explicit inert fixture review"], ...extra })
|
||||
}
|
||||
await test("all44 exact file pins and every top-level schema operation are inventoried", () => {
|
||||
assert.equal(inventory.length, 44); assert.equal(MIGRATION_MANIFEST.length, 44)
|
||||
await test("all 53 exact file pins and every top-level schema operation are inventoried", () => {
|
||||
assert.equal(inventory.length, 53); assert.equal(MIGRATION_MANIFEST.length, 53)
|
||||
assert.equal(inventory.filter(file => file.transaction === "owner-transaction").length, 17)
|
||||
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 27)
|
||||
assert.equal(inventory.filter(file => file.transaction === "self-managed-single-transaction").length, 36)
|
||||
assert(inventory.every(file => file.operations.length === file.statementCount && file.effects.length > 0))
|
||||
let tables = new Set<string>()
|
||||
for (const file of inventory) {
|
||||
@@ -32,8 +32,8 @@ await test("DO BEGIN and stored function BEGIN are never transaction boundaries"
|
||||
assert.equal(endpoint.topLevelTransactionControls.length, 2); assert.equal(endpoint.opaqueBodies.length, 2)
|
||||
assert.deepEqual(endpoint.opaqueBodies.map(body => body.kind), ["stored-function-body", "executed-do-body"])
|
||||
})
|
||||
await test("0035 through0044 have exact atomic file-owned terminal markers", () => {
|
||||
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges"])
|
||||
await test("0035 through 0053 have exact atomic file-owned terminal markers", () => {
|
||||
assert.deepEqual(inventory.filter(file => file.selfMarker).map(file => file.id), ["0035_server_probe_journal", "0036_diagnostic_cancellations", "0037_server_probe_storage_v2", "0038_server_poll_journal_and_writer_guards", "0039_server_poll_storage", "0040_mcp_routing_reservations", "0041_server_poll_terminal_archives", "0042_server_poll_settings_epoch", "0043_mcp_routing_forward_charges", "0044_mcp_routing_manual_inverse_charges", "0045_mcp_operation_replay_tombstones", "0046_server_poll_operation_retirement", "0047_server_poll_retirement_claims", "0048_mcp_ordinary_retirement_claims", "0049_mcp_ordinary_telemetry_retirement", "0050_server_probe_retired_sources", "0051_server_probe_retirement_claims", "0052_server_probe_retired_gc", "0053_server_probe_terminal_proofs"])
|
||||
for (const file of inventory.filter(file => file.selfMarker)) { assert.equal(file.selfMarker!.atomicWithAllFileEffects, true); assert.equal(file.selfMarker!.statement, file.statementCount - 2); assert.equal(file.currentOuterMarkerGap, false) }
|
||||
assert.equal(inventory.filter(file => file.currentOuterMarkerGap).length, 34)
|
||||
})
|
||||
|
||||
@@ -702,6 +702,132 @@ const classifiedMigrationPins: readonly MigrationReview[] = [
|
||||
"Adds one one-unit immutable manual inverse charge table, two charge indexes, one manual-history page index, and four triggers. Existing43 bytes, reserved2 and generic debit ownership remain unchanged.",
|
||||
"No inferred/backfilled identity, legacy timestamp debit, permission change, source enrollment, public controller activation or READY. Full RT3/all-writer/family-deletion proof remains open."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0045_mcp_operation_replay_tombstones.sql",
|
||||
"sha256": "8edf5b074c7eef49dad7835c44ac2539df4347f19eb9dd9b2aa7005bafaadbc0",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 0,
|
||||
"bodyReview": [],
|
||||
"effects": [
|
||||
"Adds one empty replay identity table with plan/operation uniqueness and connection-scoped idempotency digest uniqueness.",
|
||||
"Table creation refuses an unmarked collision; the exact marker commits with the schema. No backfill, operation deletion, or worker activation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0046_server_poll_operation_retirement.sql",
|
||||
"sha256": "75d7956ea8134b01bcc88da083b77eb52f6e1ad70ad854918ed3380cb0e8cf88",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 4,
|
||||
"bodyReview": [
|
||||
"Archive guard permits only fixed receipt-scoped V1 to V2 updates with matching approved terminal operation, tombstone and SQL 180-day cutoff.",
|
||||
"Deferred guard independently requires the tombstone, physically absent operation and plan, and exact ledger accounting.",
|
||||
"Admission guard retains existing paths and applies exact V2 archive-byte delta to both owners without changing row charges or lifetime fields.",
|
||||
"Existing GC retains its authority/activity cutoff and admits retired identity only with exact poll tombstone and compatible V2 archive when present."
|
||||
],
|
||||
"effects": [
|
||||
"Extends the same archive table to a bounded version 2, changes three guards and adds one deferred retirement guard.",
|
||||
"No automatic conversion, worker activation, operation deletion or retention-window adjustment in migration."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0047_server_poll_retirement_claims.sql",
|
||||
"sha256": "8c01ab14d241f715af021e8a7d7f5bb273b450b32f566140359260282524e01d",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 2,
|
||||
"bodyReview": [
|
||||
"Protects exact current-transaction V1 claim insertion and permits deletion only as the owning admission is cascaded by GC.",
|
||||
"Requires exact receipt, V1 archive proof and claim token already committed in another transaction before V2 update."
|
||||
],
|
||||
"effects": [
|
||||
"Adds one durable one-shot poll retirement claim per admission, cascading only with that admission's existing guarded GC.",
|
||||
"No migration-time conversion, backfill, dispatch, or quota change."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0048_mcp_ordinary_retirement_claims.sql",
|
||||
"sha256": "5bc3598c9f5d09611e98d05345accda6d39b199a583bba789b97a24140fafbfd",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 1,
|
||||
"bodyReview": [
|
||||
"Makes ordinary claims immutable and allows insert only for the exact linked settled and aged logs.settings.update v1 operation."
|
||||
],
|
||||
"effects": [
|
||||
"Adds one durable ordinary-operation claim with an authenticated source digest. No migration-time data purge or backfill."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0049_mcp_ordinary_telemetry_retirement.sql",
|
||||
"sha256": "68993e924d20db6b08973846a0da367fe89fe2948c414c810e9b171dbbae6761",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 1,
|
||||
"bodyReview": [
|
||||
"Extends the same immutable ordinary claim guard to four explicitly versioned settled telemetry settings families."
|
||||
],
|
||||
"effects": [
|
||||
"Replaces only the ordinary claim guard function. No backfill, data purge, or changes to poll or diagnostic storage."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0050_server_probe_retired_sources.sql",
|
||||
"sha256": "0468ac7e53638b532557273f55d065ff4cde092d8b08073f5860bab35263aead",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 1,
|
||||
"bodyReview": [
|
||||
"Refuses update, delete, and truncate of a signed retired diagnostic source pending a separately reviewed lifecycle."
|
||||
],
|
||||
"effects": [
|
||||
"Creates empty bounded storage linking exact diagnostic admission and replay tombstone IDs. No backfill, retirement, or GC activation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0051_server_probe_retirement_claims.sql",
|
||||
"sha256": "b38876b5da44cf14e20a9df0baaa233ea4de81e471b38dddd370e149c8116243",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 2,
|
||||
"bodyReview": [
|
||||
"Restricts diagnostic claims to their original settled 180-day source and makes each claim permanent.",
|
||||
"Restricts archival insertion to the exact committed claim and current original source and tombstone identity."
|
||||
],
|
||||
"effects": [
|
||||
"Creates empty diagnostic retirement claims and guards the archive insert. No conversion, backfill, or GC activation."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0052_server_probe_retired_gc.sql",
|
||||
"sha256": "ea56a1f1bdabfce606bdfec56b6a7b31d45b0187fbea4c900bc656ae320f2fdd",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 3,
|
||||
"bodyReview": [
|
||||
"Validates the exact retired diagnostic identity and the later of current and persisted retention plus the original safety window.",
|
||||
"Only permits archive removal during an exact guarded admission cascade after journal removal.",
|
||||
"Adds the exact retired GC branch while retaining the live admission eligibility predicate."
|
||||
],
|
||||
"effects": [
|
||||
"Changes only the retired source admission FK to guarded cascade and adds an explicit exact GC path. No collection runs during migration."
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "0053_server_probe_terminal_proofs.sql",
|
||||
"sha256": "759ec8c349fcb24c43af410ad6ade714432678b3a021c94bd982080d6a8986be",
|
||||
"transaction": "self-managed-single-transaction",
|
||||
"selfMarker": true,
|
||||
"opaqueBodies": 2,
|
||||
"bodyReview": [
|
||||
"Pins terminal proof to the current retained original on insert and an exact committed retirement claim on delete.",
|
||||
"Extends claims only to the identical signed terminal proof when the original diagnostic admission is gone."
|
||||
],
|
||||
"effects": [
|
||||
"Creates empty bounded private terminal proof storage. Existing payload retention, scheduler, and archive GC remain unchanged."
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -724,7 +850,7 @@ export const migrationManifestDigest = () => hash(JSON.stringify(MIGRATION_MANIF
|
||||
// resetting the accumulated public table set at the reviewed destructive0002.
|
||||
// This is a necessary presence check, not a substitute for exact live column,
|
||||
// constraint/index/function/trigger comparison by the control protocol.
|
||||
export const CURRENT_REQUIRED_TABLES: readonly string[] = Object.freeze(["acme_settings","alert_engine_confirm_pending","alert_engine_cursor","alert_engine_prev_live","alert_engine_state","alert_groups","alert_history","alert_outbox","alert_rule_conditions","alert_rule_targets","alert_rules","alert_telegram_settings","app_users","backup_entries","backup_schedule_settings","backup_storage_settings","certificate_issue_jobs","certificate_renew_settings","config_revisions","data_migration","events","evobgp_settings","filter_rules","flow_asn_meta","flow_buckets","flow_daily_dims","flow_daily_facts","flow_hour_facts","flow_ip_meta","flow_minute_dims","flow_minute_stats","flow_settings_intents","flow_write_receipts","geoip_settings","internet_path_settings","internet_path_snapshots","mcp_artifacts","mcp_audit","mcp_budget_buckets","mcp_connections","mcp_credentials","mcp_diagnostic_cancellations","mcp_drift","mcp_grants","mcp_management_channels","mcp_oauth_clients","mcp_oauth_requests","mcp_operation_checkpoints","mcp_operation_steps","mcp_operations","mcp_owner_revocations","mcp_path_checks","mcp_plans","mcp_portal_events","mcp_protections","mcp_read_leases","mcp_recovery_intents","mcp_requests","mcp_routing_forward_charges","mcp_routing_manual_inverse_charges","mcp_routing_observations","mcp_routing_profiles","mcp_routing_reservation_slots","mcp_routing_reservations","mcp_scoped_recovery_intents","mcp_secrets","mcp_settings","mcp_unauthorized_buckets","recursive_routes","router_log_collection_state","router_log_entries","router_log_settings","scheduler_runs","schema_migrations","server_poll_acks","server_poll_archives","server_poll_executions","server_poll_intents","server_poll_quarantines","server_poll_receipts","server_poll_storage_admissions","server_poll_storage_usage","server_probe_execution_payloads","server_probe_execution_quarantines","server_probe_execution_stages","server_probe_executions","server_probe_storage_admissions","server_probe_storage_usage","server_snapshots","servers","servers_api_ping_settings","servers_rest_ping_samples","traffic_collection_state","traffic_flow_settings","traffic_samples","traffic_settings","uptime_probe_samples","uptime_probes","uptime_resource_samples","uptime_settings","uptime_speed_probes","uptime_speed_test_runs","user_interface_bindings","vrf_production_tickets"])
|
||||
export const CURRENT_REQUIRED_TABLES: readonly string[] = Object.freeze(["acme_settings","alert_engine_confirm_pending","alert_engine_cursor","alert_engine_prev_live","alert_engine_state","alert_groups","alert_history","alert_outbox","alert_rule_conditions","alert_rule_targets","alert_rules","alert_telegram_settings","app_users","backup_entries","backup_schedule_settings","backup_storage_settings","certificate_issue_jobs","certificate_renew_settings","config_revisions","data_migration","events","evobgp_settings","filter_rules","flow_asn_meta","flow_buckets","flow_daily_dims","flow_daily_facts","flow_hour_facts","flow_ip_meta","flow_minute_dims","flow_minute_stats","flow_settings_intents","flow_write_receipts","geoip_settings","internet_path_settings","internet_path_snapshots","mcp_artifacts","mcp_audit","mcp_budget_buckets","mcp_connections","mcp_credentials","mcp_diagnostic_cancellations","mcp_drift","mcp_grants","mcp_management_channels","mcp_oauth_clients","mcp_oauth_requests","mcp_operation_checkpoints","mcp_operation_replay_tombstones","mcp_operation_steps","mcp_operations","mcp_ordinary_retirement_claims","mcp_owner_revocations","mcp_path_checks","mcp_plans","mcp_portal_events","mcp_protections","mcp_read_leases","mcp_recovery_intents","mcp_requests","mcp_routing_forward_charges","mcp_routing_manual_inverse_charges","mcp_routing_observations","mcp_routing_profiles","mcp_routing_reservation_slots","mcp_routing_reservations","mcp_scoped_recovery_intents","mcp_secrets","mcp_settings","mcp_unauthorized_buckets","recursive_routes","router_log_collection_state","router_log_entries","router_log_settings","scheduler_runs","schema_migrations","server_poll_acks","server_poll_archives","server_poll_executions","server_poll_intents","server_poll_quarantines","server_poll_receipts","server_poll_retirement_claims","server_poll_storage_admissions","server_poll_storage_usage","server_probe_execution_payloads","server_probe_execution_quarantines","server_probe_execution_stages","server_probe_executions","server_probe_retired_sources","server_probe_retirement_claims","server_probe_storage_admissions","server_probe_storage_usage","server_probe_terminal_proofs","server_snapshots","servers","servers_api_ping_settings","servers_rest_ping_samples","traffic_collection_state","traffic_flow_settings","traffic_samples","traffic_settings","uptime_probe_samples","uptime_probes","uptime_resource_samples","uptime_settings","uptime_speed_probes","uptime_speed_test_runs","user_interface_bindings","vrf_production_tickets"])
|
||||
|
||||
export interface LegacyBaselineEvidence {
|
||||
inventoryComplete: boolean; databaseGeneration: string; unresolvedAttemptIds: readonly string[]
|
||||
|
||||
@@ -14,6 +14,7 @@ import { logFiltersSchema, logSettingsPatchSchema } from "@mmapp/contracts"
|
||||
import { appUserCreateSchema, appUserUpdateSchema, appUserRoleSchema, sectionPermSchema, serverPermSchema, userBindingCreateSchema } from "@mmapp/contracts/users"
|
||||
import type { McpOperationDescriptor } from "@mmapp/contracts/mcp"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { assertOperationReplayScope, findOperationReplayTombstone, operationHistoryExpired } from "./operation-replay-tombstones.js"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { getServerReadById } from "../servers/service/servers-service.js"
|
||||
import { listUsers, getUserById, listInterfaceCatalog, getInterfaceCatalogState } from "../users/service/users-service.js"
|
||||
@@ -473,7 +474,12 @@ add("changes.cancel", "mm_change_cancel", "Отменить свой незап
|
||||
add("operations.read", "mm_operation_get", "Результат своей операции", "mcp", null, "read", "connection", z.object({ operationId: z.string().uuid() }).strict(), {
|
||||
read: async (input, context) => {
|
||||
const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND connection_id=$2", [input.operationId, context.connection.id])
|
||||
if (!operation) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
if (!operation) {
|
||||
const tombstone = await findOperationReplayTombstone(pool, context.connection.id, String(input.operationId))
|
||||
if (!tombstone) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
assertOperationReplayScope(tombstone, context.connection)
|
||||
operationHistoryExpired()
|
||||
}
|
||||
await assertPlanReadScope(await (await import("./runtime.js")).loadPlan(operation.plan_id), context)
|
||||
const { rows: steps } = await pool.query("SELECT * FROM mcp_operation_steps WHERE operation_id=$1 ORDER BY step_index", [input.operationId])
|
||||
return { ...operation, steps }
|
||||
|
||||
@@ -115,3 +115,9 @@ export const createDiagnosticCancellationSourceForTests = sourceLoader
|
||||
/** Historical authentication only; confers no execution/cancellation grant. */
|
||||
export const readServerProbeExecutionSource = readSourceLoader(productionCrypto)
|
||||
export const createReadServerProbeExecutionSourceForTests = readSourceLoader
|
||||
/** Exact previously locked source projection for private history conversion. */
|
||||
export function authenticateServerProbeExecutionSourceRow(row: Record<string, unknown>, operationId: string,
|
||||
owner: Readonly<DiagnosticCancellationOwner>) {
|
||||
checkLookup(operationId, owner)
|
||||
return validateSourceRow(row, operationId, owner, productionCrypto)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import type { McpAction, ActionContext } from "./action-types.js"
|
||||
import { registerGlobalReconciliationActionsForTests } from "./global-reconciliation-actions.js"
|
||||
|
||||
type Register = Parameters<typeof registerGlobalReconciliationActionsForTests>[0]
|
||||
type Dependencies = Parameters<typeof registerGlobalReconciliationActionsForTests>[1]
|
||||
|
||||
function fixture() {
|
||||
const connectionId = randomUUID(), operationId = randomUUID(), ownerId = "current-retired-owner"
|
||||
const context = { principal: { connectionId }, connection: { id: connectionId, owner_id: ownerId, owner_permissions: ["mm:servers:write"],
|
||||
policy: { permissions: ["mm:servers:write"], scope: { allowGlobal: true, operationIds: ["servers.poll", "operations.global.reconcile"], serverIds: [1, 2] } } } } as ActionContext
|
||||
let action!: Pick<McpAction, "target" | "read">
|
||||
let history: { affectedServerIds: number[] } | undefined = { affectedServerIds: [1, 2] }
|
||||
let revoke = false, retiredCalls = 0, live = false
|
||||
const deps = {
|
||||
connection: async () => {
|
||||
if (revoke) throw Error("revoked-current-grant")
|
||||
return structuredClone(context.connection)
|
||||
},
|
||||
query: async (sql: string) => {
|
||||
if (sql.startsWith("SELECT id FROM servers")) return [{ id: 1 }]
|
||||
if (live) return [{ id: operationId, state: "applied", operation_id: "servers.poll", evidence: { sealed: "live" } }]
|
||||
return []
|
||||
},
|
||||
operation: (id: string) => ({ id, scope: "global", requiredPermissions: ["mm:servers:write"] }),
|
||||
service: () => ({ reconcile: async () => ({ matchesDesiredState: true, matchesOriginalState: false }) }),
|
||||
decrypt: async () => JSON.stringify({ target: { scope: "global" }, objects: [{ serverId: 1 }], steps: [{ kind: "service", service: "servers.poll", input: {} }] }),
|
||||
retired: async (identity, authorize) => {
|
||||
retiredCalls++
|
||||
assert.deepEqual(identity, { operationId, connectionId, ownerId })
|
||||
await authorize()
|
||||
return history
|
||||
},
|
||||
} satisfies Omit<Dependencies, "operation" | "service"> & Partial<Record<"operation" | "service", unknown>>
|
||||
registerGlobalReconciliationActionsForTests(((...args: Parameters<Register>) => { action = args[8] }) as Register, deps as unknown as Dependencies)
|
||||
return { context, operationId, history: (value: typeof history) => { history = value }, live: () => { live = true },
|
||||
revoke: () => { revoke = true }, calls: () => retiredCalls, read: () => action.read!({ operationId }, context),
|
||||
target: () => action.target({ operationId }, context) }
|
||||
}
|
||||
|
||||
test("global action reconciles retired poll only under current ownership and returns no archived material", async () => {
|
||||
const f = fixture()
|
||||
assert.deepEqual(await f.target(), { scope: "global" })
|
||||
const result = await f.read() as { observedAt: string }
|
||||
const { observedAt, ...rest } = result
|
||||
assert(Number.isFinite(Date.parse(observedAt)))
|
||||
assert.deepEqual(rest, { operationId: f.operationId, state: "applied", automaticRetry: false,
|
||||
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }] })
|
||||
assert.equal(f.calls(), 2)
|
||||
})
|
||||
|
||||
test("global action refuses missing or inaccessible retired history", async () => {
|
||||
const f = fixture()
|
||||
f.history(undefined)
|
||||
await assert.rejects(f.read(), { code: "NOT_FOUND" })
|
||||
f.history({ affectedServerIds: [1, 2] })
|
||||
f.context.connection.policy.scope.serverIds = [1]
|
||||
await assert.rejects(f.read(), { code: "SERVER_DENIED" })
|
||||
f.context.connection.policy.scope.serverIds = [1, 2]
|
||||
f.context.connection.policy.scope.operationIds = ["operations.global.reconcile"]
|
||||
await assert.rejects(f.read(), { code: "PERMISSION_DENIED" })
|
||||
f.revoke()
|
||||
await assert.rejects(f.read(), /revoked-current-grant/)
|
||||
})
|
||||
|
||||
test("global action uses the live reconciliation path while original source exists", async () => {
|
||||
const f = fixture()
|
||||
f.live()
|
||||
await f.read()
|
||||
assert.equal(f.calls(), 0)
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { readRetiredServerPollHistory } from "../../services/server-poll-history-reader.js"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { effectiveMcpConnection } from "./store.js"
|
||||
import { decryptSecret, fingerprint, McpError } from "./security.js"
|
||||
@@ -7,40 +8,84 @@ import { findMcpOperation } from "./registry.js"
|
||||
import { findMcpService } from "./service-registry.js"
|
||||
import type { add } from "./actions.js"
|
||||
import type { ActionContext, ChangeEvidence } from "./action-types.js"
|
||||
import type { ConnectionRow } from "./store.js"
|
||||
|
||||
async function authorizedSource(operationId: string, context: ActionContext) {
|
||||
const production = {
|
||||
connection: effectiveMcpConnection,
|
||||
query: async (sql: string, values: unknown[] = []) => (await pool.query(sql, values)).rows,
|
||||
operation: findMcpOperation,
|
||||
service: findMcpService,
|
||||
decrypt: decryptSecret,
|
||||
retired: readRetiredServerPollHistory,
|
||||
}
|
||||
type Dependencies = typeof production
|
||||
const notFound = () => new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
|
||||
async function currentAccess(context: ActionContext, deps: Dependencies) {
|
||||
context.signal?.throwIfAborted()
|
||||
const current = await effectiveMcpConnection(context.principal)
|
||||
const current = await deps.connection(context.principal)
|
||||
if (!current.policy.scope.allowGlobal || fingerprint(current.policy) !== fingerprint(context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(context.connection.owner_permissions)) throw new McpError("PERMISSION_DENIED", "Глобальный допуск сверки изменился", 403)
|
||||
const ids = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).rows.map(row => Number(row.id))
|
||||
const ids = (await deps.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).map(row => Number(row.id))
|
||||
if (ids.length > 1000) throw new McpError("LIMIT_EXCEEDED", "Каталог глобальной сверки превышает лимит", 409)
|
||||
if (ids.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Глобальная сверка требует доступа ко всему каталогу серверов", 403)
|
||||
const row = (await pool.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id])).rows[0]
|
||||
if (!row) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
const original = findMcpOperation(row.operation_id)
|
||||
return current
|
||||
}
|
||||
|
||||
function requireOriginal(actionId: string, current: ConnectionRow, deps: Dependencies) {
|
||||
const original = deps.operation(actionId)
|
||||
if (!original || original.scope !== "global" || !current.policy.scope.operationIds.includes(original.id) || original.requiredPermissions.some(permission => !hasPermission(current.policy.permissions, permission) || !hasPermission(current.owner_permissions, permission))) throw new McpError("PERMISSION_DENIED", "Утрачено право на исходную глобальную операцию", 403)
|
||||
const evidence = JSON.parse(await decryptSecret(row.evidence.sealed)) as ChangeEvidence
|
||||
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !findMcpService(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
|
||||
const declared = evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId])
|
||||
}
|
||||
|
||||
function requireServerScope(declared: number[], current: ConnectionRow) {
|
||||
if (declared.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Зависимость операции вне текущего глобального допуска", 403)
|
||||
}
|
||||
|
||||
async function authorizedSource(operationId: string, context: ActionContext, deps: Dependencies) {
|
||||
const current = await currentAccess(context, deps)
|
||||
const row = (await deps.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id]))[0]
|
||||
if (!row) {
|
||||
const authorize = async () => {
|
||||
const fresh = await currentAccess(context, deps)
|
||||
if (fresh.id !== current.id || fresh.owner_id !== current.owner_id) throw notFound()
|
||||
}
|
||||
const archived = await deps.retired({ operationId, connectionId: current.id, ownerId: current.owner_id }, authorize,
|
||||
{ deadlineAt: Date.now() + 10000, signal: context.signal })
|
||||
context.signal?.throwIfAborted()
|
||||
if (!archived) throw notFound()
|
||||
requireOriginal("servers.poll", current, deps)
|
||||
requireServerScope(archived.affectedServerIds, current)
|
||||
return { kind: "retired-poll" as const }
|
||||
}
|
||||
requireOriginal(row.operation_id, current, deps)
|
||||
const evidence = JSON.parse(await deps.decrypt(row.evidence.sealed)) as ChangeEvidence
|
||||
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !deps.service(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
|
||||
requireServerScope(evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId]), current)
|
||||
if (["pending", "running"].includes(row.state)) throw new McpError("STATE_CONFLICT", "Операция ещё выполняется", 409)
|
||||
context.signal?.throwIfAborted()
|
||||
return { row, evidence }
|
||||
return { kind: "live" as const, row, evidence }
|
||||
}
|
||||
export function registerGlobalReconciliationActions(register: typeof add) {
|
||||
|
||||
function registerActions(register: typeof add, deps: Dependencies) {
|
||||
register("operations.global.reconcile", "mm_global_operation_reconcile", "Сверить глобальную сервисную операцию без повторной записи", "mcp", null, "read", "global", z.object({ operationId: z.string().uuid() }).strict(), {
|
||||
target: async (input, context) => { await authorizedSource(String(input.operationId), context); return { scope: "global" } },
|
||||
target: async (input, context) => { await authorizedSource(String(input.operationId), context, deps); return { scope: "global" } },
|
||||
read: async (input, context) => {
|
||||
const { row, evidence } = await authorizedSource(String(input.operationId), context)
|
||||
const authorize = async () => { await authorizedSource(String(input.operationId), context) }
|
||||
const operationId = String(input.operationId)
|
||||
const source = await authorizedSource(operationId, context, deps)
|
||||
if (source.kind === "retired-poll") return { operationId, state: "applied", observedAt: new Date().toISOString(),
|
||||
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }], automaticRetry: false }
|
||||
const { row, evidence } = source
|
||||
const authorize = async () => { if ((await authorizedSource(operationId, context, deps)).kind !== "live") throw notFound() }
|
||||
const results = []
|
||||
for (const [index, step] of evidence.steps.entries()) {
|
||||
if (step.kind !== "service") throw new McpError("UNSUPPORTED_CAPABILITY", "Нативная глобальная сверка недоступна", 409)
|
||||
await authorize()
|
||||
results.push({ stepIndex: index, service: step.service, ...await findMcpService(step.service)!.reconcile!(step.input, authorize) })
|
||||
results.push({ stepIndex: index, service: step.service, ...await deps.service(step.service)!.reconcile!(step.input, authorize) })
|
||||
}
|
||||
await authorize()
|
||||
return { operationId: row.id, state: row.state, observedAt: new Date().toISOString(), results, automaticRetry: false }
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const registerGlobalReconciliationActions = (register: typeof add) => registerActions(register, production)
|
||||
export const registerGlobalReconciliationActionsForTests = registerActions
|
||||
@@ -5,6 +5,8 @@ await prepareOwnedBandwidthState()
|
||||
const { pool, closePool } = await import("../../db/index.js")
|
||||
try {
|
||||
const { applySqlMigrations } = await import("../../db/migrate.js"); await applySqlMigrations(pool)
|
||||
const { acceptOperationReplayTombstones } = await import("./operation-replay-tombstones.sql-helper.js")
|
||||
await acceptOperationReplayTombstones()
|
||||
const { acceptOwnedMcpOperationRoot } = await import("./operation-execution-origin.sql-helper.js")
|
||||
console.log("Owned actual MCP operation root:", JSON.stringify(await acceptOwnedMcpOperationRoot()))
|
||||
} finally { await closePool() }
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
export async function acceptOperationReplayTombstones() {
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true")
|
||||
const { pool } = await import("../../db/index.js")
|
||||
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
||||
const { createMcpConnection, authenticateMcpToken } = await import("./store.js")
|
||||
const { applyPlan, invokeMcpTool } = await import("./runtime.js")
|
||||
const { operationIdempotencyDigest } = await import("./operation-replay-tombstones.js")
|
||||
const { encryptSecret, McpError } = await import("./security.js")
|
||||
const { default: Fastify } = await import("fastify")
|
||||
const { default: authPlugin } = await import("../../plugins/auth.js")
|
||||
const { createMcpRoutes } = await import("../../routes/mcp.js")
|
||||
const { env } = await import("../../config.js")
|
||||
const { SignJWT } = await import("jose")
|
||||
const makeConnection = async () => {
|
||||
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned replay fixture", apps: ["mm"],
|
||||
permissions: ["mm:users:write", "mm:mcp:read"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 }
|
||||
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval",
|
||||
permissions: actor.permissions, expiresAt: new Date(actor.expiresAt).toISOString(),
|
||||
scope: { serverIds: [], operationIds: ["users.create", "operations.read", "changes.apply"], allowGlobal: true },
|
||||
limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor)
|
||||
return { actor, principal: await authenticateMcpToken(issued.token) }
|
||||
}
|
||||
const app = Fastify()
|
||||
const ids: string[] = []
|
||||
const matches = (code: string, status: number) => (error: unknown) =>
|
||||
error instanceof McpError && error.code === code && error.statusCode === status
|
||||
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
||||
try {
|
||||
const owner = await makeConnection(), foreign = await makeConnection()
|
||||
// A real completed plan has already initialized the owned encryption state.
|
||||
// The fixture seeds its tombstone directly, so exercise the same initializer.
|
||||
await encryptSecret("owned replay fixture state")
|
||||
const connectionId = owner.principal.connectionId
|
||||
ids.push(connectionId, foreign.principal.connectionId)
|
||||
const operationId = randomUUID(), planId = randomUUID(), key = "owned-replay-" + randomUUID()
|
||||
await pool.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,'users.create','{}',true)`, [operationId, planId, connectionId, operationIdempotencyDigest(connectionId,key)])
|
||||
await assert.rejects(applyPlan(owner.principal, planId, key), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(owner.principal, planId, randomUUID()), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(owner.principal, randomUUID(), key), matches("STATE_CONFLICT",409))
|
||||
await assert.rejects(applyPlan(foreign.principal, planId, key), matches("NOT_FOUND",404))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_change_apply",{ planId,idempotencyKey:key }), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(foreign.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
|
||||
await app.register(authPlugin)
|
||||
await app.register(createMcpRoutes(async () => {}))
|
||||
for (const subject of [owner, foreign]) {
|
||||
const token = await new SignJWT({ sub: subject.actor.id, email: subject.actor.email, name: subject.actor.name,
|
||||
apps: subject.actor.apps, permissions: subject.actor.permissions })
|
||||
.setProtectedHeader({ alg:"HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("4m")
|
||||
.sign(new TextEncoder().encode(env.jwtSecret))
|
||||
const reply = await app.inject({ method: "POST", url: "/mcp/plans/"+planId+"/apply", headers: { authorization:"Bearer "+token } })
|
||||
assert.equal(reply.statusCode, subject === owner ? 410 : 404)
|
||||
assert.equal(reply.json().code, subject === owner ? "OPERATION_HISTORY_EXPIRED" : "NOT_FOUND")
|
||||
}
|
||||
await pool.query("UPDATE public.mcp_connections SET policy=jsonb_set(policy,'{scope,allowGlobal}','false') WHERE id=$1", [connectionId])
|
||||
await assert.rejects(applyPlan(owner.principal, planId, key), matches("NOT_FOUND",404))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
|
||||
const { rows:[proof] } = await pool.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE connection_id=ANY($1::uuid[])) AS no_operation,
|
||||
NOT EXISTS(SELECT 1 FROM public.mcp_budget_buckets WHERE connection_id=ANY($1::uuid[]) AND kind='mutations') AS no_write_budget`, [ids])
|
||||
assert.deepEqual(proof,{no_operation:true,no_write_budget:true})
|
||||
console.log("Owned operation tombstones: scoped MCP and Web 410, replay conflict and no dispatch")
|
||||
} finally {
|
||||
await app.close()
|
||||
await pool.query("DELETE FROM public.mcp_operation_replay_tombstones WHERE connection_id=ANY($1::uuid[])", [ids])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import type { PoolClient } from "pg"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { listActions } from "./actions.js"
|
||||
import { McpError } from "./security.js"
|
||||
import { effectiveMcpConnection, type ConnectionRow, type McpPrincipal } from "./store.js"
|
||||
|
||||
type Query = Pick<PoolClient, "query">
|
||||
export interface OperationReplayTombstone {
|
||||
operation_id: string
|
||||
plan_id: string
|
||||
connection_id: string
|
||||
action_id: string
|
||||
target_server_ids: string[]
|
||||
target_global: boolean
|
||||
}
|
||||
|
||||
// Idempotency tokens are caller input and already stored in plaintext on live
|
||||
// operations. This domain-separated digest avoids copying that plaintext and
|
||||
// survives DB restore and access-key rotation. It is not a keyed secret proof.
|
||||
export function operationIdempotencyDigest(connectionId: string, key: string): string {
|
||||
return createHash("sha256").update("mcp-operation-replay-v1\0").update(connectionId).update("\0").update(key).digest("hex")
|
||||
}
|
||||
|
||||
export function operationHistoryExpired(): never {
|
||||
throw new McpError("OPERATION_HISTORY_EXPIRED", "Срок хранения завершённой операции истёк. Повторное применение запрещено.", 410)
|
||||
}
|
||||
|
||||
export function assertOperationReplayScope(row: OperationReplayTombstone, connection: ConnectionRow): void {
|
||||
const scope = connection.policy.scope
|
||||
if (row.connection_id !== connection.id || !scope.operationIds.includes(row.action_id)) throw operationReplayNotFound()
|
||||
const action = listActions().find(candidate => candidate.operation.id === row.action_id)
|
||||
if (!action || action.operation.requiredPermissions.some(permission =>
|
||||
!hasPermission(connection.policy.permissions, permission) || !hasPermission(connection.owner_permissions, permission))) throw operationReplayNotFound()
|
||||
if (row.target_global && !scope.allowGlobal || row.target_server_ids.some(id => !scope.serverIds.some(server => String(server) === id))) throw operationReplayNotFound()
|
||||
}
|
||||
|
||||
function operationReplayNotFound(): McpError {
|
||||
return new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
}
|
||||
|
||||
const projection = "t.operation_id,t.plan_id,t.connection_id,t.action_id,t.target_server_ids::text[] AS target_server_ids,t.target_global"
|
||||
|
||||
export async function findOperationReplayTombstone(db: Query, connectionId: string, operationId: string) {
|
||||
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
WHERE t.connection_id=$1 AND t.operation_id=$2`, [connectionId, operationId])).rows[0]
|
||||
}
|
||||
|
||||
export async function findOwnerPlanReplayTombstone(db: Query, ownerId: string, planId: string) {
|
||||
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
JOIN mcp_connections c ON c.id=t.connection_id WHERE c.owner_id=$1 AND t.plan_id=$2`, [ownerId, planId])).rows[0]
|
||||
}
|
||||
|
||||
export async function refuseExpiredOperationReplay(db: Query, principal: McpPrincipal, connectionId: string, planId: string, key: string): Promise<void> {
|
||||
const rows = (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
WHERE t.connection_id=$1 AND (t.plan_id=$2 OR t.idempotency_digest=$3)`,
|
||||
[connectionId, planId, operationIdempotencyDigest(connectionId, key)])).rows
|
||||
if (!rows.length) return
|
||||
const conflict = rows.find(row => row.plan_id !== planId.toLowerCase())
|
||||
const current = await effectiveMcpConnection(principal, db)
|
||||
assertOperationReplayScope(conflict ?? rows[0], current)
|
||||
if (conflict) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409)
|
||||
operationHistoryExpired()
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import { assertOwnedBandwidthSql,prepareOwnedBandwidthState } from "../../services/bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true")
|
||||
await prepareOwnedBandwidthState()
|
||||
const {pool,closePool}=await import("../../db/index.js")
|
||||
const {applySqlMigrations}=await import("../../db/migrate.js")
|
||||
const {mcpConnectionCreateSchema}=await import("@mmapp/contracts/mcp")
|
||||
const {createMcpConnection,authenticateMcpToken}=await import("./store.js")
|
||||
const {invokeMcpTool,approvePlan,applyPlan}=await import("./runtime.js")
|
||||
const {OrdinaryRetirementCommitUnknown,retireOrdinaryMcpOperations}=await import("./ordinary-operation-retirement.js")
|
||||
const {tickMcpWorkers}=await import("./workers.js")
|
||||
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
|
||||
const {Client}=await import("pg")
|
||||
const {McpError}=await import("./security.js")
|
||||
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
|
||||
|
||||
try {
|
||||
await applySqlMigrations(pool)
|
||||
await pauseSchedulerJobsAndWait()
|
||||
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
||||
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
|
||||
async function access(){
|
||||
const actor={id:randomUUID(),email:"[email protected]",name:"Owned settings retention",apps:["mm"],
|
||||
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"],
|
||||
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
|
||||
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
|
||||
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
|
||||
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
|
||||
"servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true},
|
||||
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
|
||||
return {actor,principal:await authenticateMcpToken(issued.token)}
|
||||
}
|
||||
const {actor,principal}=await access(),foreign=(await access()).principal
|
||||
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
|
||||
settings:Record<string,unknown>={retentionDays}){
|
||||
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
|
||||
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
|
||||
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
|
||||
let terminal
|
||||
for(const deadline=Date.now()+15000;Date.now()<deadline;){
|
||||
terminal=(await pool.query("SELECT state,verification,error_code FROM mcp_operations WHERE id=$1",[applied.id])).rows[0]
|
||||
if(terminal&&!["pending","running"].includes(terminal.state))break
|
||||
await delay(50)
|
||||
}
|
||||
assert.deepEqual(terminal,{state:"applied",verification:"passed",error_code:null})
|
||||
// Only the disposable fixture ages rows, after real authenticated apply.
|
||||
// No cutoff override or production encryption/approval bypass.
|
||||
const finished=new Date(Date.now()-ageMs),created=new Date(finished.getTime()-1000)
|
||||
await pool.query("UPDATE mcp_plans SET created_at=$2,approved_at=$2,expires_at=$3 WHERE id=$1",
|
||||
[plan.planId,created,new Date(finished.getTime()+1000)])
|
||||
await pool.query("UPDATE mcp_operations SET created_at=$2,finished_at=$3 WHERE id=$1",[applied.id,created,finished])
|
||||
await pool.query("UPDATE mcp_operation_steps SET created_at=$2,finished_at=$3 WHERE operation_id=$1",[applied.id,created,finished])
|
||||
return {id:applied.id,planId:plan.planId,key}
|
||||
}
|
||||
const age=180*86400000
|
||||
const held=await operation(51,age+180000),spent=await operation(52,age+120000)
|
||||
const due=await operation(53,age+60000),young=await operation(54,age-60000)
|
||||
const boundary=await pool.connect()
|
||||
try{
|
||||
await boundary.query("BEGIN")
|
||||
await boundary.query("UPDATE mcp_operations SET finished_at=transaction_timestamp()-interval '180 days' WHERE id=$1",[young.id])
|
||||
await assert.rejects(boundary.query(`INSERT INTO mcp_ordinary_retirement_claims
|
||||
(operation_id,plan_id,connection_id,source_proof,claim_token) VALUES($1,$2,$3,$4,$5)`,
|
||||
[young.id,young.planId,principal.connectionId,"a".repeat(64),randomUUID()]),/cannot be replaced or reopened/)
|
||||
}finally{await boundary.query("ROLLBACK");boundary.release()}
|
||||
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[randomUUID(),held.id])
|
||||
const ids=[held.id,spent.id,due.id,young.id]
|
||||
const projection=async()=>(await pool.query(`SELECT i.id,o.state,p.state AS plan_state,s.state AS step_state,
|
||||
k.operation_id AS checkpoint,t.operation_id AS tombstone,c.claim_token
|
||||
FROM unnest($1::uuid[]) i(id)
|
||||
LEFT JOIN mcp_operations o ON o.id=i.id LEFT JOIN mcp_plans p ON p.id=o.plan_id
|
||||
LEFT JOIN mcp_operation_steps s ON s.operation_id=i.id LEFT JOIN mcp_operation_checkpoints k ON k.operation_id=i.id
|
||||
LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=i.id
|
||||
LEFT JOIN mcp_ordinary_retirement_claims c ON c.operation_id=i.id ORDER BY i.id`,[ids])).rows
|
||||
const originalQuery=Client.prototype.query
|
||||
let commits=0
|
||||
try{
|
||||
Client.prototype.query=function(this:InstanceType<typeof Client>,...args:unknown[]){
|
||||
if((args[0] as {text?:string})?.text==="COMMIT"&&++commits===3)
|
||||
return Promise.reject(Error("owned ordinary destructive COMMIT transport loss"))
|
||||
return Reflect.apply(originalQuery,this,args)
|
||||
} as typeof Client.prototype.query
|
||||
await assert.rejects(retireOrdinaryMcpOperations(),
|
||||
(error:unknown)=>error instanceof OrdinaryRetirementCommitUnknown&&error.automaticReplay===false)
|
||||
}finally{Client.prototype.query=originalQuery}
|
||||
assert.equal(commits,3)
|
||||
const before=await projection(),spentBefore=before.find(row=>row.id===spent.id)
|
||||
assert.equal(spentBefore?.state,"applied");assert(spentBefore?.claim_token);assert.equal(spentBefore?.tombstone,null)
|
||||
await tickMcpWorkers()
|
||||
const after=await projection()
|
||||
assert.deepEqual(after.filter(row=>row.id!==due.id),before.filter(row=>row.id!==due.id))
|
||||
const retired=after.find(row=>row.id===due.id)
|
||||
assert(retired?.claim_token);assert.equal(retired?.tombstone,due.id)
|
||||
for(const field of ["state","plan_state","step_state","checkpoint"])assert.equal(retired[field],null)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_plans WHERE id=$1",[due.planId])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT retention_days FROM router_log_settings WHERE id=1")).rows[0].retention_days,54)
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await projection(),after)
|
||||
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(principal,randomUUID(),due.key),errorIs("STATE_CONFLICT",409))
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:due.id}),errorIs("NOT_FOUND",404))
|
||||
const telemetry=[
|
||||
{action:"traffic.settings.update",version:2,record:await operation(55,age+60000,"mm_traffic_settings_update_plan")},
|
||||
{action:"uptime.settings.update",version:2,record:await operation(56,age+60000,"mm_uptime_settings_update_plan",{enabled:false,retentionDays:56})},
|
||||
{action:"servers.api-ping.settings.update",version:2,record:await operation(0,age+60000,"mm_servers_api_ping_settings_update_plan",{intervalSec:123})},
|
||||
{action:"internet-path.settings.update",version:1,record:await operation(57,age+60000,"mm_internet_path_settings_update_plan")},
|
||||
]
|
||||
const telemetryRows=async()=>(await pool.query(`SELECT p.operation_id,p.operation_version,o.id
|
||||
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=ANY($1::uuid[]) ORDER BY p.operation_id`,
|
||||
[telemetry.map(item=>item.record.id)])).rows
|
||||
assert.deepEqual((await telemetryRows()).map(row=>[row.operation_id,row.operation_version]),
|
||||
telemetry.map(item=>[item.action,item.version]).sort((a,b)=>String(a[0]).localeCompare(String(b[0]))))
|
||||
const telemetrySettings=async()=>(await pool.query(`SELECT to_jsonb(t) AS traffic,to_jsonb(u) AS uptime,
|
||||
to_jsonb(a) AS api_ping,to_jsonb(i) AS internet FROM traffic_settings t,uptime_settings u,
|
||||
servers_api_ping_settings a,internet_path_settings i WHERE t.id=1 AND u.id=1 AND a.id=1 AND i.id=1`)).rows[0]
|
||||
const settingsBefore=await telemetrySettings()
|
||||
const traffic=telemetry[0].record
|
||||
const {rows:[originalStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[traffic.id])
|
||||
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{retentionDays}','999') WHERE operation_id=$1",[traffic.id])
|
||||
await tickMcpWorkers()
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[traffic.id])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[traffic.id])).rows.length,1)
|
||||
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[traffic.id,originalStep.result])
|
||||
await tickMcpWorkers()
|
||||
await tickMcpWorkers()
|
||||
// The keyset owner may need one tick to wrap after the rejected oldest row.
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await telemetryRows(),[])
|
||||
assert.deepEqual(await telemetrySettings(),settingsBefore)
|
||||
for(const {action,record} of telemetry){
|
||||
const {rows:[proof]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
|
||||
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[record.id,record.planId])
|
||||
assert.deepEqual(proof,{action_id:action,target_global:true,server_ids:[String(serverId)],plan_gone:true,step_gone:true,checkpoint_gone:true})
|
||||
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
|
||||
}
|
||||
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
|
||||
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
|
||||
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
|
||||
console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
|
||||
}finally{resumeSchedulerJobs();await closePool()}
|
||||
@@ -0,0 +1,199 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
|
||||
import { z } from "zod"
|
||||
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
|
||||
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
|
||||
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
||||
import { operationIdempotencyDigest } from "./operation-replay-tombstones.js"
|
||||
import { decryptSecretExisting, fingerprint, privateFingerprinterExisting, redact } from "./security.js"
|
||||
|
||||
const uuid = z.string().uuid()
|
||||
const positive = z.number().int().positive().safe()
|
||||
const object = z.record(z.string(),z.unknown())
|
||||
const sealed = z.object({sealed:z.string().min(1).max(262144)}).strict()
|
||||
const ids = z.array(positive).max(1000)
|
||||
const retention = z.number().int().min(1).max(365)
|
||||
const bool = z.boolean()
|
||||
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
|
||||
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
|
||||
// offset. Accept that exact wire form as well as an ISO timestamptz.
|
||||
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
|
||||
.refine(value=>Number.isFinite(Date.parse(value.replace(" ","T").replace(/([+-]\d\d)$/,"$1:00"))))
|
||||
const metadata = {
|
||||
id:z.literal(1),createdAt:timestamp,updatedAt:timestamp,
|
||||
lastCollectedAt:timestamp.nullable(),lastDurationMs:z.number().int().safe().nullable(),
|
||||
lastError:z.string().max(16384).nullable(),
|
||||
}
|
||||
function telemetryFamily(version:number,domain:string,settings:z.ZodObject,hasEpoch:boolean,hasConfigurationProof:boolean){
|
||||
const originalSettings=hasConfigurationProof?settings.extend({configurationProof:z.string().regex(/^[0-9a-f]{64}$/)}):settings
|
||||
const effect=settings.extend({...metadata,...(hasEpoch?{pollConfigurationEpoch:uuid}:{})}).strict()
|
||||
return {version,domain,patch:settings.partial().strict().refine(value=>Object.keys(value).length>0),
|
||||
before:z.object({value:originalSettings.strict(),serverIds:ids}).strict(),effect}
|
||||
}
|
||||
const families={
|
||||
"logs.settings.update":{version:1,domain:null,patch:logSettingsPatchSchema,before:logSettingsSchema.strict(),effect:logSettingsSchema.strict()},
|
||||
"traffic.settings.update":telemetryFamily(2,"traffic-settings",z.object({enabled:bool,intervalSec:interval(5,3600),retentionDays:retention}),true,true),
|
||||
"uptime.settings.update":telemetryFamily(2,"uptime-settings",z.object({enabled:bool,resourcesEnabled:bool,pingEnabled:bool,
|
||||
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
|
||||
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
|
||||
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
|
||||
} as const
|
||||
const planEvidence = z.object({
|
||||
target:z.object({scope:z.literal("global")}).strict(),
|
||||
objects:z.array(z.object({kind:z.string(),id:z.string(),serverId:positive.optional()}).strict()).max(1001),
|
||||
steps:z.tuple([z.object({kind:z.literal("service"),service:z.string(),input:z.object({settings:object}).strict()}).strict()]),
|
||||
snapshots:z.tuple([]),before:object,warnings:z.array(z.string()).max(8),
|
||||
verificationScope:z.literal("configuration").optional(),
|
||||
requiresHuman:z.literal(true).optional(),
|
||||
}).strict()
|
||||
|
||||
class RetirementRefused extends Error {}
|
||||
export class OrdinaryRetirementCommitUnknown extends Error {
|
||||
readonly automaticReplay = false
|
||||
constructor() { super("Ordinary operation retirement outcome unknown; automatic replay forbidden") }
|
||||
}
|
||||
const requireThat: (value: unknown) => asserts value = value => {
|
||||
if (!value) throw new RetirementRefused()
|
||||
}
|
||||
|
||||
// Fixed reviewed settings families without artifact, historic receipt or
|
||||
// inverse-plan consumers. Other action families need separate review.
|
||||
const ELIGIBLE = `FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
||||
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
|
||||
WHERE (p.operation_id,p.operation_version) IN (
|
||||
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
|
||||
) AND p.state='completed'
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND isfinite(p.expires_at) AND p.expires_at<transaction_timestamp()
|
||||
AND s.state='verified' AND s.server_id IS NULL AND isfinite(s.finished_at)
|
||||
AND k.after_sealed IS NOT NULL
|
||||
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
|
||||
AND (SELECT count(*) FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)=1
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
|
||||
const CANDIDATES = `SELECT o.id,o.connection_id,o.finished_at::text AS finished_at ${ELIGIBLE}
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_ordinary_retirement_claims WHERE operation_id=o.id)
|
||||
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
|
||||
ORDER BY o.finished_at,o.id LIMIT 8`
|
||||
const SOURCE = `SELECT to_jsonb(o) AS operation,to_jsonb(p) AS plan,to_jsonb(s) AS step,to_jsonb(k) AS checkpoint ${ELIGIBLE}
|
||||
AND o.id=$1 AND o.connection_id=$2
|
||||
AND octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)+octet_length(to_jsonb(k)::text)<=1048576
|
||||
FOR UPDATE OF o,p,s,k NOWAIT`
|
||||
|
||||
async function transaction<T>(deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise<T>):Promise<T> {
|
||||
const sql = await connectProbeStorage({deadlineAt})
|
||||
let committing=false,committed=false
|
||||
try {
|
||||
await sql.query("BEGIN")
|
||||
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
|
||||
const result=await work(sql)
|
||||
committing=true
|
||||
await sql.query("COMMIT")
|
||||
committed=true
|
||||
return result
|
||||
} catch(error) {
|
||||
if(committing)throw new OrdinaryRetirementCommitUnknown()
|
||||
try{await sql.query("ROLLBACK")}catch{/* Destroy this checkout. */}
|
||||
throw error
|
||||
} finally {
|
||||
sql.release(!committed)
|
||||
}
|
||||
}
|
||||
|
||||
async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:string) {
|
||||
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT",[connectionId])).rows.length===1)
|
||||
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",["mcp-operation:"+operationId])).rows[0]?.locked===true)
|
||||
const rows=(await sql.query(SOURCE,[operationId,connectionId])).rows
|
||||
requireThat(rows.length===1)
|
||||
const raw=rows[0], operation=object.parse(raw.operation), plan=object.parse(raw.plan)
|
||||
const step=object.parse(raw.step),checkpoint=object.parse(raw.checkpoint)
|
||||
const open=async(value:unknown)=>JSON.parse(await decryptSecretExisting(sealed.parse(value).sealed))
|
||||
const original=planEvidence.parse(await open(plan.evidence))
|
||||
const action=z.enum(Object.keys(families) as [keyof typeof families,...(keyof typeof families)[]]).parse(plan.operation_id)
|
||||
const family=families[action],service=original.steps[0]
|
||||
requireThat(plan.operation_version===family.version&&service.service===action)
|
||||
requireThat(fingerprint(await open(plan.payload))===fingerprint(service.input))
|
||||
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
|
||||
const patch=object.parse(family.patch.parse(service.input.settings))
|
||||
const effect=object.parse(family.effect.parse(step.result))
|
||||
requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value))
|
||||
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
|
||||
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
|
||||
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
|
||||
}
|
||||
const serverObjects=original.objects.filter(item=>item.kind==="server")
|
||||
const serverIds=ids.parse(serverObjects.map(item=>item.serverId))
|
||||
requireThat(new Set(serverIds).size===serverIds.length&&serverObjects.every(item=>item.id===String(item.serverId)))
|
||||
const domainObjects=original.objects.filter(item=>item.kind!=="server")
|
||||
const before=family.before.parse(original.before)
|
||||
if(family.domain===null){
|
||||
requireThat(domainObjects.length===0&&original.requiresHuman===undefined)
|
||||
}else{
|
||||
requireThat(original.requiresHuman===true&&original.verificationScope==="configuration"
|
||||
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
|
||||
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
|
||||
}
|
||||
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
|
||||
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
|
||||
}
|
||||
|
||||
async function retire(operationId:string,connectionId:string,deadlineAt:number) {
|
||||
const token=randomUUID()
|
||||
await transaction(deadlineAt,async sql=>{
|
||||
const source=await prepare(sql,operationId,connectionId)
|
||||
const claim=await sql.query(`INSERT INTO public.mcp_ordinary_retirement_claims(operation_id,plan_id,connection_id,source_proof,claim_token)
|
||||
VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING RETURNING operation_id`,[operationId,source.planId,connectionId,source.proof,token])
|
||||
requireThat(claim.rows.length===1&&claim.rows[0].operation_id===operationId)
|
||||
})
|
||||
await transaction(deadlineAt,async sql=>{
|
||||
const claim=(await sql.query(`SELECT source_proof,plan_id FROM public.mcp_ordinary_retirement_claims
|
||||
WHERE operation_id=$1 AND connection_id=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,[operationId,connectionId,token])).rows
|
||||
requireThat(claim.length===1)
|
||||
const source=await prepare(sql,operationId,connectionId)
|
||||
requireThat(claim[0].source_proof===source.proof&&claim[0].plan_id===source.planId)
|
||||
const tombstone=await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,$5,$6,true) RETURNING operation_id`,
|
||||
[operationId,source.planId,connectionId,operationIdempotencyDigest(connectionId,source.key),source.action,source.serverIds])
|
||||
requireThat(tombstone.rows.length===1&&tombstone.rows[0].operation_id===operationId)
|
||||
const checkpoint=await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
||||
const step=await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
||||
const operation=await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id AS operation_id",[operationId,connectionId,source.planId])
|
||||
const plan=await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id AS plan_id",[source.planId,connectionId])
|
||||
requireThat([checkpoint,step,operation].every(result=>result.rows.length===1&&result.rows[0].operation_id===operationId)
|
||||
&&plan.rows.length===1&&plan.rows[0].plan_id===source.planId)
|
||||
})
|
||||
}
|
||||
|
||||
let cursor:{finishedAt:string;operationId:string}|undefined
|
||||
export function retireOrdinaryMcpOperations() {
|
||||
return sharedDatabaseGate.request(async()=>{
|
||||
const deadlineAt=Date.now()+15000
|
||||
const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES,[cursor?.finishedAt??null,cursor?.operationId??null])).rows)
|
||||
requireThat(rows.length<=8)
|
||||
let scanned=0,retired=0
|
||||
for(const row of rows){
|
||||
if(Date.now()>=deadlineAt-2000||retired>=2)break
|
||||
const id=uuid.parse(row.id),connectionId=uuid.parse(row.connection_id)
|
||||
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
|
||||
cursor={finishedAt:row.finished_at,operationId:id}
|
||||
scanned++
|
||||
try{await retire(id,connectionId,deadlineAt);retired++}
|
||||
catch(error){if(error instanceof OrdinaryRetirementCommitUnknown)throw error}
|
||||
}
|
||||
if(scanned===rows.length&&rows.length<8)cursor=undefined
|
||||
return {scanned,retired,automaticReplay:false as const}
|
||||
})
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { readFile } from "node:fs/promises"
|
||||
import { test } from "node:test"
|
||||
import { runInNewContext } from "node:vm"
|
||||
import ts from "typescript"
|
||||
import type { ActionContext } from "./action-types.js"
|
||||
import { loadReservedScopedRecoverySource } from "./routing-community-scoped-recovery-source.js"
|
||||
import { executeReservedScopedRecovery, scopedRecoveryIntentFromCheckpoint, validateReservedInverseVerified } from "./routing-community-scoped-recovery.js"
|
||||
@@ -16,6 +19,67 @@ test("a serialized allocation cannot enter the shared inverse engine", async ()
|
||||
await assert.rejects(executeReservedScopedRecovery(forged, {} as Parameters<typeof executeReservedScopedRecovery>[1]), /capability/)
|
||||
})
|
||||
|
||||
test("confirmed inverse spend quarantines the exact slot on post-spend refusal without replay", async () => {
|
||||
// Execute the actual boundary body with only its I/O dependencies substituted.
|
||||
// This does not mint a production capability or claim native/SQL acceptance.
|
||||
const source = ts.createSourceFile("routing-community-scoped-recovery.ts",
|
||||
await readFile(new URL("./routing-community-scoped-recovery.ts", import.meta.url), "utf8"), ts.ScriptTarget.ES2022)
|
||||
const declaration = source.statements.find(node => ts.isFunctionDeclaration(node) && node.name?.text === "executeReservedScopedRecovery")
|
||||
assert(declaration)
|
||||
const compiled = ts.transpileModule(declaration.getText(source), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText
|
||||
const allocation = Object.freeze({ kind: "routing-reserved-inverse-allocation" })
|
||||
const execution = Object.freeze({ kind: "routing-reserved-inverse-execution" })
|
||||
const principal = { connectionId: "owned", credentialId: "original" }
|
||||
const original = new Error("original authority failure"), result = Object.freeze({ outcome: "recovered" })
|
||||
const scenarios = {
|
||||
spend: ["spend"],
|
||||
inspect: ["spend", "inspect", "quarantine"],
|
||||
authority: ["spend", "inspect", "authority", "quarantine"],
|
||||
source: ["spend", "inspect", "authority", "source", "quarantine"],
|
||||
proof: ["spend", "inspect", "authority", "source", "proof", "quarantine"],
|
||||
engine: ["spend", "inspect", "authority", "source", "proof", "engine", "quarantine"],
|
||||
success: ["spend", "inspect", "authority", "source", "proof", "engine"],
|
||||
} as const
|
||||
for (const [scenario, expected] of Object.entries(scenarios)) {
|
||||
const events: string[] = [], exports: Record<string, unknown> = {}, capabilities = new WeakMap()
|
||||
let request: object | undefined
|
||||
const at = (phase: string) => { events.push(phase); if (phase === scenario) throw original }
|
||||
runInNewContext(compiled, {
|
||||
exports, reservedExecutions: capabilities, fingerprint: JSON.stringify,
|
||||
createRoutingReservationStore: () => ({
|
||||
async beginInverseAttempt(value: unknown) {
|
||||
assert.equal(value, allocation); at("spend"); return { reservationId: "spent-slot", execution }
|
||||
},
|
||||
async inspectInverseExecution(value: unknown) {
|
||||
assert.equal(value, execution); at("inspect")
|
||||
return { principal, sourceOperationId: "source", deadline: "2026-10-29T01:00:00.000Z" }
|
||||
},
|
||||
async markUnknown(id: string, lane: string) {
|
||||
assert.equal(id, "spent-slot"); assert.equal(lane, "inverse"); at("quarantine")
|
||||
throw new Error("quarantine persistence unavailable")
|
||||
},
|
||||
}),
|
||||
effectiveMcpConnection: async () => { at("authority"); return { owner_id: "owner", policy: {}, owner_permissions: [] } },
|
||||
loadReservedScopedRecoverySource: async () => {
|
||||
at("source"); return { profile: { serverId: 1 }, input: {}, configuration: { packet: {} } }
|
||||
},
|
||||
accessGeneration: async () => "generation",
|
||||
deadline: () => Date.parse("2026-10-29T01:00:00.000Z"),
|
||||
sealedProof: async () => { at("proof"); return "proof" },
|
||||
executeScopedRecovery: async (value: object) => {
|
||||
request = value; assert.equal(capabilities.get(value), execution); at("engine"); return result
|
||||
},
|
||||
})
|
||||
const execute = exports.executeReservedScopedRecovery as
|
||||
(allocation: unknown, expected: { sourceOperationId: string; principal: typeof principal }) => Promise<unknown>
|
||||
const outcome = execute(allocation, { sourceOperationId: "source", principal })
|
||||
if (scenario === "success") assert.equal(await outcome, result)
|
||||
else await assert.rejects(outcome, error => error === original)
|
||||
assert.deepEqual(events, expected, scenario)
|
||||
if (request) assert.equal(capabilities.has(request), false)
|
||||
}
|
||||
})
|
||||
|
||||
test("configuration equality or generic VERIFIED cannot replace reserved signed receipts", async () => {
|
||||
const expected = {} as Parameters<typeof validateReservedInverseVerified>[1]
|
||||
for (const row of [{}, { state: "verified", phase: "RECOVERY_VERIFIED", configurationVerified: true }, { inverse_intent_sealed: "x".repeat(524289) }]) {
|
||||
|
||||
@@ -35,8 +35,9 @@ import { registerManagementChannel } from "./management.js"
|
||||
import { isSchedulerJobsPaused, pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../../services/scheduler-running.js"
|
||||
|
||||
/** Disposable SQL + loopback HTTP only. Does not access or configure CHR. */
|
||||
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward" } = {}) {
|
||||
const sid = 131, writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
|
||||
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward"; lateInverseAuthorityChange?: "edit" | "disable" | "grant" } = {}) {
|
||||
const sid = options.lateInverseAuthorityChange ? { edit: 131, disable: 132, grant: 133 }[options.lateInverseAuthorityChange] : 131
|
||||
const writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
|
||||
const initial = compileManagedFilterRules([{ community: "65000:10", gateway: "192.0.2.1", outIface: "", action: "route", description: "" }, { community: "65000:20", gateway: "192.0.2.5", outIface: "", action: "route", description: "" }])
|
||||
const filters = ["a", "b"].map(key => ({ ".id": "*f" + key, chain: "custom-" + key, rule: initial, disabled: "false", comment: managedComment("NEVER_SCOPED_COMMENT_" + key), opaque: "NEVER_SCOPED_OPAQUE" }))
|
||||
let mode: "normal" | "ignored" | "lost" | "collateral" | "post-failed" | "post-unknown" = "normal", writes = 0, requests = 0
|
||||
@@ -423,8 +424,9 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
|
||||
await assertOriginalCredentialRevocation()
|
||||
return {accepted:true,originalCredentialRevocation:true,unauthorized401:true,fullAdmissionRollback:true,nativeReads:0,nativeWrites:0,realSql:true,allWriterLedgerComplete:false}
|
||||
}
|
||||
if (options.manualInverseChargeOnly) {
|
||||
if (options.manualInverseChargeOnly || options.lateInverseAuthorityChange) {
|
||||
const chargeCase = options.manualInverseChargeCase ?? "positive"
|
||||
if (options.lateInverseAuthorityChange) assert.equal(chargeCase, "positive")
|
||||
const persistent = chargeCase !== "nonpersistent"
|
||||
// Same genuine fixed-reader baseline/CAS used by the existing persistent
|
||||
// acceptance below. No hand-written baseline, sealed plan or authority.
|
||||
@@ -573,6 +575,48 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
|
||||
realSql: true, allWriterLedgerComplete: false, reservedDispatchEnabled: false, autoRecoveryEnabled: false }
|
||||
}
|
||||
console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-apply" }))
|
||||
const lateChange = options.lateInverseAuthorityChange
|
||||
if (lateChange) {
|
||||
const priorIntent = (await pool.query("SELECT coalesce(max(id),0)::text AS id FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent'", [connectionId])).rows[0].id
|
||||
let writeId: string | undefined
|
||||
afterIntent = async () => {
|
||||
const intents = (await pool.query(`SELECT payload FROM mcp_audit WHERE connection_id=$1 AND id>$2::bigint
|
||||
AND action='configuration.intent' AND payload->>'path'=$3`,
|
||||
[connectionId, priorIntent, "/routing/filter/rule/" + loaded.configuration.packet.nativeId])).rows
|
||||
assert.equal(intents.length, 1)
|
||||
assert.equal(intents[0].payload.method, "PATCH")
|
||||
assert.equal(intents[0].payload.serverId, sid)
|
||||
writeId = intents[0].payload.writeId
|
||||
assert.match(writeId ?? "", /^[0-9a-f-]{36}$/)
|
||||
const sql = {
|
||||
edit: "UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late inverse edit\"') WHERE id=$1",
|
||||
disable: "UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1",
|
||||
grant: "UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1",
|
||||
}[lateChange]
|
||||
assert.equal((await pool.query(sql, [id])).rowCount, 1)
|
||||
}
|
||||
const inverse = await execute(recovery.planId)
|
||||
assert.equal(afterIntent, undefined)
|
||||
assert.ok(writeId)
|
||||
assert.notEqual(inverse.state, "applied")
|
||||
assert.equal(writes, afterForward.writes)
|
||||
assert.equal(inversePostChecks, 0)
|
||||
const auditRows = (await pool.query(`SELECT action FROM mcp_audit WHERE connection_id=$1 AND payload->>'writeId'=$2
|
||||
ORDER BY id`, [connectionId, writeId])).rows
|
||||
assert.deepEqual(auditRows.map(row => row.action), ["configuration.intent", "configuration.unconfirmed"])
|
||||
const refusal = (await pool.query(`SELECT payload->>'phase' AS phase, payload->>'code' AS code
|
||||
FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed'`, [connectionId])).rows
|
||||
assert.deepEqual(refusal, [{ phase: "before-dispatch-context", code: "STATE_CONFLICT" }])
|
||||
assert.deepEqual((await pool.query(`SELECT state,verified_sealed FROM mcp_scoped_recovery_intents
|
||||
WHERE source_operation_id=$1 AND recovery_operation_id=$2`, [source.id, inverse.id])).rows,
|
||||
[{ state: "unknown", verified_sealed: null }])
|
||||
assert.equal((await pool.query(`SELECT count(*)::int AS n FROM mcp_operation_checkpoints
|
||||
WHERE operation_id=$1 AND configuration_sealed IS NOT NULL`, [inverse.id])).rows[0].n, 0)
|
||||
assert.equal((await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows[0].error_code,
|
||||
"MUTATION_OUTCOME_UNKNOWN")
|
||||
return { accepted: true, lateInverseAuthorityChange: lateChange, durableIntents: 1, inverseNativeWrites: 0,
|
||||
realSql: true, realMcpDispatcher: true, fullInverseLifecycleRepeated: false }
|
||||
}
|
||||
if (chargeCase === "quota" || chargeCase === "revoked") {
|
||||
if (chargeCase === "revoked") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id])
|
||||
const beforeOperations = (await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n
|
||||
|
||||
@@ -257,24 +257,30 @@ async function checkpoint(value: Session, intent: Intent, phase: Checkpoint["pha
|
||||
export async function executeReservedScopedRecovery(allocation: RoutingReservedInverseAllocation, expected: { sourceOperationId: string; principal: McpPrincipal }) {
|
||||
const store = createRoutingReservationStore(), spent = await store.beginInverseAttempt(allocation)
|
||||
const execution = spent.execution
|
||||
const binding = await store.inspectInverseExecution(execution)
|
||||
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
|
||||
const current = await effectiveMcpConnection(binding.principal)
|
||||
const context: ActionContext = { principal: binding.principal, connection: current }
|
||||
const source = await loadReservedScopedRecoverySource(execution, context)
|
||||
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
|
||||
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
|
||||
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
|
||||
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
|
||||
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
|
||||
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
|
||||
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
|
||||
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
|
||||
input.requestProof = await sealedProof(input, "request")
|
||||
reservedExecutions.set(input, execution)
|
||||
try { return await executeScopedRecovery(input) }
|
||||
catch (error) { await store.markUnknown(binding.reservationId, "inverse").catch(() => {}); throw error }
|
||||
finally { reservedExecutions.delete(input) }
|
||||
try {
|
||||
const binding = await store.inspectInverseExecution(execution)
|
||||
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
|
||||
const current = await effectiveMcpConnection(binding.principal)
|
||||
const context: ActionContext = { principal: binding.principal, connection: current }
|
||||
const source = await loadReservedScopedRecoverySource(execution, context)
|
||||
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
|
||||
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
|
||||
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
|
||||
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
|
||||
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
|
||||
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
|
||||
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
|
||||
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
|
||||
input.requestProof = await sealedProof(input, "request")
|
||||
reservedExecutions.set(input, execution)
|
||||
try { return await executeScopedRecovery(input) }
|
||||
finally { reservedExecutions.delete(input) }
|
||||
} catch (error) {
|
||||
// Acknowledged spend owns this exact slot even when its authority can no
|
||||
// longer be inspected. A refused or uncertain spend never enters here.
|
||||
await store.markUnknown(spent.reservationId, "inverse").catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Fixed-domain receipt verifier used by the reservation store. It recognizes
|
||||
|
||||
@@ -14,7 +14,9 @@ import { nativeBgpRows, bgpTemplateReferences } from "../../services/bgp-export-
|
||||
import { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow } from "../../services/bgp-filter-binding-model.js"
|
||||
import { canonicalVrfMembers } from "../../services/vrf-members.js"
|
||||
import { readOspfEffectiveDisabled } from "../../services/ospf-effective-disabled.js"
|
||||
import { isConfirmedRoutingFibTable } from "../../services/routing-table-fib.js"
|
||||
import * as contextModel from "./routing-community-context-model.js"
|
||||
import { prepareOriginalEffectSelection } from "./routing-original-effects.js"
|
||||
import { readMeasurementAddress } from "./routing-measurement-address.js"
|
||||
import { routeContextClients } from "./routing-community-context-native.js"
|
||||
import { pathProofTag } from "./path-proof-purpose.js"
|
||||
@@ -65,9 +67,8 @@ async function fixture(autoRecovery = false) {
|
||||
afterHttp?.(); afterHttp = undefined
|
||||
if (afterPath?.path === path) { const run = afterPath.run; afterPath = undefined; run() }
|
||||
})
|
||||
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
||||
const now = Date.now(), fingerprint = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
|
||||
const endpoint = { id: 1, host: "127.0.0.1", port: (router.address() as { port: number }).port, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
|
||||
const endpoint = { id: 1, host: "127.0.0.1", port: 0, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
|
||||
const principal = { connectionId: randomUUID(), credentialId: randomUUID(), credentialExpiresAt: new Date(now + 600000).toISOString() }
|
||||
const connection = { id: principal.connectionId, owner_id: "owner", generation: "generation", owner_permissions: ["mm:network:read"],
|
||||
policy: { permissions: ["mm:network:read"], scope: { serverIds: [1] }, expiresAt: new Date(now + 600000).toISOString() }, owner_valid_until: new Date(now + 600000) }
|
||||
@@ -92,6 +93,7 @@ async function fixture(autoRecovery = false) {
|
||||
"./routing-community-context-native.js": { routeContextClients }, "../../services/vrf-members.js": { canonicalVrfMembers }, "../../services/ospf-effective-disabled.js": { readOspfEffectiveDisabled },
|
||||
"./routing-measurement-address.js": { readMeasurementAddress } }
|
||||
await loadSource("./routing-community-context.ts", { ...common, "./routing-network-native-session.js": native,
|
||||
"../../services/routing-table-fib.js": { isConfirmedRoutingFibTable }, "./routing-original-effects.js": { prepareOriginalEffectSelection },
|
||||
"../../services/bgp-export-model.js": { nativeBgpRows, bgpTemplateReferences }, "../../services/bgp-filter-binding-model.js": { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow },
|
||||
"./routing-community-context-model.js": contextModel, "./path-proof-purpose.js": { pathProofTag },
|
||||
"./management-transport.js": { assertManagementTransport: async () => { throw Error("No alternate management fixture") } } }, context)
|
||||
@@ -103,14 +105,23 @@ async function fixture(autoRecovery = false) {
|
||||
await loadSource("./routing-network-scoped-context.ts", { "./routing-network-native-session.js": native }, privateApi)
|
||||
const create = native.createRoutingNetworkNativeSession as (options: unknown) => Promise<Session>
|
||||
const abort = new AbortController()
|
||||
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } })
|
||||
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
||||
endpoint.port = (router.address() as { port: number }).port
|
||||
const closeRouter = () => {
|
||||
router.closeAllConnections()
|
||||
return new Promise<void>(resolve => router.close(() => resolve()))
|
||||
}
|
||||
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } }).catch(async error => {
|
||||
await closeRouter()
|
||||
throw error
|
||||
})
|
||||
const readContext = privateApi.readRoutingNetworkScopedCommunityContext as (token: Token) => Promise<Context>
|
||||
return { session, readContext: () => readContext(session.sharedContext), forged: () => readContext(JSON.parse(JSON.stringify(session.sharedContext))),
|
||||
readResource: native.readRoutingNetworkSharedResource as (token: Token, server: number, path: string, body?: Record<string, string>) => Promise<unknown>,
|
||||
bypassTail: context.readRouteCommunityContextWithSharedNativeSession as (token: Token) => Promise<unknown>,
|
||||
counters: () => ({ requests, wireBytes, wireRows, rawGetObjects }), bulk: (value: number) => { bulk = value }, padding: (value: number) => { padding = value },
|
||||
revoke: () => { revoked = true }, abort: () => abort.abort(), afterHttp: (hook: () => void) => { afterHttp = hook }, afterPath: (path: string, hook: () => void) => { afterPath = { path, run: hook } },
|
||||
cleanup: async () => { session.close(); router.closeAllConnections(); await new Promise<void>(resolve => router.close(() => resolve())) } }
|
||||
cleanup: async () => { session.close(); await closeRouter() } }
|
||||
}
|
||||
|
||||
test("actual source context and snapshots share one pinned HTTP request/byte/row budget including object /get", async () => {
|
||||
|
||||
@@ -19,7 +19,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const { pool } = await import("../../db/index.js")
|
||||
const { mcpConnectionPolicySchema } = await import("@mmapp/contracts/mcp")
|
||||
const { routeAiProfileSchema } = await import("@mmapp/contracts/mcp-routing")
|
||||
const { accessGeneration } = await import("./security.js")
|
||||
const { accessGeneration, McpError } = await import("./security.js")
|
||||
const { readRoutingEndpoints, createRoutingSourceContext } = await import("./routing-observation-source.js")
|
||||
const { resolveRoutingProfileAuthority } = await import("./routing-profile-authority.js")
|
||||
const { captureScopedRecoveryAuthority, scopedRecoveryDeadlineCeiling } = await import("./routing-community-scoped-recovery.js")
|
||||
@@ -27,7 +27,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
assert.equal((await pool.query("SELECT current_database() AS name")).rows[0].name, "postgres")
|
||||
const previousEnabled = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled as boolean
|
||||
const profileIds: string[] = [], ids: number[] = []
|
||||
let nativeHttpRequests = 0, callbackDispatches = 0, durableIntents = 0
|
||||
let nativeHttpRequests = 0, callbackDispatches = 0, dispatchFences = 0
|
||||
const listener = createServer((req, res) => { nativeHttpRequests++; req.resume(); res.setHeader("content-type", "application/json"); res.end("{}") })
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", () => { listener.off("error", reject); resolve() }) })
|
||||
@@ -65,12 +65,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const path = "/routing/filter/rule/*1"
|
||||
await assert.rejects(withVerifiedConfigurationChange({ actorId: ownerId, connectionId, approved: true, serverIds: ids, serverWide: true, objects: [],
|
||||
authorize: async () => { await captured.assert() }, beforeDispatch: async () => {
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent' AND payload->>'path'=$2", [connectionId, path])).rows[0].n, 1)
|
||||
durableIntents++
|
||||
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
|
||||
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
|
||||
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
|
||||
await captured.assert()
|
||||
dispatchFences++
|
||||
},
|
||||
}, () => executeGuardedConfigurationWrite(ids[0], "PATCH", path, async () => {
|
||||
callbackDispatches++
|
||||
@@ -78,12 +73,17 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const req = request({ hostname: "127.0.0.1", port, path: "/rest" + path, method: "PATCH", timeout: 1000 }, res => { res.resume(); res.once("end", resolve) })
|
||||
req.once("error", reject); req.once("timeout", () => req.destroy(new Error("Synthetic loopback timeout"))); req.end("{}")
|
||||
})
|
||||
})), /Постоянный допуск/)
|
||||
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.unconfirmed'", [connectionId])).rows[0].n, 1)
|
||||
})), (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT" && /Активный routing profile/.test(error.message))
|
||||
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0); assert.equal(dispatchFences, 0)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action IN ('configuration.intent','configuration.unconfirmed','configuration.acknowledged')", [connectionId])).rows[0].n, 0)
|
||||
|
||||
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
|
||||
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
|
||||
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
|
||||
await assert.rejects(captured.assert(), /Постоянный допуск/)
|
||||
}
|
||||
return { cases: 3, lateMutations: ["edit", "disable", "grant"], deepCaptureCases: 3, durableIntents, callbackDispatches, nativeHttpRequests,
|
||||
productionSessionAuthorityFence: true, realSql: true, productionHmac: true, fullInverseLifecycleRepeated: false, controllerEnabled: false, externalNetwork: false }
|
||||
return { cases: 3, deepCaptureCases: 3, directGuardBeforeIntent: true, dispatchFences, callbackDispatches, nativeHttpRequests,
|
||||
capturedAuthorityChanges: ["edit", "disable", "grant"], realSql: true, productionHmac: true, controllerEnabled: false, externalNetwork: false }
|
||||
} finally {
|
||||
if (profileIds.length) await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=ANY($1::uuid[])", [profileIds])
|
||||
if (ids.length) await pool.query("UPDATE servers SET enabled=false WHERE id=ANY($1::int[])", [ids])
|
||||
|
||||
@@ -2,9 +2,26 @@ import { assertOwnedRoutingRecoveryAuthoritySql, acceptRoutingRecoveryAuthorityS
|
||||
|
||||
// Guard precedes imports that configure a pool or load private MCP state.
|
||||
assertOwnedRoutingRecoveryAuthoritySql()
|
||||
process.env.AUTH_REQUIRED = "true"
|
||||
process.env.AUTH_JWT_SECRET = "owned-routing-recovery-authority-acceptance-secret"
|
||||
process.env.AUTH_ISSUER = "https://owned-routing-recovery-authority.test"
|
||||
const { pool, closePool } = await import("../../db/index.js")
|
||||
try {
|
||||
const { applySqlMigrations } = await import("../../db/migrate.js")
|
||||
await applySqlMigrations(pool)
|
||||
console.log("captured recovery authority SQL acceptance:", JSON.stringify(await acceptRoutingRecoveryAuthoritySql()))
|
||||
} finally { await closePool() }
|
||||
const { updateMcpSettings } = await import("./store.js")
|
||||
await import("./actions.js")
|
||||
const { acceptScopedRouteCommunitySwitch } = await import("./routing-community-scoped-acceptance.js")
|
||||
const actor = { id: "owned-routing-recovery-authority", email: "[email protected]", name: "Owned recovery authority",
|
||||
apps: ["mm"], permissions: ["mm:settings:admin", "mm:mcp:admin", "mm:mcp:approve", "mm:network:read", "mm:network:write", "mm:filters:write", "mm:servers:admin"],
|
||||
issuedAt: Date.now() - 1000, expiresAt: Date.now() + 900000 }
|
||||
await updateMcpSettings({ enabled: true }, actor)
|
||||
for (const lateInverseAuthorityChange of ["edit", "disable", "grant"] as const) {
|
||||
const evidence = await acceptScopedRouteCommunitySwitch(actor, { lateInverseAuthorityChange })
|
||||
console.log("genuine late inverse authority SQL acceptance:", JSON.stringify(evidence))
|
||||
}
|
||||
} finally {
|
||||
await (await import("../../services/scheduler.js")).stopSchedulerForDatabaseOperation()
|
||||
await closePool()
|
||||
}
|
||||
@@ -352,6 +352,47 @@ export async function acceptRoutingReservationSql(options: { runtimeAdmission?:
|
||||
await assert.rejects(store.beginForwardAttempt(reservation.forwardAllocation), /capability/)
|
||||
names.push("spent forward/no genuine checkpoint and preterminal unused-proof refuse; unknown never replays or refunds")
|
||||
}
|
||||
{
|
||||
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
|
||||
const { saveScopedConfigurationCheckpoint, saveScopedNetworkCheckpoint } = await import("./routing-community-scoped-checkpoint.js")
|
||||
const { executeReservedScopedRecovery } = await import("./routing-community-scoped-recovery.js")
|
||||
await pool.query("UPDATE mcp_operations SET state='running' WHERE id=$1", [source.operationId])
|
||||
const step = { kind: "service", service: "routing.community.scoped", input: source.input }
|
||||
await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state) VALUES($1,0,'intent',$2::jsonb)",
|
||||
[source.operationId, JSON.stringify(security.redact(step))])
|
||||
await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,0,$2)",
|
||||
[source.operationId, await security.encryptSecret(JSON.stringify(source.input.beforeRules))])
|
||||
assert(reservation.forwardAllocation)
|
||||
await store.beginForwardAttempt(reservation.forwardAllocation)
|
||||
// Use the actual checkpoint writers for production crypto and SQL, but
|
||||
// deliberately seed a synthetic failed-forward observation. No PATCH
|
||||
// or genuine network observation is part of this owned component case.
|
||||
const requestSeal = String(source.input.requestProof)
|
||||
const configuration = await saveScopedConfigurationCheckpoint(source.input, requestSeal)
|
||||
await saveScopedNetworkCheckpoint(source.input, requestSeal, { id: randomUUID(), candidateId: "b",
|
||||
status: "failed", observedAt: configuration.observedAt })
|
||||
const published = await store.recordForwardOutcome(reservation.reservationId)
|
||||
assert(published.allocation)
|
||||
assert.equal(published.receipt.state.inverse, "held")
|
||||
const refused = { sourceOperationId: randomUUID(), principal: f.principal }
|
||||
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /Reserved runner identity differs/)
|
||||
const unknown = await store.read(reservation.reservationId, f.principal)
|
||||
assert.deepEqual(unknown.state, { ...published.receipt.state, revision: published.receipt.state.revision + 2, inverse: "unknown" })
|
||||
assert.equal(unknown.chargedMutationUnits, 2)
|
||||
assert.equal(unknown.chargedProfileSlots, 2)
|
||||
assert.equal(unknown.refundsAvailable, false)
|
||||
const shared = (await pool.query(`SELECT state,slot_phase,inverse_intent_sealed,inverse_configuration_sealed,verified_sealed
|
||||
FROM mcp_scoped_recovery_intents WHERE reservation_id=$1`, [reservation.reservationId])).rows
|
||||
assert.deepEqual(shared, [{ state: "unknown", slot_phase: "reserved-attempt", inverse_intent_sealed: null,
|
||||
inverse_configuration_sealed: null, verified_sealed: null }])
|
||||
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /capability/)
|
||||
await assert.rejects(store.recordForwardOutcome(reservation.reservationId), /republished/)
|
||||
await assert.rejects(store.proveUnusedInverse(reservation.reservationId), /terminal applied/)
|
||||
assert.equal(await used(f.connectionId), 2)
|
||||
assert.equal((await store.readHourlyCharges(f.profileId, f.principal)).reservedCharges, 2)
|
||||
assert.deepEqual((await store.read(reservation.reservationId, f.principal)).state, unknown.state)
|
||||
names.push("genuine inverse capability on synthetic failed-forward checkpoints: acknowledged spend, late identity refusal, signed same-slot unknown and both charges conserved without replay or refund")
|
||||
}
|
||||
{
|
||||
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
|
||||
const saved = (await pool.query("SELECT state_proof FROM mcp_routing_reservations WHERE id=$1", [reservation.reservationId])).rows[0].state_proof
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { pollErrorLog } from "../../services/server-poll-error-provenance.js"
|
||||
import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js"
|
||||
import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js"
|
||||
import { preparedPlanExpiry } from "./plan-expiry.js"
|
||||
import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js"
|
||||
import { randomUUID, createHash } from "node:crypto"
|
||||
@@ -23,6 +25,7 @@ import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOpera
|
||||
import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js"
|
||||
import { evaluateMcpPolicy } from "./policy.js"
|
||||
import { findAction, findActionByOperation } from "./actions.js"
|
||||
import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js"
|
||||
import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
|
||||
import { audit, getConnection, transaction, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js"
|
||||
import { executeRosStep, readRosSnapshot, resourceById, routerClient, type RosSnapshot } from "./resources.js"
|
||||
@@ -291,6 +294,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
|
||||
reservedRouting = !!await routingReservationEvidence(original, originalEvidence)
|
||||
return existing
|
||||
}
|
||||
await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey)
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))")
|
||||
const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')")
|
||||
if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429)
|
||||
@@ -359,6 +363,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
|
||||
async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise<void> {
|
||||
return withMcpOperationExecutionOrigin(operationAdmission, async () => {
|
||||
const lease = await pool.connect()
|
||||
let terminalProbe: Awaited<ReturnType<typeof executeOperationWithLease>>
|
||||
let locked = false, released = false
|
||||
const ownerLifetime = new AbortController()
|
||||
// This closure is the SINGLE owner of physical destruction/release.
|
||||
@@ -380,7 +385,7 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
|
||||
throw error
|
||||
}
|
||||
if (released || !locked) return
|
||||
await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
|
||||
terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
|
||||
() => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant))
|
||||
} finally {
|
||||
if (!released && locked) {
|
||||
@@ -395,9 +400,14 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
|
||||
lease.release()
|
||||
}
|
||||
}
|
||||
// Capture has its own bounded checkout and may fail without rewriting the
|
||||
// acknowledged terminal outcome or redispatching the diagnostic.
|
||||
if (terminalProbe) {
|
||||
try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ }
|
||||
}
|
||||
})
|
||||
}
|
||||
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant): Promise<void> {
|
||||
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) {
|
||||
const { rows: [operation] } = await pool.query<StoredOperation>("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId])
|
||||
if (!operation) return
|
||||
const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence
|
||||
@@ -576,14 +586,18 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
})
|
||||
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks())
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await transaction(async client => {
|
||||
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
|
||||
const finishedAt = await transaction(async client => {
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await client.query("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1", [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
|
||||
const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1"
|
||||
+ (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""),
|
||||
[operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id])
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) })
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
return receiptId ? terminal.rows[0]?.finished_at : undefined
|
||||
})
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
if (routingReservationId && reservedForwardPassed) {
|
||||
@@ -592,6 +606,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {})
|
||||
}
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt }
|
||||
} catch (error) {
|
||||
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
|
||||
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
|
||||
|
||||
@@ -12,9 +12,9 @@ const principal = { connectionId, credentialId, credentialExpiresAt: new Date(no
|
||||
const proof = async (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
|
||||
const context = { principal, connection: { id: connectionId, owner_id: "owner", owner_claims_issued_at: new Date(now - 1000) } } as ActionContext
|
||||
let assertions = 0
|
||||
function fixture() {
|
||||
function fixture(retiredSource?: ServerProbeMcpDependencies["retiredSource"]) {
|
||||
const events: string[] = [], receipts = new Map<string, ServerProbeReceipt>(), contexts: ServerProbeJournalContext[] = [], sealed = new Map<string, string>()
|
||||
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now
|
||||
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now, liveHistory = true
|
||||
const access: ServerProbeAuthority = { transport: "mcp", ownerId: "owner", connectionId, credentialId, expiresAt: now + 600000, generation: "generation", policyProof: "1".repeat(64), ownerProof: "2".repeat(64), active: true, operationAllowed: true, scopeServerIds: scope, ownerPermissions: ["mm:servers:write"], policyPermissions: ["mm:servers:write"] }
|
||||
const readAuthority = async () => { events.push("authority"); if (revoked) throw new ServerProbeExecutionError("PERMISSION_DENIED", "PRIVATE_PROVIDER_SECRET", 403); return { ...access, expiresAt: clock + 600000, scopeServerIds: [...scope] } }
|
||||
const assertProtections = async (ids: number[]) => { events.push("protections"); if (ids.some(id => !scope.includes(id))) throw new Error("out of scope") }
|
||||
@@ -28,6 +28,7 @@ function fixture() {
|
||||
read: async id => { events.push("journal.read"); return receipts.get(id) },
|
||||
}
|
||||
const deps: ServerProbeMcpDependencies = {
|
||||
retiredSource,
|
||||
authority: (() => ({ readAuthority, assertProtections })) as ServerProbeMcpDependencies["authority"],
|
||||
historicalAuthority: ((currentPrincipal: typeof principal) => ({ readAuthority: async () => { events.push("read-authority"); return { ...await readAuthority(), connectionId: currentPrincipal.connectionId, credentialId: currentPrincipal.credentialId } }, assertProtections })) as ServerProbeMcpDependencies["historicalAuthority"],
|
||||
history: dependencies => createServerProbeJournalHistoryReader({ ...dependencies, loadArchive: async binding => {
|
||||
@@ -51,7 +52,7 @@ function fixture() {
|
||||
journal: (ctx, dependencies, execution) => { assert.equal(execution?.deadlineAt, Date.parse(prepared().expiresAt)); contexts.push(ctx); return { ...journal, read: async (...args) => { await dependencies.assertAuthority(ctx, args[1]); return journal.read(...args) }, claim: async binding => { await dependencies.assertAuthority(ctx, binding.bindingProof); return journal.claim(binding) } } },
|
||||
query: async (sql, values) => {
|
||||
events.push("sql"); if (revokeOnQuery) revoked = true
|
||||
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] }
|
||||
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return liveHistory ? [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] : [] }
|
||||
if (sql.includes("FROM servers")) { assert.ok(sql.includes("ANY($1::int[])")); assert.deepEqual(values[0], scope); return sql.includes("AND id=$2") ? [] : [{ id: 2, host: "peer", enabled: true }] }
|
||||
if (!sql.includes("mcp_operation_steps")) return [{ evidence: { sealed: await seal(JSON.stringify(evidence)) } }]
|
||||
assert.equal(values[0], connectionId); assert.equal(values[1], "owner"); assert.equal(values[2], "servers.probes.run")
|
||||
@@ -68,8 +69,9 @@ function fixture() {
|
||||
}, seal, open, proof, now: () => clock,
|
||||
}
|
||||
const adapter = createServerProbeMcpAdapter(deps)
|
||||
const retireHistory = () => { liveHistory = false }
|
||||
const prepare = async (tool: PreparedServerProbe["tool"] = "ping") => { evidence = await adapter.prepare({ serverId: 9, request: tool === "bandwidth" ? { tool, bwRemoteAddress: "peer", dstServerId: 8 } : { tool, target: "192.0.2.1" } }, context); const step = evidence.steps[0]; assert.equal(step.kind, "service"); return step.kind === "service" ? step.input : assert.fail() }
|
||||
return { adapter, events, receipts, contexts, prepare, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
|
||||
return { adapter, events, receipts, contexts, prepare, retireHistory, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
|
||||
}
|
||||
|
||||
for (const tool of ["ping", "traceroute", "bandwidth", "dns", "route", "mtu"] as const) {
|
||||
@@ -128,6 +130,38 @@ for (const input of [{ serverId: 9, request: { tool: "ping", target: "x", creden
|
||||
await assert.rejects(f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0, signal: controller.signal })); assert.equal(f.events.filter(event => event === "execute").length, attempted); assertions += 3
|
||||
}
|
||||
assert.ok(!("diagnosticProgress" in serverProbeSafeMcpError(new ServerProbeExecutionError("X", "secret", 500, "unknown", 99), { receiptId, operationId, stageBudget: 1 }))); assertions++
|
||||
{
|
||||
const hash = "3".repeat(64), date = (value: number) => new Date(value).toISOString().replace("Z", "000Z")
|
||||
const retired = { source: { version: 1 as const, operation: "servers.probes.run" as const, operationVersion: 1 as const, tool: "ping" as const,
|
||||
receiptId, executionId: operationId, connectionId, ownerId: "owner", planId: randomUUID(), originalCredentialId: credentialId,
|
||||
sourceServerId: 9, serverIds: [9], issuedAt: now - 1000, authorityExpiresAt: date(now + 600000), terminalAt: date(now + 600000),
|
||||
activityAt: date(now + 600000), gcNotBefore: date(now + 87060000), retentionDays: 1, safetyMs: 60000,
|
||||
bindingProof: hash, sourceDigest: hash, terminalDigest: hash, stageSnapshotDigest: hash, receiptProof: hash, ciphertextSha256: hash,
|
||||
}, payloadPurged: false }
|
||||
let lookups = 0, changeOnRecheck = false
|
||||
const f = fixture(async lookup => {
|
||||
lookups++
|
||||
assert.deepEqual(lookup, { receiptId, operationId, connectionId, ownerId: "owner", scopeServerIds: [2, 9] })
|
||||
return changeOnRecheck && lookups % 2 === 0 ? undefined : retired
|
||||
})
|
||||
const input = await f.prepare(), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 })
|
||||
f.setResult(output); f.finishOperation()
|
||||
const request = { operationId, receiptId, page: 0 }
|
||||
const live = await f.adapter.readPage(request, context)
|
||||
assert.equal(lookups, 0)
|
||||
f.retireHistory()
|
||||
const rotated = { ...context, principal: { ...principal, credentialId: randomUUID() } }
|
||||
assert.deepEqual(await f.adapter.readPage(request, rotated), live)
|
||||
assert.deepEqual(await f.adapter.receiptTarget(request, rotated), { scope: "servers", serverIds: [9] })
|
||||
const beforeForeign = lookups
|
||||
await assert.rejects(f.adapter.readPage(request, { ...rotated, connection: { ...context.connection, owner_id: "foreign" } }))
|
||||
assert.equal(lookups, beforeForeign)
|
||||
lookups = 0; changeOnRecheck = true
|
||||
const privateReads = f.events.filter(event => event === "history.read").length
|
||||
await assert.rejects(f.adapter.readPage(request, rotated))
|
||||
assert.equal(f.events.filter(event => event === "history.read").length, privateReads)
|
||||
assertions += 7
|
||||
}
|
||||
{
|
||||
const f = fixture(), input = await f.prepare("bandwidth"), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 }); f.setResult(output); f.finishOperation(); f.expirePrepared()
|
||||
await assert.rejects(f.adapter.service.current(input))
|
||||
|
||||
@@ -10,6 +10,7 @@ import { serverProbeReceiptPage } from "../../services/server-probe-public-resul
|
||||
import { decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
|
||||
import type { ActionContext, ChangeEvidence } from "./action-types.js"
|
||||
import type { McpServiceHandler } from "./service-registry.js"
|
||||
import type { findRetiredServerProbeHistory, ServerProbeRetiredLookup } from "../../services/server-probe-retired-history.js"
|
||||
|
||||
/** Foundations only: importing this module registers no tools or services. */
|
||||
export const serverProbeActionSchema = z.object({ serverId: z.number().int().positive().safe(), request: serverProbeExecutionRequest }).strict()
|
||||
@@ -32,6 +33,7 @@ export interface ServerProbeMcpDependencies {
|
||||
executor: typeof createProductionServerProbeExecutor
|
||||
journal: (context: ServerProbeJournalContext, dependencies: ServerProbeJournalDependencies, execution?: { signal?: AbortSignal; deadlineAt?: number }) => ServerProbeJournal
|
||||
query: Query
|
||||
retiredSource?: (lookup: ServerProbeRetiredLookup) => ReturnType<typeof findRetiredServerProbeHistory>
|
||||
seal: (value: string) => Promise<string>
|
||||
open: (value: string) => Promise<string>
|
||||
proof: (value: unknown) => Promise<string>
|
||||
@@ -52,7 +54,7 @@ export async function openServerProbeMcpBinding(raw: Record<string, unknown>, cr
|
||||
return { input, binding }
|
||||
}
|
||||
const ids = (prepared: PreparedServerProbe) => [...new Set([prepared.serverId, ...(prepared.destinationServerId ? [prepared.destinationServerId] : [])])].sort((a, b) => a - b)
|
||||
function metadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
|
||||
export function serverProbeReceiptMetadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
|
||||
const page = serverProbeReceiptPage(receipt, 0)
|
||||
if (!Number.isSafeInteger(receipt.attemptedStages) || !Number.isSafeInteger(receipt.completedStages) || receipt.completedStages < 0 || receipt.completedStages > receipt.attemptedStages || receipt.attemptedStages > stageBudget) return denied()
|
||||
return { operationId, receiptId: receipt.receiptId, tool: receipt.tool, status: receipt.state, httpStatus: receipt.httpStatus, stageBudget, attemptedStages: receipt.attemptedStages, completedStages: receipt.completedStages, sha256: page.sha256, totalPages: page.totalPages, bytes: page.bytes, automaticReplay: false, forwardingVerified: false }
|
||||
@@ -119,7 +121,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
await owned(input, binding, row.operation_id)
|
||||
await preflight(binding, signal)
|
||||
if (!receipt || receipt.receiptId !== input.receiptId || receipt.serverId !== binding.prepared.serverId || receipt.destinationServerId !== binding.prepared.destinationServerId || receipt.tool !== binding.prepared.tool) return denied()
|
||||
return { row, receipt, result: metadata(receipt, row.operation_id, binding.stageBudget) }
|
||||
return { row, receipt, result: serverProbeReceiptMetadata(receipt, row.operation_id, binding.stageBudget) }
|
||||
}
|
||||
const target = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
|
||||
const input = serverProbeActionSchema.parse(raw), access = deps.authority(context.principal, { signal: context.signal })
|
||||
@@ -177,7 +179,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
const { executor } = execution(input, binding, row, true, context.signal)
|
||||
const receipt = await executor.execute(binding.prepared, context.signal)
|
||||
await preflight(binding, context.signal); await owned(input, binding, context.operationId, true); await preflight(binding, context.signal)
|
||||
return metadata(receipt, context.operationId, binding.stageBudget)
|
||||
return serverProbeReceiptMetadata(receipt, context.operationId, binding.stageBudget)
|
||||
} catch (error) { throw serverProbeSafeMcpError(error, progress) }
|
||||
},
|
||||
verify: async (raw, result) => {
|
||||
@@ -200,19 +202,47 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
AND j.source_server_id=ANY($6::int[]) AND (j.destination_server_id IS NULL OR j.destination_server_id=ANY($6::int[])) LIMIT 2`,
|
||||
[request.receiptId, request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION, initial.scopeServerIds])
|
||||
await fresh()
|
||||
if (rows.length !== 1) return denied()
|
||||
const row = rows[0], serverIds = [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
|
||||
const loadRetired = deps.retiredSource
|
||||
const readRetired = rows.length === 0 && loadRetired ? () => loadRetired({
|
||||
receiptId: request.receiptId, operationId: request.operationId, connectionId: context.principal.connectionId,
|
||||
ownerId: initial.ownerId, scopeServerIds: initial.scopeServerIds,
|
||||
}) : undefined
|
||||
const retired = readRetired ? await readRetired() : undefined
|
||||
if (readRetired) await fresh()
|
||||
if (rows.length !== 1 && !retired) return denied()
|
||||
const row = rows[0]
|
||||
const serverIds = retired ? retired.source.serverIds : [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
|
||||
const fence = async () => { const current = await fresh(); if (serverIds.some(id => !current.scopeServerIds.includes(id))) return denied(); await access.assertProtections(serverIds, current); await fresh() }
|
||||
await fence()
|
||||
return { initial, serverIds, fence }
|
||||
const retiredFence = retired && readRetired ? async () => {
|
||||
await fence()
|
||||
const current = await readRetired()
|
||||
await fence()
|
||||
if (!current || fingerprint(current) !== fingerprint(retired)) return denied()
|
||||
} : undefined
|
||||
return { initial, serverIds, fence, retired, retiredFence }
|
||||
}
|
||||
const receiptTarget = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence } = await historicalAccess(request, context)
|
||||
await fence(); return { scope: "servers", serverIds }
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence, retiredFence } = await historicalAccess(request, context)
|
||||
await (retiredFence ?? fence)(); return { scope: "servers", serverIds }
|
||||
}
|
||||
const readPage = async (raw: Record<string, unknown>, context: ActionContext) => {
|
||||
try {
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence } = await historicalAccess(request, context)
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence, retired, retiredFence } = await historicalAccess(request, context)
|
||||
if (retired && retiredFence) {
|
||||
const source = retired.source
|
||||
const originalContext: ServerProbeJournalContext = {
|
||||
identity: { kind: "mcp", ownerId: source.ownerId, executionId: source.executionId, connectionId: source.connectionId, credentialId: source.originalCredentialId,
|
||||
issuedAt: source.issuedAt, expiresAt: Date.parse(source.authorityExpiresAt), operationName: SERVER_PROBE_OPERATION },
|
||||
sourceServerId: source.sourceServerId, ...(source.destinationServerId !== undefined ? { destinationServerId: source.destinationServerId } : {}), tool: source.tool,
|
||||
}
|
||||
await retiredFence()
|
||||
const reader = deps.history({ assertReadAuthority: retiredFence }, { signal: context.signal })
|
||||
const receipt = await reader.read({ context: originalContext, receiptId: source.receiptId, bindingProof: source.bindingProof })
|
||||
const page = serverProbeReceiptPage(receipt, request.page)
|
||||
await retiredFence()
|
||||
return page
|
||||
}
|
||||
await fence()
|
||||
const rows = await deps.query(`SELECT p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN mcp_connections c ON c.id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2 AND c.owner_id=$3 AND p.operation_id=$4 AND p.operation_version=1 AND octet_length(p.evidence::text)<=2097152 LIMIT 2`, [request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION])
|
||||
await fence(); if (rows.length !== 1) return denied()
|
||||
|
||||
@@ -171,6 +171,16 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
assert.equal(done.result.confirmedWrites, 0); assert.equal(done.result.completedServiceSteps, 1); assert.equal(done.result.verificationScope, "acknowledgement")
|
||||
const receipt = done.result.diagnosticReceipt as { receiptId: string; stageBudget: number; attemptedStages: number; completedStages: number; totalPages: number; sha256: string }
|
||||
assert(receipt); assert.equal(receipt.attemptedStages, receipt.completedStages); assert(receipt.attemptedStages > 0 && receipt.attemptedStages <= receipt.stageBudget)
|
||||
let captured = false
|
||||
for (const until = Date.now()+2000; Date.now()<until;) {
|
||||
const proof = (await pool.query(`SELECT f.source->>'terminalAt'=to_char(GREATEST(o.finished_at,p.expires_at) AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS terminal_matches FROM server_probe_terminal_proofs f
|
||||
JOIN mcp_operations o ON o.id=f.operation_id JOIN mcp_plans p ON p.id=o.plan_id
|
||||
WHERE f.operation_id=$1 AND f.receipt_id=$2`, [done.id,receipt.receiptId])).rows[0]
|
||||
if (proof) { assert.equal(proof.terminal_matches,true); captured = true; break }
|
||||
await delay(20)
|
||||
}
|
||||
assert.equal(captured,true,"Owned successful diagnostic did not publish its terminal proof")
|
||||
const step = (await pool.query("SELECT state,result FROM mcp_operation_steps WHERE operation_id=$1", [done.id])).rows
|
||||
assert.equal(step.length, 1); assert.equal(step[0].state, "acknowledged")
|
||||
for (const field of ["output", "data", "contentB64", "bindingCiphertext"]) assert(!Object.hasOwn(step[0].result, field))
|
||||
@@ -201,6 +211,49 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
cases.push(tool + ": actual public plan/approval/apply, receipt metadata, full stage ledger and checksum pages without redispatch")
|
||||
}
|
||||
|
||||
// Force a genuine capture SQL failure after the real diagnostic has
|
||||
// committed. A sequence records the rejected attempt despite rollback.
|
||||
const captureFault = await fixture()
|
||||
const faultPlan = await prepare(captureFault, { tool: "route", target: "127.0.0.1" })
|
||||
await pool.query("CREATE SEQUENCE public.owned_probe_capture_failure")
|
||||
try {
|
||||
await pool.query(`CREATE FUNCTION public.owned_probe_capture_failure() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.source->>'connectionId'=TG_ARGV[0] THEN
|
||||
PERFORM nextval('public.owned_probe_capture_failure');
|
||||
RAISE EXCEPTION 'owned terminal proof insertion refused';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
try {
|
||||
assert.match(captureFault.principal.connectionId,/^[a-f0-9-]{36}$/)
|
||||
await pool.query(`CREATE TRIGGER owned_probe_capture_failure BEFORE INSERT ON public.server_probe_terminal_proofs
|
||||
FOR EACH ROW EXECUTE FUNCTION public.owned_probe_capture_failure('${captureFault.principal.connectionId}')`)
|
||||
try {
|
||||
const beforeFault = requests+dnsUdp+dnsTcp, faultKey=randomUUID()
|
||||
const applied = await applyPlan(captureFault.principal,faultPlan,faultKey), done=await terminal(applied.id)
|
||||
let attempted = false
|
||||
for (const until=Date.now()+2500;Date.now()<until;) {
|
||||
attempted = (await pool.query("SELECT is_called FROM public.owned_probe_capture_failure")).rows[0].is_called
|
||||
if (attempted) break
|
||||
await delay(20)
|
||||
}
|
||||
assert.equal(attempted,true,"Owned post-COMMIT capture insert must have reached the injected SQL failure")
|
||||
assert.equal(done.state,"applied"); assert.equal(done.error_code,null)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1",[applied.id])).rows.length,0)
|
||||
const originalReceipt = done.result.diagnosticReceipt as { receiptId:string; sha256:string }
|
||||
const page = await invokeMcpTool(captureFault.principal,"mm_server_probe_receipt_get",
|
||||
{ operationId:applied.id,receiptId:originalReceipt.receiptId,page:0 }) as {sha256:string}
|
||||
assert.equal(page.sha256,originalReceipt.sha256)
|
||||
assert.equal((await applyPlan(captureFault.principal,faultPlan,faultKey)).id,applied.id)
|
||||
assert.equal((await terminal(applied.id)).state,"applied")
|
||||
assert.equal(requests+dnsUdp+dnsTcp,beforeFault+1)
|
||||
assert.equal((await pool.query("SELECT last_value::int AS attempts FROM public.owned_probe_capture_failure")).rows[0].attempts,1)
|
||||
} finally { await pool.query("DROP TRIGGER owned_probe_capture_failure ON public.server_probe_terminal_proofs") }
|
||||
} finally { await pool.query("DROP FUNCTION public.owned_probe_capture_failure()") }
|
||||
} finally { await pool.query("DROP SEQUENCE public.owned_probe_capture_failure") }
|
||||
cases.push("terminal proof SQL refusal: acknowledged result remains applied and readable, exactly one loopback measurement and no retry")
|
||||
|
||||
// Pause only the one runner's pending->running SQL BEFORE sending it. This
|
||||
// leaves a real pending approved source, allowing genuine durable cancel.
|
||||
// A server-table lock after 'running' would have no intent step yet and the
|
||||
@@ -209,8 +262,9 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
const entered = gate(), resume = gate(); releasePending = resume.resolve
|
||||
let heldOperation = "", held = false
|
||||
pool.query = (async function (...args: unknown[]) {
|
||||
if (!held && typeof args[0] === "string" && args[0].startsWith("UPDATE mcp_operations SET state='running'")) {
|
||||
held = true; heldOperation = String((args[1] as unknown[])[0]); entered.resolve(); await resume.promise
|
||||
const query = args[0] as { text?: string; values?: unknown[] } | undefined
|
||||
if (!held && query?.text?.startsWith("/* mcp-operation-root:claim */") && typeof query.values?.[0] === "string") {
|
||||
held = true; heldOperation = query.values[0]; entered.resolve(); await resume.promise
|
||||
}
|
||||
return Reflect.apply(originalQuery, pool, args)
|
||||
}) as typeof pool.query
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { createMcpServerProbeAuthority, createMcpServerProbeReadAuthority } from "../../services/server-probe-authority.js"
|
||||
import { createProductionServerProbeExecutor } from "../../services/server-probe-execution.js"
|
||||
import { createServerProbeStorageHistoryReader } from "../../services/server-probe-storage-history.js"
|
||||
import { findRetiredServerProbeHistory } from "../../services/server-probe-retired-history.js"
|
||||
import { createSqlServerProbeSourceReaders } from "../../services/server-probe-storage-source.js"
|
||||
import { createProbeStorageHostIo } from "../../services/server-probe-storage-host-io.js"
|
||||
import { assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js"
|
||||
@@ -28,6 +29,7 @@ export function createStorageServerProbeMcpAdapter(limits: ProbeStorageLimits =
|
||||
},
|
||||
}
|
||||
const adapter = createServerProbeMcpAdapter({ query: io.rows, journal: io.journal,
|
||||
retiredSource: lookup => findRetiredServerProbeHistory(io.rows, lookup),
|
||||
authority: (principal, options) => createMcpServerProbeAuthority(principal, options, authorityDependencies),
|
||||
historicalAuthority: (principal, options) => createMcpServerProbeReadAuthority(principal, options, authorityDependencies),
|
||||
history: (dependencies, execution) => createServerProbeStorageHistoryReader({ assertReadAuthority: dependencies.assertReadAuthority,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
|
||||
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
|
||||
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
|
||||
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
|
||||
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
||||
import { recoverMcpOperations, verifyStep } from "./runtime.js"
|
||||
import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js"
|
||||
import { audit, getMcpSettings } from "./store.js"
|
||||
import { accessGeneration, decryptSecret, redact } from "./security.js"
|
||||
import { tickRoutingProfiles } from "./routing-profiles.js"
|
||||
@@ -48,11 +51,22 @@ export async function detectMcpDrift() {
|
||||
}
|
||||
}
|
||||
|
||||
const mcpRetentionBatchSize = 1000
|
||||
|
||||
async function retainMcpData() {
|
||||
const settings = await getMcpSettings()
|
||||
await pool.query("DELETE FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')", [settings.requestsRetentionDays])
|
||||
await pool.query("DELETE FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)", [settings.auditRetentionDays])
|
||||
await pool.query("DELETE FROM mcp_artifacts WHERE expires_at<now()")
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')
|
||||
ORDER BY created_at,id LIMIT $2 FOR UPDATE
|
||||
) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [settings.requestsRetentionDays, mcpRetentionBatchSize])
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)
|
||||
ORDER BY created_at,id LIMIT $2 FOR UPDATE
|
||||
) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [settings.auditRetentionDays, mcpRetentionBatchSize])
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_artifacts WHERE expires_at<now()
|
||||
ORDER BY expires_at,id LIMIT $1 FOR UPDATE
|
||||
) DELETE FROM mcp_artifacts USING expired WHERE mcp_artifacts.id=expired.id`, [mcpRetentionBatchSize])
|
||||
await pool.query("DELETE FROM mcp_oauth_requests WHERE expires_at<now()-interval '1 day'")
|
||||
await pool.query("DELETE FROM mcp_budget_buckets WHERE bucket<now()-interval '2 days'")
|
||||
await routingReservationTransaction(db => retainManualInverseChargesWithinOwner(db, settings.requestsRetentionDays))
|
||||
@@ -79,8 +93,11 @@ export async function tickMcpWorkers() {
|
||||
await retainMcpData()
|
||||
await retainServerProbeStorage()
|
||||
await compactCompletedServerPollStorage()
|
||||
await retireCompletedServerPollOperations()
|
||||
await collectExpiredServerPollStorage()
|
||||
await retireOrdinaryMcpOperations()
|
||||
if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() }
|
||||
await maintainServerProbeOperationRetirement()
|
||||
} finally {
|
||||
if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true }
|
||||
lease.release(unsafe)
|
||||
|
||||
@@ -18,6 +18,7 @@ import { accessGeneration, decryptSecret, encryptSecret, McpError, randomToken,
|
||||
import { createMcpConnection, listMcpConnections, getMcpSettings, updateMcpSettings, getConnection, assertConnectionVisible, rotateConnectionToken, setConnectionStatus, updateMcpConnection, transaction, audit } from "../modules/mcp/store.js"
|
||||
import { approvePlan, loadPlan, publicPlan } from "../modules/mcp/runtime.js"
|
||||
import { applyPlan, invokeMcpTool } from "../modules/mcp/runtime.js"
|
||||
import { findOwnerPlanReplayTombstone } from "../modules/mcp/operation-replay-tombstones.js"
|
||||
import { routeAiProfileSchema } from "@mmapp/contracts/mcp-routing"
|
||||
import { revalidateMcpOwner } from "../modules/mcp/store.js"
|
||||
import { resourceById } from "../modules/mcp/resources.js"
|
||||
@@ -243,7 +244,16 @@ export function createMcpRoutes(checkClaims = assertPortalClaimsCurrent, options
|
||||
return ownerDelegate(async () => invokeMcpTool(principal, action === "recover" ? "mm_operation_recovery_plan" : globalReconcile ? "mm_global_operation_reconcile" : "mm_operation_reconcile", { operationId: id }))
|
||||
})
|
||||
app.post("/mcp/plans/:id/apply", async request => {
|
||||
const plan = await loadPlan(idOf(request.params)), actor = actorOf(request.authUser)
|
||||
const id = idOf(request.params), actor = actorOf(request.authUser)
|
||||
let plan: { id: string; connection_id: string; operation_id: string }
|
||||
try { plan = await loadPlan(id) }
|
||||
catch (error) {
|
||||
if (!(error instanceof McpError) || error.code !== "NOT_FOUND") throw error
|
||||
const tombstone = await findOwnerPlanReplayTombstone(pool, actor.id, id)
|
||||
if (!tombstone) throw error
|
||||
const principal = await ownerPrincipal(tombstone.connection_id, actor, tombstone.action_id)
|
||||
return ownerDelegate(async () => applyPlan(principal, tombstone.plan_id, `web:${tombstone.plan_id}`))
|
||||
}
|
||||
const principal = await ownerPrincipal(plan.connection_id, actor, plan.operation_id)
|
||||
await audit(pool, actor.id, "plan.web.apply", plan.connection_id, { planId: plan.id })
|
||||
return ownerDelegate(async () => applyPlan(principal, plan.id, `web:${plan.id}`))
|
||||
|
||||
@@ -6,10 +6,11 @@ import { observedBoolean, observedDurationMs, observedInteger } from "./ospf-obs
|
||||
import { readGreNativeRows } from "./gre-observed-state.js"
|
||||
import { greAddressKey } from "./entity-snapshots.js"
|
||||
import type { GreMutation } from "./gre-mutation-input.js"
|
||||
import { greDependencyPaths } from "./routeros-native-paths.js"
|
||||
export { greDependencyPaths } from "./routeros-native-paths.js"
|
||||
|
||||
export const greMutationProblem = (message: string) => Object.assign(new Error(message), { statusCode: 409 })
|
||||
export type GreNativeRow = Record<string, string>
|
||||
export const greDependencyPaths = ["/interface", "/interface/bridge/port", "/interface/vlan", "/interface/list/member", "/ip/vrf", "/ip/route", "/ipv6/route", "/routing/filter/rule", "/routing/ospf/interface-template", "/routing/bfd/configuration", "/routing/bgp/connection", "/routing/rule", ...["ip","ipv6"].flatMap(family=>["filter","nat","mangle","raw"].map(table=>`/${family}/firewall/${table}`))] as const
|
||||
export interface GreMutationState {
|
||||
serverConfiguration: string; gre: GreNativeRow[]; ipv4: GreNativeRow[]; ipv6: GreNativeRow[];
|
||||
restoreProtectionProof?:string;
|
||||
|
||||
@@ -617,10 +617,7 @@ export function firewallRestPath(
|
||||
return `${root}/${table}`
|
||||
}
|
||||
|
||||
export function encodeRosId(rosId: string): string {
|
||||
const id = rosId.startsWith("*") ? rosId : `*${rosId.replace(/^\*/, "")}`
|
||||
return encodeURIComponent(id)
|
||||
}
|
||||
export { encodeRosId } from "./routeros-native-paths.js"
|
||||
|
||||
// ── MikrotikClient ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
export function encodeRosId(rosId: string): string {
|
||||
const id = rosId.startsWith("*") ? rosId : `*${rosId.replace(/^\*/, "")}`
|
||||
return encodeURIComponent(id)
|
||||
}
|
||||
|
||||
export const greDependencyPaths = ["/interface", "/interface/bridge/port", "/interface/vlan",
|
||||
"/interface/list/member", "/ip/vrf", "/ip/route", "/ipv6/route", "/routing/filter/rule",
|
||||
"/routing/ospf/interface-template", "/routing/bfd/configuration", "/routing/bgp/connection", "/routing/rule",
|
||||
...["ip", "ipv6"].flatMap(family => ["filter", "nat", "mangle", "raw"].map(table => `/${family}/firewall/${table}`))] as const
|
||||
|
||||
export const wireguardInterfaceRuntimeDependencyPaths = ["/routing/bgp/session", "/routing/ospf/neighbor",
|
||||
"/routing/ospf/interface", "/routing/bfd/session"] as const
|
||||
|
||||
export const wireguardInterfaceDependencyPaths = [...new Set<string>([...greDependencyPaths, "/interface/vxlan",
|
||||
"/interface/vrrp", "/interface/bonding", "/interface/eoip", "/interface/gre", "/interface/ipip", "/ip/dhcp-server",
|
||||
"/ip/dhcp-client", "/ipv6/dhcp-server", "/ipv6/dhcp-client", "/ip/ipsec/peer", "/ip/ipsec/policy",
|
||||
...wireguardInterfaceRuntimeDependencyPaths])]
|
||||
@@ -4,6 +4,7 @@ import assert from "node:assert/strict"
|
||||
import { createCipheriv, createDecipheriv, createHmac, createHash, randomBytes, randomUUID } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import { assertServerPollArchiveAccountingForTests } from "./server-poll-archive-accounting.js"
|
||||
import type { ServerPollRetiredArchive } from "./server-poll-retired-archive.js"
|
||||
import { createServerPollTerminalArchiveForTests, type ServerPollTerminalArchive, type ServerPollArchiveSourceMetadata } from "./server-poll-terminal-archive.js"
|
||||
import { PollStorageRefused, SERVER_POLL_STORAGE_POLICY as policy } from "./server-poll-storage-model.js"
|
||||
import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
||||
@@ -19,7 +20,7 @@ const codec = createServerPollTerminalArchiveForTests({
|
||||
verifySource: async source => source.admission.metadata as unknown as ServerPollArchiveSourceMetadata,
|
||||
})
|
||||
const id = (n: number) => "00000000-0000-4000-8000-" + n.toString(16).padStart(12, "0")
|
||||
interface Entry { row: Record<string, unknown>; archive?: ServerPollTerminalArchive }
|
||||
interface Entry { row: Record<string, unknown>; archive?: ServerPollTerminalArchive | ServerPollRetiredArchive }
|
||||
const full = (n: number, owner: string): Entry => ({ row: { admission_receipt_id: id(n), identity_kind: "rest", admission_execution_id: randomUUID(), admission_connection_id: null,
|
||||
admission_owner_id: owner, admission_binding_proof: "a".repeat(64), admission_context_proof: "b".repeat(64), storage_state: "full", compact_at: null,
|
||||
admission_archive_bytes: null, admission_activity_at: null, receipt_id: null, has_children: true } })
|
||||
@@ -34,7 +35,7 @@ async function compact(n: number, owner: string): Promise<Entry> {
|
||||
storage_state: "compacted", compact_at: new Date("2026-10-29T02:00:00Z"), admission_archive_bytes: archive.ciphertext_bytes,
|
||||
admission_activity_at: archive.original_activity_at, admission_expires_at: archive.original_expires_at, asserted_ciphertext_bytes: archive.ciphertext_bytes, has_children: false } }
|
||||
}
|
||||
function fixture(entries: Entry[]) {
|
||||
function fixture(entries: Entry[], retiredCharge?: Parameters<typeof assertServerPollArchiveAccountingForTests>[2]) {
|
||||
const totals = new Map<string, { heavy: number; compact: number; archiveBytes: number; bytes: number; rows: number }>()
|
||||
for (const entry of entries) {
|
||||
for (const scope of ["global", "owner:" + entry.row.admission_owner_id]) {
|
||||
@@ -73,10 +74,35 @@ function fixture(entries: Entry[]) {
|
||||
} }
|
||||
return { entries, usage, state, queries, batches, usageBatches, counts: () => ({ factories, charges }), run: () => assertServerPollArchiveAccountingForTests(sql, async () => {
|
||||
factories++; return async row => { charges++; return codec.verifyCharge(row) }
|
||||
}) }
|
||||
}, retiredCharge) }
|
||||
}
|
||||
const refused = (work: Promise<unknown>) => assert.rejects(work, PollStorageRefused)
|
||||
|
||||
test("mixed V1/V2 allocations authenticate actual stored ciphertext and never fall back across formats", async () => {
|
||||
const { createServerPollRetiredArchiveForTests } = await import("./server-poll-retired-archive.js")
|
||||
const unexpected = async () => { throw new Error("Metadata accounting must not open or write archives") }
|
||||
const v2 = createServerPollRetiredArchiveForTests({
|
||||
proof, decrypt: unexpected, encrypt: unexpected, openV1: unexpected, verifyHistory: unexpected,
|
||||
})
|
||||
const first = await compact(1, "owner"), second = await compact(2, "owner")
|
||||
const { archive_proof: _v1Proof, ciphertext, ...previous } = second.archive!
|
||||
const metadata = { ...previous, version: 2 as const }
|
||||
const archiveProof = await proof("server-poll-retired-archive-record-v2", { ...metadata, identity_kind: "mcp",
|
||||
storage_state: "compacted", admission_bytes: 32768, charged_rows: 2 })
|
||||
second.archive = { ...metadata, ciphertext, archive_proof: archiveProof }
|
||||
Object.assign(second.row, { version: 2, archive_proof: archiveProof })
|
||||
const f = fixture([first, second], v2.verifyCharge)
|
||||
await f.run()
|
||||
assert.deepEqual(f.counts(), { factories: 1, charges: 1 })
|
||||
second.archive.ciphertext = "A" + ciphertext.slice(1)
|
||||
await refused(f.run())
|
||||
assert.deepEqual(f.counts(), { factories: 2, charges: 2 })
|
||||
second.archive.ciphertext = ciphertext
|
||||
second.row.version = 3
|
||||
await refused(f.run())
|
||||
assert.deepEqual(f.counts(), { factories: 3, charges: 3 })
|
||||
})
|
||||
|
||||
test("empty/global0 and full128 ledgers use complete exact global/owner counters without archive crypto", async () => {
|
||||
const empty = fixture([]); await empty.run(); assert.deepEqual(empty.counts(), { factories: 0, charges: 0 })
|
||||
const f = fixture(Array.from({ length: 128 }, (_, n) => full(n + 1, "owner-" + Math.floor(n / 8))))
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
||||
import { SERVER_POLL_STORAGE_POLICY as policy, PollStorageRefused } from "./server-poll-storage-model.js"
|
||||
import { createServerPollArchiveChargeVerifier, type verifyServerPollArchiveCharge } from "./server-poll-terminal-archive.js"
|
||||
import type { verifyServerPollRetiredArchiveCharge } from "./server-poll-retired-archive.js"
|
||||
|
||||
const BATCH = 128
|
||||
const MAX_ENTRIES = Math.floor(policy.globalBytes / policy.admissionBytes)
|
||||
@@ -15,6 +16,8 @@ function integer(value: unknown): number {
|
||||
}
|
||||
interface Total { heavy: number; compact: number; archiveBytes: number; rows: number; bytes: number }
|
||||
const zero = (): Total => ({ heavy: 0, compact: 0, archiveBytes: 0, rows: 0, bytes: 0 })
|
||||
const verifyRetiredCharge: typeof verifyServerPollRetiredArchiveCharge = async row =>
|
||||
(await import("./server-poll-retired-archive.js")).verifyServerPollRetiredArchiveCharge(row)
|
||||
const QUERY = `SELECT a.receipt_id AS admission_receipt_id,a.identity_kind,a.execution_id AS admission_execution_id,
|
||||
a.connection_id AS admission_connection_id,a.owner_id AS admission_owner_id,a.binding_proof AS admission_binding_proof,
|
||||
a.context_proof AS admission_context_proof,a.storage_state,a.compact_at,a.archive_bytes AS admission_archive_bytes,
|
||||
@@ -34,7 +37,8 @@ const QUERY = `SELECT a.receipt_id AS admission_receipt_id,a.identity_kind,a.exe
|
||||
FROM server_poll_storage_admissions a LEFT JOIN server_poll_archives r ON r.receipt_id=a.receipt_id
|
||||
WHERE ($1::uuid IS NULL OR a.receipt_id>$1::uuid) ORDER BY a.receipt_id LIMIT 128`
|
||||
|
||||
async function verify(sql: ProbeStorageSql, factory: () => Promise<typeof verifyServerPollArchiveCharge>) {
|
||||
async function verify(sql: ProbeStorageSql, factory: () => Promise<typeof verifyServerPollArchiveCharge>,
|
||||
retiredCharge: typeof verifyServerPollRetiredArchiveCharge) {
|
||||
const global = zero(), owners = new Map<string, Total>()
|
||||
const seenReceipts = new Set<string>()
|
||||
let cursor: string | null = null, count = 0, charge: typeof verifyServerPollArchiveCharge | undefined
|
||||
@@ -61,8 +65,13 @@ async function verify(sql: ProbeStorageSql, factory: () => Promise<typeof verify
|
||||
row.binding_proof !== row.admission_binding_proof || row.context_proof !== row.admission_context_proof ||
|
||||
row.original_activity_at !== row.admission_activity_at || row.original_expires_at !== row.admission_expires_at ||
|
||||
integer(row.ciphertext_bytes) !== integer(row.asserted_ciphertext_bytes) || integer(row.ciphertext_bytes) !== integer(row.admission_archive_bytes)) fail()
|
||||
charge ??= await factory()
|
||||
const verified = await charge(row)
|
||||
let verified
|
||||
if (row.version === 1) {
|
||||
charge ??= await factory()
|
||||
verified = await charge(row)
|
||||
} else if (row.version === 2) {
|
||||
verified = await retiredCharge(row)
|
||||
} else return fail()
|
||||
addition = { heavy: 0, compact: 1, archiveBytes: verified.metadata.ciphertext_bytes, rows: verified.rows, bytes: verified.bytes }
|
||||
} else return fail()
|
||||
const owned = owners.get(row.admission_owner_id) ?? zero()
|
||||
@@ -102,8 +111,9 @@ async function verify(sql: ProbeStorageSql, factory: () => Promise<typeof verify
|
||||
/** Caller must already own server-poll-storage-v1. Any malformed/missing MAC or
|
||||
* counter refuses admission before an INSERT, credential read or native I/O. */
|
||||
export async function assertServerPollArchiveAccounting(sql: ProbeStorageSql): Promise<void> {
|
||||
try { await verify(sql, createServerPollArchiveChargeVerifier) } catch { return fail() }
|
||||
try { await verify(sql, createServerPollArchiveChargeVerifier, verifyRetiredCharge) } catch { return fail() }
|
||||
}
|
||||
export async function assertServerPollArchiveAccountingForTests(sql: ProbeStorageSql, factory: () => Promise<typeof verifyServerPollArchiveCharge>): Promise<void> {
|
||||
try { await verify(sql, factory) } catch { return fail() }
|
||||
export async function assertServerPollArchiveAccountingForTests(sql: ProbeStorageSql, factory: () => Promise<typeof verifyServerPollArchiveCharge>,
|
||||
retiredCharge = verifyRetiredCharge): Promise<void> {
|
||||
try { await verify(sql, factory, retiredCharge) } catch { return fail() }
|
||||
}
|
||||
@@ -5,7 +5,8 @@ import assert from "node:assert/strict"
|
||||
import { createCipheriv, createDecipheriv, createHmac, randomBytes, randomUUID } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import { fingerprint } from "../modules/mcp/security.js"
|
||||
import { readServerPollArchivedHistoryForTests } from "./server-poll-history-reader.js"
|
||||
import { readRetiredServerPollHistoryForTests, readServerPollArchivedHistoryForTests } from "./server-poll-history-reader.js"
|
||||
import type { openServerPollRetiredArchive } from "./server-poll-retired-archive.js"
|
||||
import type { openServerPollTerminalArchive, ServerPollArchiveSource } from "./server-poll-terminal-archive.js"
|
||||
|
||||
function fixture(){
|
||||
@@ -110,3 +111,61 @@ test("caller mutation while decrypt awaits cannot replace authenticated lookup i
|
||||
}}),true)
|
||||
assert.equal(f.calls[0].values[0],f.raw().receiptId)
|
||||
})
|
||||
|
||||
function retiredFixture() {
|
||||
const now = Date.now(), operationId = randomUUID(), connectionId = randomUUID(), planId = randomUUID(), ownerId = "retired-owner"
|
||||
const identity = { operationId, connectionId, ownerId }
|
||||
const row = { plan_id: planId, target_server_ids: ["2", "1"], gc_not_before: "2026-10-31T01:00:00.123456Z" }
|
||||
const failure = Error("authority revoked")
|
||||
let queries = 0, opens = 0, checks = 0
|
||||
let mode = ""
|
||||
const run = () => readRetiredServerPollHistoryForTests(identity, async () => {
|
||||
checks++
|
||||
if (mode === "revoked-after-open" && opens) throw failure
|
||||
}, { deadlineAt: now + 1000 }, {
|
||||
now: () => now,
|
||||
query: async (sql, parameters) => {
|
||||
queries++
|
||||
assert.deepEqual(parameters, [operationId, connectionId, ownerId])
|
||||
assert.match(sql, /mcp_operation_replay_tombstones/)
|
||||
assert.match(sql, /r\.version=2/)
|
||||
assert.match(sql, /NOT EXISTS\(SELECT 1 FROM public\.mcp_operations/)
|
||||
if (mode === "gone" && queries === 2) return []
|
||||
if (mode === "changed" && queries === 2) return [{ ...row, target_server_ids: ["1"] }]
|
||||
return [row]
|
||||
},
|
||||
open: async () => {
|
||||
opens++
|
||||
if (mode === "bad-proof") throw Error("private proof failure")
|
||||
return { metadata: { execution_id: operationId, connection_id: connectionId, owner_id: ownerId },
|
||||
history: { planId, affectedServerIds: [1, 2], gcNotBefore: "2026-10-31T01:00:00.123456Z" } } as Awaited<ReturnType<typeof openServerPollRetiredArchive>>
|
||||
},
|
||||
})
|
||||
return { row, failure, run, mode: (value: string) => { mode = value }, counts: () => ({ queries, opens, checks }) }
|
||||
}
|
||||
|
||||
test("retired lookup verifies a current tombstone, archive proof and a second SQL snapshot before returning only scope", async () => {
|
||||
const f = retiredFixture()
|
||||
assert.deepEqual(await f.run(), { affectedServerIds: [1, 2] })
|
||||
assert.deepEqual(f.counts(), { queries: 2, opens: 1, checks: 4 })
|
||||
})
|
||||
|
||||
test("retired lookup refuses proof, plan, scope or retention drift and GC between reads", async () => {
|
||||
for (const mode of ["bad-proof", "changed", "gone"]) {
|
||||
const f = retiredFixture()
|
||||
f.mode(mode)
|
||||
assert.equal(await f.run(), undefined)
|
||||
}
|
||||
for (const row of [{ plan_id: randomUUID() }, { target_server_ids: ["1", "1"] }, { gc_not_before: "2026-10-31T01:00:00.123457Z" }]) {
|
||||
const f = retiredFixture()
|
||||
Object.assign(f.row, row)
|
||||
assert.equal(await f.run(), undefined)
|
||||
}
|
||||
})
|
||||
|
||||
test("retired lookup propagates exact current authorization failure after archive open", async () => {
|
||||
const f = retiredFixture()
|
||||
f.mode("revoked-after-open")
|
||||
await assert.rejects(f.run(), error => error === f.failure)
|
||||
assert.equal(f.counts().queries, 1)
|
||||
})
|
||||
@@ -3,7 +3,8 @@
|
||||
* Original credentials authenticate history only; they are never renewed. */
|
||||
import { z } from "zod"
|
||||
import { decryptSecret, fingerprint, privateSecretFingerprint } from "../modules/mcp/security.js"
|
||||
import { openServerPollTerminalArchive } from "./server-poll-terminal-archive.js"
|
||||
import { canonicalPollArchiveTimestamp, openServerPollTerminalArchive } from "./server-poll-terminal-archive.js"
|
||||
import { openServerPollRetiredArchive } from "./server-poll-retired-archive.js"
|
||||
|
||||
export interface ServerPollHistoryReadWindow { deadlineAt: number; signal?: AbortSignal }
|
||||
type Query = (text: string, values: unknown[], options: ServerPollHistoryReadWindow) => Promise<Record<string, unknown>[]>
|
||||
@@ -99,3 +100,72 @@ const production:Dependencies={
|
||||
export const readServerPollArchivedHistory=(raw:Record<string,unknown>,authorize:()=>Promise<unknown>,options:ServerPollHistoryReadWindow)=>read(raw,authorize,options,production)
|
||||
/** Explicit component-only composition. No public input selects these hooks. */
|
||||
export const readServerPollArchivedHistoryForTests=(raw:Record<string,unknown>,authorize:()=>Promise<unknown>,options:ServerPollHistoryReadWindow,deps:Dependencies)=>read(raw,authorize,options,deps)
|
||||
|
||||
const RETIRED_QUERY = `SELECT r.version,r.receipt_id,r.execution_id,r.connection_id,r.owner_id,r.binding_proof,r.context_proof,
|
||||
r.source_digest,r.source_sha256,r.terminal_digest,r.decoded_bytes,r.compressed_bytes,r.archive_proof,
|
||||
to_char(r.original_activity_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_activity_at,
|
||||
to_char(r.original_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_expires_at,
|
||||
to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS gc_not_before,
|
||||
a.retention_days,a.safety_ms,octet_length(r.ciphertext) AS ciphertext_bytes,
|
||||
pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(r.ciphertext,'UTF8')),'hex') AS ciphertext_sha256,
|
||||
CASE WHEN octet_length(r.ciphertext)<=33000 THEN r.ciphertext END AS ciphertext,
|
||||
t.plan_id,t.target_server_ids
|
||||
FROM public.server_poll_archives r JOIN public.server_poll_storage_admissions a USING(receipt_id)
|
||||
JOIN public.mcp_connections c ON c.id=a.connection_id AND c.owner_id=a.owner_id
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=a.execution_id AND t.connection_id=a.connection_id
|
||||
WHERE r.execution_id=$1 AND a.execution_id=$1 AND a.connection_id=$2 AND a.owner_id=$3
|
||||
AND a.identity_kind='mcp' AND a.storage_state='compacted' AND r.version=2
|
||||
AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id AND r.binding_proof=a.binding_proof AND r.context_proof=a.context_proof
|
||||
AND r.ciphertext_bytes=a.archive_bytes AND r.ciphertext_bytes=octet_length(r.ciphertext)
|
||||
AND r.original_activity_at=a.original_activity_at AND r.original_expires_at=a.authority_expires_at
|
||||
AND t.action_id='servers.poll' AND t.target_global IS TRUE AND cardinality(t.target_server_ids) BETWEEN 1 AND 1000
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operations o WHERE o.id=t.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_plans p WHERE p.id=t.plan_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps s WHERE s.operation_id=t.operation_id)
|
||||
LIMIT 2`
|
||||
|
||||
interface RetiredDependencies {
|
||||
query: Query; now(): number; open: typeof openServerPollRetiredArchive
|
||||
}
|
||||
export interface RetiredSourceIdentity { operationId: string; connectionId: string; ownerId: string }
|
||||
|
||||
async function readRetired(identity: RetiredSourceIdentity, currentAuthorize: () => Promise<unknown>,
|
||||
rawWindow: ServerPollHistoryReadWindow, deps: RetiredDependencies): Promise<{ affectedServerIds: number[] } | undefined> {
|
||||
let authFailure: { error: unknown } | undefined
|
||||
try {
|
||||
const { operationId, connectionId, ownerId } = identity
|
||||
if (!uuid.safeParse(operationId).success || !uuid.safeParse(connectionId).success || !ownerId || ownerId.length > 200) fail()
|
||||
const options = Object.freeze({ ...rawWindow, deadlineAt: Math.min(rawWindow.deadlineAt, deps.now() + 180000) })
|
||||
const live = () => { if (!Number.isSafeInteger(options.deadlineAt) || deps.now() >= options.deadlineAt || options.signal?.aborted) fail() }
|
||||
const authorize = async () => {
|
||||
live()
|
||||
try { await currentAuthorize() } catch (error) { authFailure = { error }; throw error }
|
||||
live()
|
||||
}
|
||||
const values = [operationId, connectionId, ownerId]
|
||||
await authorize()
|
||||
const rows = await deps.query(RETIRED_QUERY, values, options)
|
||||
await authorize()
|
||||
if (rows.length !== 1) return undefined
|
||||
const row = structuredClone(rows[0])
|
||||
const { metadata, history } = await deps.open(row)
|
||||
await authorize()
|
||||
if (metadata.execution_id !== operationId || metadata.connection_id !== connectionId || metadata.owner_id !== ownerId
|
||||
|| history.planId !== row.plan_id || history.gcNotBefore !== canonicalPollArchiveTimestamp(row.gc_not_before)) fail()
|
||||
const scope = z.array(z.union([sid, z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(sid)])).min(1).max(1000).parse(row.target_server_ids)
|
||||
if (scope.length !== history.affectedServerIds.length || new Set(scope).size !== scope.length
|
||||
|| scope.some(id => !history.affectedServerIds.includes(id))) fail()
|
||||
const fresh = await deps.query(RETIRED_QUERY, values, options)
|
||||
await authorize()
|
||||
if (fresh.length !== 1 || fingerprint(fresh[0]) !== fingerprint(row)) fail()
|
||||
return { affectedServerIds: [...history.affectedServerIds] }
|
||||
} catch {
|
||||
if (authFailure) throw authFailure.error
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/** The caller owns current authority and independently checks returned scope. */
|
||||
export const readRetiredServerPollHistory = (identity: RetiredSourceIdentity, authorize: () => Promise<unknown>, options: ServerPollHistoryReadWindow) =>
|
||||
readRetired(identity, authorize, options, { query: production.query, now: Date.now, open: openServerPollRetiredArchive })
|
||||
export const readRetiredServerPollHistoryForTests = readRetired
|
||||
@@ -0,0 +1,183 @@
|
||||
/** Owned-only historical fixture. Synthetically seeds a historical timeline
|
||||
* before signing. The real V1 compactor, V2 retirement, read and GC run with
|
||||
* production crypto and all database guards enabled. */
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true")
|
||||
const { pool,closePool } = await import("../db/index.js")
|
||||
const { applySqlMigrations } = await import("../db/migrate.js")
|
||||
const { createMcpConnection,authenticateMcpToken } = await import("../modules/mcp/store.js")
|
||||
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
||||
const { invokeMcpTool } = await import("../modules/mcp/runtime.js")
|
||||
const { encryptSecret,privateSecretFingerprint,fingerprint,redact } = await import("../modules/mcp/security.js")
|
||||
const { encodeServerPollJournalAnchor } = await import("./server-poll-journal-anchor.js")
|
||||
const { compactCompletedServerPollStorage,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js")
|
||||
const { openServerPollTerminalArchive } = await import("./server-poll-terminal-archive.js")
|
||||
const { retireCompletedServerPollOperation } = await import("./server-poll-operation-retirement.js")
|
||||
const { collectExpiredServerPollStorage } = await import("./server-poll-storage-retention.js")
|
||||
const { PollStorageUnknown } = await import("./server-poll-storage.js")
|
||||
const { tickMcpWorkers } = await import("../modules/mcp/workers.js")
|
||||
const { Client } = await import("pg")
|
||||
const proof=(domain:string,value:unknown)=>privateSecretFingerprint(fingerprint(value),domain)
|
||||
const encrypt=(value:unknown)=>encryptSecret(JSON.stringify(value))
|
||||
|
||||
async function historical(days:number,serverId:number) {
|
||||
const base=Date.now()-days*86400000, stamp=(offset:number)=>new Date(base+offset).toISOString()
|
||||
const ownerId=randomUUID(),executionId=randomUUID(),receiptId=randomUUID(),planId=randomUUID()
|
||||
const actor={id:ownerId,email:"[email protected]",name:"Owned historical poll",apps:["mm"],
|
||||
permissions:["mm:servers:write","mm:servers:read"],issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
|
||||
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
|
||||
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
|
||||
scope:{serverIds:[serverId],operationIds:["servers.poll","operations.global.reconcile"],allowGlobal:true},
|
||||
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
|
||||
const current=await authenticateMcpToken(issued.token),connectionId=current.connectionId,credentialId=randomUUID()
|
||||
const core={version:1 as const,requiresHuman:true as const,scope:"global" as const,request:{serverId,receiptId},around:stamp(0),
|
||||
beforeProof:"a".repeat(64),authorityProof:"b".repeat(64),maintenance:[],affectedServerIds:[serverId],mutationCost:8}
|
||||
const plan={...core,bindingProof:await proof("server-poll-production-v1",core)}
|
||||
const context={identity:{kind:"mcp" as const,ownerId,executionId,connectionId,credentialId,operationName:"servers.poll",
|
||||
issuedAt:base,expiresAt:base+180000},plan}
|
||||
const contextProof=await proof("server-poll-journal-v1:context",context)
|
||||
const anchor=await encodeServerPollJournalAnchor(context,undefined,{proof:(domain,value)=>proof("server-poll-journal-v1:"+domain,value)})
|
||||
const binding={version:1,operation:"servers.poll",principal:{connectionId,credentialId,credentialExpiresAt:stamp(240000)},
|
||||
ownerId,issuedAt:base,expiresAt:base+180000,plan}
|
||||
const input={serverId,receiptId,inputDigest:await proof("server-poll-mcp-binding-v1",binding),bindingCiphertext:await encrypt(binding)}
|
||||
const step={kind:"service",service:"servers.poll",input}
|
||||
const evidence={target:{scope:"global"},objects:[{kind:"server",id:String(serverId),serverId}],snapshots:[],steps:[step],
|
||||
before:{proof:plan.beforeProof},mutationCost:8,requiresHuman:true,verificationScope:"configuration",validUntil:stamp(180000),
|
||||
warnings:["Полный опрос читает устройство и изменяет общие снимки и разделы истории; требуется подтверждение человеком. Автоматический повтор запрещён."]}
|
||||
const snapshot={id:42,serverId,polledAt:stamp(1000),status:"online" as const,latencyMs:0,rosVersion:null,boardName:null,
|
||||
uptime:null,cpuLoad:null,freeMemory:null,totalMemory:null,identityName:null}
|
||||
const result={operationId:executionId,receiptId,nativeReads:4,confirmedPublications:2,completedPublications:2,complete:true,
|
||||
maintenanceComplete:true,configurationVerified:true,automaticReplay:false,snapshot}
|
||||
const receipt={receiptId,nativeReads:4,confirmedPublications:2,completedPublications:2,complete:true,
|
||||
maintenanceComplete:true,configurationVerified:true,afterProof:"f".repeat(64),observation:"online",snapshot}
|
||||
const db=await pool.connect()
|
||||
try{
|
||||
await db.query("BEGIN")
|
||||
await db.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at)
|
||||
VALUES($1,$2,'servers.poll',1,1,'owned-historical',$3,$4,'completed',$5,$6,$7,$8)`,
|
||||
[planId,connectionId,{sealed:await encrypt({serverId})},{sealed:await encrypt(evidence)},stamp(190000),ownerId,stamp(50),stamp(25)])
|
||||
await db.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,result,created_at,finished_at)
|
||||
VALUES($1,$2,$3,$4,'applied','passed',$5,$6,$7)`,
|
||||
[executionId,planId,connectionId,"owned-historical-"+executionId,{confirmedWrites:1,verificationScope:"configuration"},stamp(100),stamp(1600)])
|
||||
await db.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at)
|
||||
VALUES($1,0,'verified',$2,$3,$4,$5)`,[executionId,redact(step),result,stamp(110),stamp(1500)])
|
||||
// Historical journal admission is impossible via live writes by design.
|
||||
// Only this private fixture transaction seeds the signed, pre-existing rows.
|
||||
await db.query("SET LOCAL session_replication_role='replica'")
|
||||
await db.query(`INSERT INTO server_poll_storage_admissions(receipt_id,identity_kind,execution_id,connection_id,owner_id,binding_proof,
|
||||
context_proof,authority_issued_at,authority_expires_at,max_stages,claim_attempted_at,created_at,retention_days,gc_not_before,used_rows,used_bytes)
|
||||
VALUES($1,'mcp',$2,$3,$4,$5,$6,$7,$8,8,$9,$10,365,$11,14,4000)`,
|
||||
[receiptId,executionId,connectionId,ownerId,plan.bindingProof,contextProof,stamp(0),stamp(180000),stamp(300),stamp(200),stamp(180000+365*86400000+60000)])
|
||||
await db.query(`INSERT INTO server_poll_executions(receipt_id,identity_kind,execution_id,binding_proof,context_ciphertext,context_proof,created_at)
|
||||
VALUES($1,'mcp',$2,$3,$4,$5,$6)`,[receiptId,executionId,plan.bindingProof,await encryptSecret(anchor),contextProof,stamp(400)])
|
||||
const stages=["native.identity","native.resource","native.interfaces","native.addresses","publish.0","publish.1"]
|
||||
for(const [index,stage] of stages.entries()){
|
||||
const beforeProof=index>4?"e".repeat(64):plan.beforeProof,effectProof="c".repeat(64)
|
||||
const resultProof=index===stages.length-1?receipt.afterProof:"e".repeat(64),outcome="confirmed"
|
||||
await db.query("INSERT INTO server_poll_intents VALUES($1,$2,$3,$4,$5,$6)",
|
||||
[receiptId,stage,beforeProof,effectProof,await proof("server-poll-journal-v1:intent",{receiptId,stage,beforeProof,effectProof}),stamp(500+index*100)])
|
||||
await db.query("INSERT INTO server_poll_acks VALUES($1,$2,$3,$4,$5,$6)",
|
||||
[receiptId,stage,resultProof,outcome,await proof("server-poll-journal-v1:ack",{receiptId,stage,resultProof,outcome}),stamp(550+index*100)])
|
||||
}
|
||||
const receiptText=JSON.stringify(receipt)
|
||||
await db.query("INSERT INTO server_poll_receipts VALUES($1,$2,$3,$4)",
|
||||
[receiptId,await encryptSecret(receiptText),await proof("server-poll-journal-v1:receipt",receiptText),stamp(1400)])
|
||||
await db.query("INSERT INTO server_poll_storage_usage(scope,admissions,reserved_rows,reserved_bytes) VALUES($1,1,4019,8421376)",["owner:"+ownerId])
|
||||
await db.query("UPDATE server_poll_storage_usage SET admissions=admissions+1,reserved_rows=reserved_rows+4019,reserved_bytes=reserved_bytes+8421376 WHERE scope='global'")
|
||||
await db.query("SET LOCAL session_replication_role='origin'")
|
||||
assert.equal((await db.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true)
|
||||
await db.query("COMMIT")
|
||||
}catch(error){await db.query("ROLLBACK");throw error}finally{db.release()}
|
||||
return {receiptId,executionId,planId,ownerId,connectionId,current}
|
||||
}
|
||||
|
||||
try{
|
||||
await applySqlMigrations(pool)
|
||||
await pool.query("UPDATE mcp_settings SET enabled=true,requests_retention_days=365 WHERE id=1")
|
||||
const {rows:[server]}=await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned historical poll','127.0.0.1',false) RETURNING id")
|
||||
const retained=await historical(181,Number(server.id)), expired=await historical(366,Number(server.id))
|
||||
for(const fixture of [expired,retained]){
|
||||
assert.equal((await compactCompletedServerPollStorage()).compacted,1)
|
||||
const {rows:[v1]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId])
|
||||
await openServerPollTerminalArchive(v1,Date.now())
|
||||
const before=(await pool.query("SELECT gc_not_before,authority_expires_at,retention_days,original_activity_at FROM server_poll_storage_admissions WHERE receipt_id=$1",[fixture.receiptId])).rows[0]
|
||||
await assert.rejects(retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:"0".repeat(64)}))
|
||||
const hold=randomUUID()
|
||||
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[hold,fixture.executionId])
|
||||
await assert.rejects(retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:v1.archive_proof}))
|
||||
assert.equal((await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId])).rows[0].archive_proof,v1.archive_proof)
|
||||
assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[fixture.receiptId])).rows.length,0)
|
||||
await pool.query("DELETE FROM mcp_drift WHERE id=$1",[hold])
|
||||
assert.equal((await retireCompletedServerPollOperation({receiptId:fixture.receiptId,oldArchiveProof:v1.archive_proof})).retired,true)
|
||||
const {rows:[v2]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[fixture.receiptId])
|
||||
assert.equal(v2.version,2);assert(v2.ciphertext_bytes<v1.ciphertext_bytes)
|
||||
assert.deepEqual((await pool.query("SELECT gc_not_before,authority_expires_at,retention_days,original_activity_at FROM server_poll_storage_admissions WHERE receipt_id=$1",[fixture.receiptId])).rows[0],before)
|
||||
const {rows:[removed]}=await pool.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS operation,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan,
|
||||
EXISTS(SELECT 1 FROM mcp_operation_replay_tombstones WHERE operation_id=$1 AND plan_id=$2) AS tombstone,
|
||||
server_poll_storage_accounting_valid() AS accounting`,[fixture.executionId,fixture.planId])
|
||||
assert.deepEqual(removed,{operation:true,step:true,plan:true,tombstone:true,accounting:true})
|
||||
const read=await invokeMcpTool(fixture.current,"mm_global_operation_reconcile",{operationId:fixture.executionId}) as {results:{matchesDesiredState:boolean}[];automaticRetry:boolean}
|
||||
assert.equal(read.results[0].matchesDesiredState,true);assert.equal(read.automaticRetry,false)
|
||||
const foreign=fixture===retained?expired.current:retained.current
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_global_operation_reconcile",{operationId:fixture.executionId}),
|
||||
(error:unknown)=>(error as {code?:string}).code==="NOT_FOUND")
|
||||
}
|
||||
assert.equal((await collectExpiredServerPollStorage()).collected,1)
|
||||
assert.equal((await collectExpiredServerPollStorage()).collected,0)
|
||||
const {rows}=await pool.query("SELECT r.receipt_id,r.ciphertext_bytes,u.reserved_bytes,u.reserved_rows FROM server_poll_archives r CROSS JOIN server_poll_storage_usage u WHERE u.scope='global'")
|
||||
assert.equal(rows.length,1);assert.equal(rows[0].receipt_id,retained.receiptId)
|
||||
assert.equal(Number(rows[0].reserved_bytes),32768+rows[0].ciphertext_bytes);assert.equal(Number(rows[0].reserved_rows),2)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[expired.receiptId])).rows.length,0)
|
||||
|
||||
const held=await historical(190,Number(server.id)), spent=await historical(189,Number(server.id))
|
||||
const ready=await historical(188,Number(server.id))
|
||||
// The first two are already compacted. The actual worker has to compact the
|
||||
// next eligible archive, skip older held/spent V1 rows, then retire it.
|
||||
assert.equal((await compactCompletedServerPollStorage()).compacted,1)
|
||||
assert.equal((await compactCompletedServerPollStorage()).compacted,1)
|
||||
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[randomUUID(),held.executionId])
|
||||
const {rows:[spentV1]}=await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[spent.receiptId])
|
||||
const originalQuery=Client.prototype.query
|
||||
let commits=0
|
||||
try{
|
||||
Client.prototype.query=function(this:InstanceType<typeof Client>,...args:unknown[]){
|
||||
if((args[0] as {text?:string})?.text==="COMMIT" && ++commits===2) {
|
||||
// No COMMIT ACK can reach the owner. Closing the real checked-out
|
||||
// backend rolls back uncommitted work, leaving the earlier claim spent.
|
||||
return Promise.reject(Error("owned poll retirement COMMIT transport loss"))
|
||||
}
|
||||
return Reflect.apply(originalQuery,this,args)
|
||||
} as typeof Client.prototype.query
|
||||
await assert.rejects(retireCompletedServerPollOperation({receiptId:spent.receiptId,oldArchiveProof:spentV1.archive_proof}),
|
||||
(error:unknown)=>error instanceof PollStorageUnknown && error.automaticReplay===false)
|
||||
}finally{Client.prototype.query=originalQuery}
|
||||
assert.equal(commits,2)
|
||||
assert.equal((await pool.query(SERVER_POLL_ARCHIVE_READ_SQL,[spent.receiptId])).rows[0].version,1)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_poll_retirement_claims WHERE receipt_id=$1",[spent.receiptId])).rows.length,1)
|
||||
const workerProjection=async()=>(await pool.query(`SELECT a.receipt_id,r.version,r.archive_proof,a.archive_bytes,
|
||||
o.id AS live_operation,t.operation_id AS tombstone,c.claim_token
|
||||
FROM server_poll_storage_admissions a LEFT JOIN server_poll_archives r USING(receipt_id)
|
||||
LEFT JOIN mcp_operations o ON o.id=a.execution_id
|
||||
LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=a.execution_id
|
||||
LEFT JOIN server_poll_retirement_claims c USING(receipt_id)
|
||||
WHERE a.receipt_id=ANY($1::uuid[]) ORDER BY a.receipt_id`,[[held.receiptId,spent.receiptId,ready.receiptId]])).rows
|
||||
const beforeWorker=await workerProjection()
|
||||
await tickMcpWorkers()
|
||||
const afterWorker=await workerProjection()
|
||||
assert.deepEqual(afterWorker.filter(row=>row.receipt_id!==ready.receiptId),beforeWorker.filter(row=>row.receipt_id!==ready.receiptId))
|
||||
const retiredByWorker=afterWorker.find(row=>row.receipt_id===ready.receiptId)
|
||||
assert.equal(retiredByWorker?.version,2);assert.equal(retiredByWorker?.live_operation,null)
|
||||
assert.equal(retiredByWorker?.tombstone,ready.executionId);assert(retiredByWorker?.claim_token)
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await workerProjection(),afterWorker)
|
||||
assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true)
|
||||
console.log("Owned PostgreSQL: authenticated V1/V2 transition, retained history, once-only refund, and actual worker skips held/spent rows")
|
||||
}finally{await closePool()}
|
||||
@@ -0,0 +1,162 @@
|
||||
/** Scheduled/internal maintenance only. No public HTTP or MCP registration. */
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { z } from "zod"
|
||||
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
|
||||
import { fingerprint } from "../modules/mcp/security.js"
|
||||
import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js"
|
||||
import { SERVER_POLL_ARCHIVE_READ_SQL } from "./server-poll-terminal-compaction.js"
|
||||
import { openServerPollRetiredArchive, projectServerPollRetiredArchive, type LockedServerPollRetirementSource } from "./server-poll-retired-archive.js"
|
||||
import { PollStorageRefused, runPollStorageTransaction } from "./server-poll-storage.js"
|
||||
import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
||||
import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
||||
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
||||
|
||||
const request = z.object({ receiptId: z.string().uuid(), oldArchiveProof: z.string().regex(/^[a-f0-9]{64}$/) }).strict()
|
||||
const requireThat: (value: unknown) => asserts value = value => {
|
||||
if (!value) throw new PollStorageRefused()
|
||||
}
|
||||
|
||||
/** The storage owner already holds its global xact advisory lock. Do not wait
|
||||
* for a connection writer or runner that might need the storage lock. */
|
||||
const ELIGIBLE = `FROM public.server_poll_storage_admissions a JOIN public.server_poll_archives r USING(receipt_id)
|
||||
JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
||||
WHERE a.identity_kind='mcp' AND a.storage_state='compacted'
|
||||
AND r.version=1 AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at)
|
||||
AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.operation_id='servers.poll' AND p.operation_version=1 AND p.state='completed' AND p.approved_by IS NOT NULL
|
||||
AND s.state='verified' AND s.server_id IS NULL AND s.finished_at IS NOT NULL
|
||||
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
|
||||
const SOURCE = `SELECT CASE WHEN octet_length(to_jsonb(a)::text)<=1048576 THEN to_jsonb(a) END AS admission,
|
||||
CASE WHEN octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)<=16777216
|
||||
THEN jsonb_build_object('operation',to_jsonb(o),'plan',to_jsonb(p),'step',to_jsonb(s)) END AS terminal,
|
||||
o.id AS operation_id,p.id AS plan_id,o.idempotency_key,clock_timestamp() AS observed_at
|
||||
${ELIGIBLE} AND a.receipt_id=$1 AND r.archive_proof=$2 FOR UPDATE OF a,r,o,p,s NOWAIT`
|
||||
const CANDIDATES = `SELECT a.receipt_id,r.archive_proof,o.finished_at::text AS finished_at ${ELIGIBLE}
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c WHERE c.receipt_id=a.receipt_id)
|
||||
AND ($1::timestamptz IS NULL OR (o.finished_at,a.receipt_id)>($1::timestamptz,$2::uuid))
|
||||
ORDER BY o.finished_at,a.receipt_id LIMIT 8`
|
||||
|
||||
const deps = { connect:connectProbeStorage,now:Date.now }
|
||||
const transaction = <T>(deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise<T>) =>
|
||||
runPollStorageTransaction(deps,{deadlineAt},async()=>{},work)
|
||||
type Target = z.infer<typeof request>
|
||||
|
||||
async function prepare(sql: ProbeStorageSql,{receiptId,oldArchiveProof}:Target) {
|
||||
const identity = (await sql.query(`SELECT a.connection_id,a.execution_id,a.owner_id FROM public.server_poll_storage_admissions a
|
||||
JOIN public.server_poll_archives r USING(receipt_id) WHERE a.receipt_id=$1 AND a.storage_state='compacted'
|
||||
AND a.identity_kind='mcp' AND r.version=1 AND r.archive_proof=$2`, [receiptId,oldArchiveProof])).rows
|
||||
requireThat(identity.length===1)
|
||||
const { connection_id: connectionId,execution_id: operationId,owner_id: ownerId } = identity[0]
|
||||
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 AND owner_id=$2 FOR UPDATE NOWAIT",
|
||||
[connectionId,ownerId])).rows.length===1)
|
||||
for (const key of ["mcp-operation:"+operationId,"server-poll-execution-v1:mcp:"+operationId,"server-poll-receipt-v1:"+receiptId]) {
|
||||
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",[key])).rows[0]?.locked===true)
|
||||
}
|
||||
const rows = (await sql.query(SOURCE,[receiptId,oldArchiveProof])).rows
|
||||
requireThat(rows.length===1 && rows[0].operation_id===operationId && rows[0].admission && rows[0].terminal)
|
||||
const { admission,terminal,plan_id: planId,idempotency_key: key,observed_at: observedAt } = rows[0]
|
||||
requireThat(typeof key==="string" && typeof connectionId==="string" && typeof planId==="string")
|
||||
await assertServerPollArchiveAccounting(sql)
|
||||
const oldArchive = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
|
||||
requireThat(oldArchive.length===1 && oldArchive[0].archive_proof===oldArchiveProof)
|
||||
const archived = await projectServerPollRetiredArchive({ archive:oldArchive[0],admission,terminal } as LockedServerPollRetirementSource,
|
||||
new Date(observedAt as string | Date).getTime())
|
||||
requireThat(archived)
|
||||
const projection = await openServerPollRetiredArchive(archived)
|
||||
requireThat(projection.metadata.receipt_id===receiptId && projection.metadata.execution_id===operationId
|
||||
&& projection.metadata.connection_id===connectionId && projection.metadata.owner_id===ownerId && projection.history.planId===planId)
|
||||
return {connectionId,operationId,planId,key,archived,projection,previousBytes:oldArchive[0].ciphertext_bytes}
|
||||
}
|
||||
|
||||
async function retire(target:Target,deadlineAt:number) {
|
||||
const {receiptId,oldArchiveProof}=target
|
||||
const token=randomUUID()
|
||||
await transaction(deadlineAt,async sql=>{
|
||||
// Check the live hold predicates, all locks and both archive proofs before
|
||||
// spending the irreversible one-shot claim.
|
||||
await prepare(sql,target)
|
||||
const claim=await sql.query(`INSERT INTO public.server_poll_retirement_claims(receipt_id,old_archive_proof,claim_token)
|
||||
VALUES($1,$2,$3) ON CONFLICT DO NOTHING RETURNING receipt_id`,[receiptId,oldArchiveProof,token])
|
||||
requireThat(claim.rows.length===1&&claim.rows[0].receipt_id===receiptId)
|
||||
})
|
||||
// No destructive transaction after an unacknowledged claim. A claim is
|
||||
// never cleared, even when a later checkout/hold/COMMIT is inconclusive.
|
||||
return transaction(deadlineAt,async sql=>{
|
||||
const claimed=await sql.query(`SELECT receipt_id FROM public.server_poll_retirement_claims
|
||||
WHERE receipt_id=$1 AND old_archive_proof=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,
|
||||
[receiptId,oldArchiveProof,token])
|
||||
requireThat(claimed.rows.length===1&&claimed.rows[0].receipt_id===receiptId)
|
||||
const {connectionId,operationId,planId,key,archived,projection,previousBytes}=await prepare(sql,target)
|
||||
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,'servers.poll',$5::bigint[],true) RETURNING operation_id`,
|
||||
[operationId,planId,connectionId,operationIdempotencyDigest(connectionId,key),projection.history.affectedServerIds])
|
||||
requireThat(tombstone.rows.length===1 && tombstone.rows[0].operation_id===operationId)
|
||||
await sql.query("SELECT set_config('mm.poll_retire_receipt',$1,true),set_config('mm.poll_retire_claim',$2,true)",[receiptId,token])
|
||||
const updated = await sql.query(`UPDATE public.server_poll_archives SET version=2,source_digest=$3,source_sha256=$4,
|
||||
decoded_bytes=$5,compressed_bytes=$6,ciphertext=$7,ciphertext_bytes=$8,archive_proof=$9
|
||||
WHERE receipt_id=$1 AND version=1 AND archive_proof=$2 RETURNING receipt_id`,
|
||||
[receiptId,oldArchiveProof,archived.source_digest,archived.source_sha256,archived.decoded_bytes,archived.compressed_bytes,
|
||||
archived.ciphertext,archived.ciphertext_bytes,archived.archive_proof])
|
||||
requireThat(updated.rows.length===1 && updated.rows[0].receipt_id===receiptId)
|
||||
const charged = await sql.query(`UPDATE public.server_poll_storage_admissions SET archive_bytes=$2
|
||||
WHERE receipt_id=$1 AND storage_state='compacted' AND archive_bytes=$3 RETURNING receipt_id`,
|
||||
[receiptId,archived.ciphertext_bytes,previousBytes])
|
||||
requireThat(charged.rows.length===1 && charged.rows[0].receipt_id===receiptId)
|
||||
const readback = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
|
||||
requireThat(readback.length===1 && readback[0].archive_proof===archived.archive_proof
|
||||
&& fingerprint(await openServerPollRetiredArchive(readback[0]))===fingerprint(projection))
|
||||
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
||||
requireThat(step.rows.length===1 && step.rows[0].operation_id===operationId)
|
||||
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id",
|
||||
[operationId,connectionId,planId])
|
||||
requireThat(operation.rows.length===1 && operation.rows[0].id===operationId)
|
||||
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id",[planId,connectionId])
|
||||
requireThat(plan.rows.length===1 && plan.rows[0].id===planId)
|
||||
await assertServerPollArchiveAccounting(sql)
|
||||
return { retired:true as const, automaticReplay:false as const }
|
||||
})
|
||||
}
|
||||
|
||||
/** Exact internal entry used by owned fixtures. It cannot bypass the claim. */
|
||||
export function retireCompletedServerPollOperation(raw:Target) {
|
||||
const target=request.parse(raw)
|
||||
return sharedDatabaseGate.request(()=>retire(target,Date.now()+20000))
|
||||
}
|
||||
|
||||
let cursor:{finishedAt:string;receiptId:string}|undefined
|
||||
export function retireCompletedServerPollOperations() {
|
||||
return sharedDatabaseGate.request(async()=>{
|
||||
const deadlineAt=Date.now()+20000
|
||||
const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES,
|
||||
[cursor?.finishedAt??null,cursor?.receiptId??null])).rows)
|
||||
requireThat(rows.length<=8)
|
||||
let retired=0,scanned=0
|
||||
for(const row of rows){
|
||||
if(Date.now()>=deadlineAt-2000||retired>=2)break
|
||||
const target=request.parse({receiptId:row.receipt_id,oldArchiveProof:row.archive_proof})
|
||||
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
|
||||
cursor={finishedAt:row.finished_at,receiptId:target.receiptId}
|
||||
scanned++
|
||||
try{await retire(target,deadlineAt);retired++}
|
||||
catch(error){if(!(error instanceof PollStorageRefused))throw error}
|
||||
}
|
||||
// Keep the cursor for any unvisited rows. On exhaustion the next tick
|
||||
// starts at the beginning, revisiting formerly held or busy candidates.
|
||||
if(scanned===rows.length&&rows.length<8)cursor=undefined
|
||||
return {scanned,retired,automaticReplay:false as const}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { createCipheriv, createDecipheriv, createHmac, randomBytes } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import { fingerprint } from "../modules/mcp/security.js"
|
||||
import { createServerPollRetiredArchiveForTests } from "./server-poll-retired-archive.js"
|
||||
import { createServerPollTerminalArchiveForTests, ServerPollArchiveRefused } from "./server-poll-terminal-archive.js"
|
||||
import type { VerifiedServerPollHistory } from "./server-poll-history-verification.js"
|
||||
|
||||
const receiptId = "11111111-1111-4111-8111-111111111111"
|
||||
const executionId = "22222222-2222-4222-8222-222222222222"
|
||||
const connectionId = "33333333-3333-4333-8333-333333333333"
|
||||
const planId = "44444444-4444-4444-8444-444444444444"
|
||||
const ownerId = "owned-retired-archive"
|
||||
const secret = "ORIGINAL-CREDENTIAL-OR-PAYLOAD-MUST-DISAPPEAR"
|
||||
const expiredAt = "2026-10-29T03:20:00.123456Z"
|
||||
const terminalAt = "2026-10-29T03:21:00.123457Z"
|
||||
const completedAt = "2026-10-29T03:18:00.123458Z"
|
||||
const createdAt = "2026-10-29T03:17:00.123459Z"
|
||||
const gcNotBefore = "2026-11-29T03:20:00.123460Z"
|
||||
const now = Date.parse("2026-11-30T00:00:00Z")
|
||||
const key = randomBytes(32), mac = randomBytes(32)
|
||||
const proof = async (purpose: string, value: unknown) => createHmac("sha256", mac).update(purpose).update(fingerprint(value)).digest("hex")
|
||||
const encrypt = async (plain: Buffer) => {
|
||||
const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv)
|
||||
const ciphertext = Buffer.concat([cipher.update(plain), cipher.final()])
|
||||
return Buffer.concat([iv, cipher.getAuthTag(), ciphertext])
|
||||
}
|
||||
const decrypt = async (bytes: Buffer) => {
|
||||
const cipher = createDecipheriv("aes-256-gcm", key, bytes.subarray(0, 12))
|
||||
cipher.setAuthTag(bytes.subarray(12, 28))
|
||||
return Buffer.concat([cipher.update(bytes.subarray(28)), cipher.final()])
|
||||
}
|
||||
async function fixture(v2Encrypt = encrypt) {
|
||||
const admission = { storage_state: "full", gc_not_before: gcNotBefore, opaque: secret }
|
||||
const terminal = { operation: { finished_at: terminalAt, result: secret }, plan: { id: planId, expires_at: expiredAt, evidence: secret },
|
||||
step: { intended_state: secret, result: secret } }
|
||||
const journal = { execution: { created_at: createdAt, context_ciphertext: secret }, receipt: { created_at: completedAt, receipt_ciphertext: secret },
|
||||
intents: [{ opaque: randomBytes(2048).toString("hex") }], acks: [{ opaque: secret }], quarantines: [] }
|
||||
// Tests use genuine V1 codec authentication with a controlled source callback.
|
||||
// Full production V1 source parity is tested by its existing verifier suite.
|
||||
const v1 = createServerPollTerminalArchiveForTests({ encrypt, decrypt, proof, verifySource: async () => ({
|
||||
receiptId, executionId, connectionId, ownerId, bindingProof: "a".repeat(64), contextProof: "b".repeat(64), terminalDigest: "c".repeat(64),
|
||||
originalActivityAt: terminalAt, originalExpiresAt: expiredAt, retentionDays: 30, safetyMs: 60000,
|
||||
}) })
|
||||
const archive = await v1.create(JSON.stringify({ admission, terminal, journal }), now)
|
||||
assert(archive)
|
||||
const history = {
|
||||
context: { identity: { kind: "mcp", executionId, connectionId, ownerId, credentialId: secret },
|
||||
plan: { request: { receiptId, serverId: 23 }, affectedServerIds: [12, 23], maintenance: [secret] } },
|
||||
receipt: { receiptId, complete: true, maintenanceComplete: true, configurationVerified: true, observation: "online",
|
||||
nativeReads: 4, confirmedPublications: 8, completedPublications: 8, snapshot: { identityName: secret } },
|
||||
source: { createdAt, completedAt, stageCount: 12, publications: 8,
|
||||
nativeOutcomes: Array.from({ length: 4 }, (_, index) => ({ stage: "native." + index, outcome: "confirmed", resultProof: "d".repeat(64) })) },
|
||||
} as unknown as VerifiedServerPollHistory
|
||||
const calls = { decrypt: 0, history: 0 }
|
||||
const codec = createServerPollRetiredArchiveForTests({ openV1: v1.open, verifyHistory: async () => { calls.history++; return history },
|
||||
encrypt: v2Encrypt, proof, decrypt: async bytes => { calls.decrypt++; return decrypt(bytes) } })
|
||||
return { codec, history, calls, input: { archive, terminal, admission: { ...admission, storage_state: "compacted",
|
||||
archive_bytes: archive.ciphertext_bytes, original_activity_at: terminalAt, compact_at: "2026-10-29T03:23:00.123456Z" } } }
|
||||
}
|
||||
const refusal = (work: Promise<unknown>) => assert.rejects(work, (error: unknown) => {
|
||||
assert(error instanceof ServerPollArchiveRefused && error.automaticReplay === false)
|
||||
assert(!String(error).includes(secret))
|
||||
return true
|
||||
})
|
||||
|
||||
test("retirement authenticates V1 and projects only completed history, scope, and original lifetime", async () => {
|
||||
const { codec, calls, input } = await fixture()
|
||||
const retired = await codec.project(input, now)
|
||||
assert(retired)
|
||||
assert.equal(calls.history, 1)
|
||||
assert(retired.ciphertext_bytes < input.archive.ciphertext_bytes)
|
||||
const { history, metadata } = await codec.open(retired)
|
||||
assert.deepEqual(history, { version: 2, operation: "servers.poll", operationVersion: 1, planId, serverId: 23, affectedServerIds: [12, 23],
|
||||
gcNotBefore, terminalAt, previous: { archiveProof: input.archive.archive_proof, sourceDigest: input.archive.source_digest, sourceSha256: input.archive.source_sha256 },
|
||||
history: { createdAt, completedAt, stageCount: 12, publications: 8, nativeReads: 4, observation: "online", verification: "passed" } })
|
||||
assert.equal(metadata.terminal_digest, input.archive.terminal_digest)
|
||||
assert.equal(metadata.original_activity_at, terminalAt)
|
||||
assert.equal(metadata.original_expires_at, expiredAt)
|
||||
assert.equal(metadata.retention_days, input.archive.retention_days)
|
||||
assert(!JSON.stringify(history).includes(secret))
|
||||
assert.equal((await codec.verifyCharge(retired)).bytes, retired.ciphertext_bytes + 32768)
|
||||
})
|
||||
|
||||
test("retirement refuses unauthenticated V1 and any current admission or original terminal drift", async () => {
|
||||
const { codec, input, calls } = await fixture()
|
||||
const changed = [
|
||||
{ ...input, archive: { ...input.archive, archive_proof: "e".repeat(64) } },
|
||||
{ ...input, admission: { ...input.admission, gc_not_before: terminalAt } },
|
||||
{ ...input, admission: { ...input.admission, original_activity_at: "2026-10-29T03:21:00.123458Z" } },
|
||||
{ ...input, admission: { ...input.admission, archive_bytes: 1 } },
|
||||
{ ...input, terminal: { ...input.terminal, step: { result: "changed" } } },
|
||||
]
|
||||
for (const row of changed) await refusal(codec.project(row, now))
|
||||
assert.equal(calls.history, 0)
|
||||
})
|
||||
|
||||
test("V2 metadata or ciphertext tamper never reaches V2 decrypt", async () => {
|
||||
const { codec, input, calls } = await fixture()
|
||||
const retired = await codec.project(input, now)
|
||||
assert(retired)
|
||||
for (const row of [
|
||||
{ ...retired, version: 1 }, { ...retired, owner_id: "another-owner" },
|
||||
{ ...retired, original_expires_at: "2026-10-29T03:20:00.123457Z" },
|
||||
{ ...retired, terminal_digest: "e".repeat(64) },
|
||||
{ ...retired, ciphertext: (retired.ciphertext[0] === "A" ? "B" : "A") + retired.ciphertext.slice(1) },
|
||||
]) await refusal(codec.open(row))
|
||||
assert.equal(calls.decrypt, 0)
|
||||
await refusal(codec.verifyCharge(input.archive))
|
||||
await refusal(codec.verifyCharge({ ...retired, ciphertext_bytes: retired.ciphertext_bytes - 1 }))
|
||||
})
|
||||
|
||||
test("retirement refuses unknown history or a scope that omits its original target", async () => {
|
||||
const { codec, history, input } = await fixture()
|
||||
history.source.nativeOutcomes[0].outcome = "read-failed"
|
||||
await refusal(codec.project(input, now))
|
||||
history.source.nativeOutcomes[0].outcome = "confirmed"
|
||||
history.context.plan.affectedServerIds = [12]
|
||||
await refusal(codec.project(input, now))
|
||||
})
|
||||
|
||||
test("a larger V2 ciphertext keeps the authenticated V1 instead of increasing the charge", async () => {
|
||||
const { codec, input } = await fixture(async value => encrypt(Buffer.concat([Buffer.alloc(4096), value])))
|
||||
assert.equal(await codec.project(input, now), undefined)
|
||||
})
|
||||
@@ -0,0 +1,168 @@
|
||||
/** Private retirement projection. Publication still requires the SQL owner's
|
||||
* locks, exact compare-and-swap, dependency checks and atomic quota adjustment. */
|
||||
import { createHash, timingSafeEqual } from "node:crypto"
|
||||
import { gzipSync, gunzipSync } from "node:zlib"
|
||||
import { z } from "zod"
|
||||
import { canonicalPollArchiveTimestamp, openServerPollTerminalArchive, ServerPollArchiveRefused } from "./server-poll-terminal-archive.js"
|
||||
import { verifyServerPollHistory } from "./server-poll-history-verification.js"
|
||||
import { decryptBinary, encryptBinary, fingerprint, privateSecretFingerprint } from "../modules/mcp/security.js"
|
||||
|
||||
const MAX_PAYLOAD_BYTES = 24576
|
||||
const MAX_COMPRESSED_BYTES = MAX_PAYLOAD_BYTES
|
||||
const MAX_CIPHERTEXT_BYTES = 33000
|
||||
const RECORD_PURPOSE = "server-poll-retired-archive-record-v2"
|
||||
const SOURCE_PURPOSE = "server-poll-retired-archive-source-v2"
|
||||
const hash = z.string().regex(/^[a-f0-9]{64}$/), uuid = z.string().uuid()
|
||||
const integer = z.number().int().positive().safe()
|
||||
const owner = z.string().min(1).max(200).refine(value => value.trim() === value && !/[\x00-\x1f\x7f]/.test(value))
|
||||
const utc = z.string().transform(canonicalPollArchiveTimestamp)
|
||||
const metadataSchema = z.object({
|
||||
version: z.literal(2), receipt_id: uuid, execution_id: uuid, connection_id: uuid, owner_id: owner,
|
||||
binding_proof: hash, context_proof: hash, terminal_digest: hash,
|
||||
original_activity_at: utc, original_expires_at: utc, retention_days: integer.max(365), safety_ms: z.literal(60000),
|
||||
source_digest: hash, source_sha256: hash, decoded_bytes: integer.max(MAX_PAYLOAD_BYTES),
|
||||
compressed_bytes: integer.max(MAX_COMPRESSED_BYTES), ciphertext_bytes: integer.max(MAX_CIPHERTEXT_BYTES), ciphertext_sha256: hash,
|
||||
})
|
||||
const projectionSchema = z.object({
|
||||
version: z.literal(2), operation: z.literal("servers.poll"), operationVersion: z.literal(1),
|
||||
planId: uuid, serverId: integer, affectedServerIds: z.array(integer).min(1).max(1000),
|
||||
gcNotBefore: utc, terminalAt: utc,
|
||||
previous: z.object({ archiveProof: hash, sourceDigest: hash, sourceSha256: hash }).strict(),
|
||||
history: z.object({
|
||||
createdAt: utc, completedAt: utc, stageCount: integer.max(2008),
|
||||
publications: integer.max(2008), nativeReads: z.literal(4),
|
||||
observation: z.literal("online"), verification: z.literal("passed"),
|
||||
}).strict(),
|
||||
}).strict().refine(value => value.affectedServerIds.includes(value.serverId)
|
||||
&& value.affectedServerIds.length === new Set(value.affectedServerIds).size
|
||||
&& value.history.createdAt <= value.history.completedAt
|
||||
&& value.history.completedAt <= value.terminalAt)
|
||||
|
||||
export type ServerPollRetiredArchiveMetadata = z.infer<typeof metadataSchema>
|
||||
export type ServerPollRetiredHistory = z.infer<typeof projectionSchema>
|
||||
export type ServerPollRetiredArchive = ServerPollRetiredArchiveMetadata & { ciphertext: string; archive_proof: string }
|
||||
|
||||
export interface LockedServerPollRetirementSource {
|
||||
/** Existing V1 reader projection, including retention and actual ciphertext digest. */
|
||||
archive: Record<string, unknown>
|
||||
/** Current compacted admission JSON and exact operation, plan and step JSON. */
|
||||
admission: Record<string, unknown>
|
||||
terminal: { operation: Record<string, unknown>; plan: Record<string, unknown>; step: Record<string, unknown> }
|
||||
}
|
||||
|
||||
interface Dependencies {
|
||||
openV1: typeof openServerPollTerminalArchive
|
||||
verifyHistory: typeof verifyServerPollHistory
|
||||
encrypt(value: Buffer): Promise<Buffer>
|
||||
decrypt(value: Buffer): Promise<Buffer>
|
||||
proof(purpose: string, value: unknown): Promise<string>
|
||||
}
|
||||
const sha = (value: Buffer | string) => createHash("sha256").update(value).digest("hex")
|
||||
const equal = (left: string, right: string) => timingSafeEqual(Buffer.from(hash.parse(left), "hex"), Buffer.from(hash.parse(right), "hex"))
|
||||
const refuse = (): never => { throw new ServerPollArchiveRefused() }
|
||||
const requireThat: (value: unknown) => asserts value = value => { if (!value) refuse() }
|
||||
|
||||
function immutableAdmission(raw: Record<string, unknown>) {
|
||||
const result = { ...raw }
|
||||
for (const name of ["storage_state", "compact_at", "archive_bytes", "original_activity_at"]) delete result[name]
|
||||
return result
|
||||
}
|
||||
|
||||
function codec(deps: Dependencies) {
|
||||
const proof = async (purpose: string, value: unknown) => hash.parse(await deps.proof(purpose, value))
|
||||
const recordProof = (metadata: ServerPollRetiredArchiveMetadata) => proof(RECORD_PURPOSE, {
|
||||
...metadata, identity_kind: "mcp", storage_state: "compacted", admission_bytes: 32768, charged_rows: 2,
|
||||
})
|
||||
async function readMetadata(raw: Record<string, unknown>) {
|
||||
const metadata = metadataSchema.parse(raw)
|
||||
requireThat(equal(hash.parse(raw.archive_proof), await recordProof(metadata)))
|
||||
return metadata
|
||||
}
|
||||
|
||||
async function project(raw: LockedServerPollRetirementSource, now: number): Promise<ServerPollRetiredArchive | undefined> {
|
||||
try {
|
||||
requireThat(Number.isSafeInteger(now) && now > 0)
|
||||
const input = structuredClone(raw)
|
||||
const opened = await deps.openV1(input.archive, now)
|
||||
const { source, metadata: previous } = opened
|
||||
const current = input.admission
|
||||
requireThat(source.admission.storage_state === "full" && current.storage_state === "compacted"
|
||||
&& current.archive_bytes === previous.ciphertext_bytes
|
||||
&& canonicalPollArchiveTimestamp(current.original_activity_at) === previous.original_activity_at
|
||||
&& fingerprint(immutableAdmission(source.admission)) === fingerprint(immutableAdmission(current))
|
||||
&& fingerprint(source.terminal) === fingerprint(input.terminal))
|
||||
canonicalPollArchiveTimestamp(current.compact_at)
|
||||
|
||||
// The existing codec authenticates the entire original. The shared
|
||||
// verifier supplies its typed history rather than reading opaque fields
|
||||
// from unverified context or preserving original authority in V2.
|
||||
const { context, receipt, source: journal } = await deps.verifyHistory(source.journal)
|
||||
requireThat(context.identity.kind === "mcp" && context.identity.executionId === previous.execution_id
|
||||
&& context.identity.connectionId === previous.connection_id && context.identity.ownerId === previous.owner_id
|
||||
&& context.plan.request.receiptId === previous.receipt_id && receipt.receiptId === previous.receipt_id
|
||||
&& receipt.complete && receipt.maintenanceComplete && receipt.configurationVerified && receipt.observation === "online"
|
||||
&& receipt.confirmedPublications === receipt.completedPublications && journal.publications === receipt.completedPublications
|
||||
&& journal.nativeOutcomes.length === 4 && journal.nativeOutcomes.every(item => item.outcome === "confirmed"))
|
||||
const terminalAt = [source.terminal.operation.finished_at, source.terminal.plan.expires_at].map(canonicalPollArchiveTimestamp).sort()[1]
|
||||
const projection = projectionSchema.parse({
|
||||
version: 2, operation: "servers.poll", operationVersion: 1, planId: source.terminal.plan.id,
|
||||
serverId: context.plan.request.serverId, affectedServerIds: context.plan.affectedServerIds,
|
||||
gcNotBefore: source.admission.gc_not_before, terminalAt,
|
||||
previous: { archiveProof: input.archive.archive_proof, sourceDigest: previous.source_digest, sourceSha256: previous.source_sha256 },
|
||||
history: { createdAt: source.journal.execution.created_at, completedAt: source.journal.receipt.created_at,
|
||||
stageCount: journal.stageCount, publications: journal.publications, nativeReads: receipt.nativeReads,
|
||||
observation: receipt.observation, verification: "passed" },
|
||||
})
|
||||
requireThat(projection.terminalAt <= previous.original_activity_at && previous.original_expires_at <= projection.gcNotBefore)
|
||||
const plaintext = Buffer.from(JSON.stringify(projection))
|
||||
requireThat(plaintext.length <= MAX_PAYLOAD_BYTES)
|
||||
const compressed = gzipSync(plaintext, { level: 6, maxOutputLength: MAX_COMPRESSED_BYTES })
|
||||
const ciphertext = (await deps.encrypt(compressed)).toString("base64")
|
||||
const bytes = Buffer.byteLength(ciphertext)
|
||||
if (bytes > previous.ciphertext_bytes || bytes > MAX_CIPHERTEXT_BYTES) return undefined
|
||||
const sourceSha = sha(plaintext)
|
||||
const metadata = metadataSchema.parse({ ...previous, version: 2, source_sha256: sourceSha, source_digest: await proof(SOURCE_PURPOSE, sourceSha),
|
||||
decoded_bytes: plaintext.length, compressed_bytes: compressed.length, ciphertext_bytes: bytes, ciphertext_sha256: sha(ciphertext) })
|
||||
return { ...metadata, ciphertext, archive_proof: await recordProof(metadata) }
|
||||
} catch { return refuse() }
|
||||
}
|
||||
|
||||
async function verifyCharge(raw: Record<string, unknown>) {
|
||||
try {
|
||||
const metadata = await readMetadata(structuredClone(raw))
|
||||
return { bytes: metadata.ciphertext_bytes + 32768, rows: 2 as const, metadata }
|
||||
} catch { return refuse() }
|
||||
}
|
||||
|
||||
async function open(raw: Record<string, unknown>) {
|
||||
try {
|
||||
const input = structuredClone(raw)
|
||||
const metadata = await readMetadata(input), ciphertext = input.ciphertext
|
||||
requireThat(typeof ciphertext === "string" && Buffer.byteLength(ciphertext) === metadata.ciphertext_bytes
|
||||
&& sha(ciphertext) === metadata.ciphertext_sha256)
|
||||
const binary = Buffer.from(ciphertext, "base64")
|
||||
requireThat(binary.toString("base64") === ciphertext)
|
||||
const compressed = await deps.decrypt(binary)
|
||||
requireThat(compressed.length === metadata.compressed_bytes)
|
||||
const plaintext = gunzipSync(compressed, { maxOutputLength: MAX_PAYLOAD_BYTES })
|
||||
const text = plaintext.toString("utf8")
|
||||
requireThat(plaintext.length === metadata.decoded_bytes && Buffer.from(text).equals(plaintext)
|
||||
&& sha(plaintext) === metadata.source_sha256 && equal(metadata.source_digest, await proof(SOURCE_PURPOSE, metadata.source_sha256)))
|
||||
const history = projectionSchema.parse(JSON.parse(text))
|
||||
requireThat(history.terminalAt <= metadata.original_activity_at && metadata.original_expires_at <= history.gcNotBefore)
|
||||
return { metadata, history }
|
||||
} catch { return refuse() }
|
||||
}
|
||||
return Object.freeze({ project, verifyCharge, open })
|
||||
}
|
||||
|
||||
const production = codec({
|
||||
openV1: openServerPollTerminalArchive, verifyHistory: verifyServerPollHistory,
|
||||
encrypt: encryptBinary, decrypt: decryptBinary,
|
||||
proof: (purpose, value) => privateSecretFingerprint(fingerprint(value), purpose),
|
||||
})
|
||||
export const projectServerPollRetiredArchive = production.project
|
||||
export const verifyServerPollRetiredArchiveCharge = production.verifyCharge
|
||||
export const openServerPollRetiredArchive = production.open
|
||||
/** Isolated component crypto composition. No public/runtime caller selects it. */
|
||||
export const createServerPollRetiredArchiveForTests = (deps: Dependencies) => codec(Object.freeze({ ...deps }))
|
||||
@@ -34,8 +34,40 @@ try{
|
||||
}finally{await other.end();await pool.query(`DROP DATABASE "${name}"`)}
|
||||
const {tickMcpWorkers}=await import("../modules/mcp/workers.js")
|
||||
assert.equal((await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled,false)
|
||||
const expiredRequestIds = Array.from({ length: 1001 }, () => randomUUID()).sort()
|
||||
const [runningId, freshId] = [randomUUID(), randomUUID()].sort()
|
||||
await pool.query(`INSERT INTO mcp_requests(id,tool,status,created_at)
|
||||
SELECT id,'owned-retention-test','completed',now()-interval '31 days' FROM unnest($1::uuid[]) AS ids(id)
|
||||
UNION ALL SELECT $2::uuid,'owned-retention-test','running',now()-interval '32 days'
|
||||
UNION ALL SELECT $3::uuid,'owned-retention-test','completed',now()-interval '29 days'`, [expiredRequestIds, runningId, freshId])
|
||||
const expiredAudit = (await pool.query(`INSERT INTO mcp_audit(actor_id,action,created_at)
|
||||
SELECT 'owned-retention-test','expired',now()-interval '181 days' FROM generate_series(1,1001)
|
||||
RETURNING id`)).rows.map(row => String(row.id))
|
||||
const lastExpiredAuditId = expiredAudit.reduce((last, id) => BigInt(id) > BigInt(last) ? id : last)
|
||||
const freshAuditId = (await pool.query(`INSERT INTO mcp_audit(actor_id,action,created_at)
|
||||
VALUES ('owned-retention-test','fresh',now()-interval '179 days') RETURNING id`)).rows[0].id
|
||||
const artifactConnectionId = randomUUID()
|
||||
const expiredArtifactIds = Array.from({ length: 1001 }, () => randomUUID()).sort()
|
||||
const freshArtifactId = randomUUID()
|
||||
await pool.query(`INSERT INTO mcp_connections(id,name,owner_id,owner_permissions,policy,generation)
|
||||
VALUES ($1,'owned-retention-test','owned-retention-test','[]','{}','owned-retention-test')`, [artifactConnectionId])
|
||||
await pool.query(`INSERT INTO mcp_artifacts(id,connection_id,filename,media_type,ciphertext,expires_at)
|
||||
SELECT id,$1::uuid,'test','application/octet-stream','test',now()-interval '1 day' FROM unnest($2::uuid[]) AS ids(id)
|
||||
UNION ALL SELECT $3::uuid,$1::uuid,'test','application/octet-stream','test',now()+interval '1 day'`, [artifactConnectionId, expiredArtifactIds, freshArtifactId])
|
||||
const requestIds = async () => (await pool.query("SELECT id FROM mcp_requests WHERE tool='owned-retention-test' ORDER BY id")).rows.map(row => row.id)
|
||||
const auditIds = async () => (await pool.query("SELECT id FROM mcp_audit WHERE actor_id='owned-retention-test' ORDER BY id")).rows.map(row => String(row.id))
|
||||
const artifactIds = async () => (await pool.query("SELECT id FROM mcp_artifacts WHERE connection_id=$1 ORDER BY id", [artifactConnectionId])).rows.map(row => row.id)
|
||||
await tickMcpWorkers()
|
||||
console.log("Owned fresh poll storage:",JSON.stringify({normalStartupTwice:true,migrations:MIGRATION_MANIFEST.length,globalZeroSeed:true,legacyPrefix:38,nonemptyJournalRefusedBeforeDdl:true,oldRowsPreserved:true,noForgedMarker:true,actualWorkerTickWithMcpDisabled:true}))
|
||||
assert.deepEqual(await requestIds(), [...expiredRequestIds.slice(-1), runningId, freshId].sort())
|
||||
assert.deepEqual(await auditIds(), [lastExpiredAuditId, String(freshAuditId)])
|
||||
assert.deepEqual(await artifactIds(), [...expiredArtifactIds.slice(-1), freshArtifactId].sort())
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await requestIds(), [runningId, freshId].sort())
|
||||
assert.deepEqual(await auditIds(), [String(freshAuditId)])
|
||||
assert.deepEqual(await artifactIds(), [freshArtifactId])
|
||||
await pool.query("DELETE FROM mcp_artifacts WHERE id=$1", [freshArtifactId])
|
||||
await pool.query("DELETE FROM mcp_connections WHERE id=$1", [artifactConnectionId])
|
||||
console.log("Owned fresh poll storage:",JSON.stringify({normalStartupTwice:true,migrations:MIGRATION_MANIFEST.length,globalZeroSeed:true,legacyPrefix:38,nonemptyJournalRefusedBeforeDdl:true,oldRowsPreserved:true,noForgedMarker:true,actualWorkerTickWithMcpDisabled:true,boundedRequestAuditAndArtifactRetention:true}))
|
||||
if (process.env.MMAPP_OWN_SERVER_POLL_SETTINGS_EPOCH === "true") {
|
||||
await (await import("../db/server-poll-settings-epoch.owned-helper.js")).acceptServerPollSettingsEpochMigrationSql(pool)
|
||||
} else if (process.env.MMAPP_OWN_SERVER_POLL_ATOMIC_MAINTENANCE === "true") {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { createHash, createHmac, randomUUID } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import { gzipSync } from "node:zlib"
|
||||
import { fingerprint } from "../modules/mcp/security.js"
|
||||
import { collectExpiredServerPollStorageForTests } from "./server-poll-storage-retention.js"
|
||||
import { PollStorageRefused, PollStorageUnknown } from "./server-poll-storage.js"
|
||||
import type { ProbeStorageOptions } from "./server-probe-journal-maintenance.js"
|
||||
@@ -8,8 +10,10 @@ import { sharedDatabaseGate } from "./shared-database-gate.js"
|
||||
|
||||
const tables = ["server_poll_acks", "server_poll_intents", "server_poll_receipts", "server_poll_quarantines", "server_poll_executions", "server_poll_archives", "server_poll_storage_admissions"]
|
||||
const deferred = () => { let resolve!: () => void; const promise = new Promise<void>(done => { resolve = done }); return { promise, resolve } }
|
||||
function fixture() {
|
||||
const now = Date.now(), candidates: Record<string, unknown>[] = [{ receipt_id: randomUUID(), owner_id: "owned-gc" }, { receipt_id: randomUUID(), owner_id: "owned-gc" }]
|
||||
function fixture(openRetired?: Parameters<typeof collectExpiredServerPollStorageForTests>[1]) {
|
||||
const now = Date.now(), candidates: Record<string, unknown>[] = [{ receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: null },
|
||||
{ receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: 1 }]
|
||||
const retiredRows: Record<string, unknown>[] = []
|
||||
const commands: { text: string; values: unknown[]; timeout: number }[] = [], releases: boolean[] = [], options: ProbeStorageOptions[] = []
|
||||
let settings: Record<string, unknown>[] = [{ requests_retention_days: 30 }], fail = "", residual = "", removed: Record<string, unknown>[] | undefined, hold: ReturnType<typeof deferred> | undefined
|
||||
const entered = deferred()
|
||||
@@ -25,12 +29,13 @@ function fixture() {
|
||||
return { rows: [{ scope: "global", admissions: 0, compacted_admissions: 0, archive_bytes: 0, reserved_rows: 0, reserved_bytes: 0 }] }
|
||||
if (text.includes("requests_retention_days")) { entered.resolve(); if (hold) await hold.promise; return { rows: settings } }
|
||||
if (text.startsWith("SELECT receipt_id,owner_id")) return { rows: structuredClone(candidates) }
|
||||
if (text.startsWith("SELECT r.version,r.receipt_id")) return { rows: structuredClone(retiredRows) }
|
||||
if (text.startsWith("DELETE FROM server_poll_storage_admissions")) return { rows: removed ?? candidates.map(row => ({ receipt_id: row.receipt_id })) }
|
||||
if (text.startsWith("SELECT receipt_id FROM ") && text.includes("FROM " + residual + " ")) return { rows: [{ receipt_id: candidates[0]?.receipt_id }] }
|
||||
return { rows: [] }
|
||||
}, release(unsafe = false) { releases.push(unsafe) } }
|
||||
} })
|
||||
return { run, now, candidates, commands, releases, options, entered: entered.promise, block: () => { hold = deferred(); return hold },
|
||||
} }, openRetired)
|
||||
return { run, now, candidates, retiredRows, commands, releases, options, entered: entered.promise, block: () => { hold = deferred(); return hold },
|
||||
setting: (rows: Record<string, unknown>[]) => { settings = rows }, fail: (text: string) => { fail = text }, residual: (table: string) => { residual = table }, removed: (rows: Record<string, unknown>[]) => { removed = rows } }
|
||||
}
|
||||
|
||||
@@ -61,6 +66,67 @@ test("selected GC owns one bounded backend, locks current retention, deletes all
|
||||
assert.deepEqual(f.releases, [false])
|
||||
})
|
||||
|
||||
async function retiredFixture() {
|
||||
const { createServerPollRetiredArchiveForTests } = await import("./server-poll-retired-archive.js")
|
||||
const proof = async (purpose: string, value: unknown) =>
|
||||
createHmac("sha256", "gc-fixture-only").update(purpose).update(fingerprint(value)).digest("hex")
|
||||
const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex")
|
||||
let opened = 0
|
||||
const codec = createServerPollRetiredArchiveForTests({
|
||||
proof,
|
||||
decrypt: async value => { opened++; return value },
|
||||
encrypt: async () => { throw new Error("Unexpected archive write") },
|
||||
openV1: async () => { throw new Error("Unexpected archive fallback") },
|
||||
verifyHistory: async () => { throw new Error("Unexpected historical fallback") },
|
||||
})
|
||||
const f = fixture(codec.open)
|
||||
const selected = f.candidates[1]
|
||||
selected.archive_version = 2
|
||||
const timestamp = "2026-10-01T00:00:00.123456Z", gcNotBefore = "2026-11-01T00:00:00.123456Z"
|
||||
const history = { version: 2, operation: "servers.poll", operationVersion: 1, planId: randomUUID(), serverId: 3,
|
||||
affectedServerIds: [3], gcNotBefore, terminalAt: timestamp,
|
||||
previous: { archiveProof: "1".repeat(64), sourceDigest: "2".repeat(64), sourceSha256: "3".repeat(64) },
|
||||
history: { createdAt: timestamp, completedAt: timestamp, stageCount: 12, publications: 8,
|
||||
nativeReads: 4, observation: "online", verification: "passed" } }
|
||||
const plaintext = Buffer.from(JSON.stringify(history)), compressed = gzipSync(plaintext), ciphertext = compressed.toString("base64")
|
||||
const metadata = { version: 2, receipt_id: selected.receipt_id, execution_id: randomUUID(), connection_id: randomUUID(), owner_id: selected.owner_id,
|
||||
binding_proof: "a".repeat(64), context_proof: "b".repeat(64), terminal_digest: "c".repeat(64),
|
||||
original_activity_at: timestamp, original_expires_at: timestamp, retention_days: 30, safety_ms: 60000,
|
||||
source_digest: await proof("server-poll-retired-archive-source-v2", sha(plaintext)), source_sha256: sha(plaintext),
|
||||
decoded_bytes: plaintext.length, compressed_bytes: compressed.length, ciphertext_bytes: ciphertext.length, ciphertext_sha256: sha(ciphertext) }
|
||||
const archiveProof = await proof("server-poll-retired-archive-record-v2", { ...metadata, identity_kind: "mcp",
|
||||
storage_state: "compacted", admission_bytes: 32768, charged_rows: 2 })
|
||||
f.retiredRows.push({ ...metadata, ciphertext, archive_proof: archiveProof, admission_gc_not_before: gcNotBefore })
|
||||
return { ...f, opened: () => opened }
|
||||
}
|
||||
|
||||
test("GC opens only selected V2 under a row lock and authenticates the original lifetime before any DELETE", async () => {
|
||||
const f = await retiredFixture()
|
||||
assert.deepEqual(await f.run(), { collected: 2, automaticReplay: false })
|
||||
assert.equal(f.opened(), 1)
|
||||
const index = f.commands.findIndex(command => command.text.startsWith("SELECT r.version,r.receipt_id"))
|
||||
assert.deepEqual(f.commands[index].values, [f.candidates[1].receipt_id])
|
||||
assert(f.commands[index].text.endsWith("FOR UPDATE OF r"))
|
||||
assert(index < f.commands.findIndex(command => command.text.startsWith("DELETE")))
|
||||
})
|
||||
|
||||
test("missing V2, forged ciphertext or proof, and microsecond lifetime drift refuse before any DELETE", async () => {
|
||||
for (const fault of ["missing", "ciphertext", "actual-hash", "proof", "lifetime", "expiry", "version"] as const) {
|
||||
const f = await retiredFixture(), row = f.retiredRows[0]
|
||||
if (fault === "missing") f.retiredRows.length = 0
|
||||
if (fault === "ciphertext") row.ciphertext = "A" + String(row.ciphertext).slice(1)
|
||||
if (fault === "actual-hash") row.ciphertext_sha256 = "a".repeat(64)
|
||||
if (fault === "proof") row.archive_proof = "a".repeat(64)
|
||||
if (fault === "lifetime") row.admission_gc_not_before = "2026-11-01T00:00:00.123455Z"
|
||||
if (fault === "expiry") row.original_expires_at = "2026-10-01T00:00:00.123455Z"
|
||||
if (fault === "version") f.candidates[1].archive_version = 3
|
||||
await assert.rejects(f.run(), PollStorageRefused)
|
||||
assert.equal(f.opened(), fault === "lifetime" ? 1 : 0)
|
||||
assert(!f.commands.some(command => command.text.startsWith("DELETE") || command.text === "COMMIT"))
|
||||
assert.equal(f.commands.filter(command => command.text === "ROLLBACK").length, 1)
|
||||
}
|
||||
})
|
||||
|
||||
test("empty candidate set returns zero without deleting or refunding", async () => {
|
||||
const f = fixture(); f.candidates.length = 0
|
||||
assert.deepEqual(await f.run(), { collected: 0, automaticReplay: false })
|
||||
|
||||
@@ -4,21 +4,50 @@ import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
||||
import { runPollStorageTransaction, type PollStorageDependencies } from "./server-poll-storage.js"
|
||||
import { PollStorageRefused } from "./server-poll-storage-model.js"
|
||||
import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js"
|
||||
import type { openServerPollRetiredArchive } from "./server-poll-retired-archive.js"
|
||||
|
||||
const uuid=/^[a-f\d]{8}(?:-[a-f\d]{4}){3}-[a-f\d]{12}$/i
|
||||
const fail=():never=>{throw new PollStorageRefused()}
|
||||
const production:PollStorageDependencies={connect:connectProbeStorage,now:Date.now}
|
||||
async function collect(deps:PollStorageDependencies){
|
||||
const openRetiredArchive:typeof openServerPollRetiredArchive=async row=>
|
||||
(await import("./server-poll-retired-archive.js")).openServerPollRetiredArchive(row)
|
||||
const retiredCandidate = `SELECT r.version,r.receipt_id,r.execution_id,r.connection_id,r.owner_id,
|
||||
r.binding_proof,r.context_proof,r.terminal_digest,r.source_digest,r.source_sha256,
|
||||
r.decoded_bytes,r.compressed_bytes,r.ciphertext_bytes,r.ciphertext,r.archive_proof,a.retention_days,a.safety_ms,
|
||||
to_char(r.original_activity_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_activity_at,
|
||||
to_char(r.original_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS original_expires_at,
|
||||
to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS admission_gc_not_before,
|
||||
pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(r.ciphertext,'UTF8')),'hex') AS ciphertext_sha256
|
||||
FROM server_poll_archives r JOIN server_poll_storage_admissions a USING(receipt_id)
|
||||
WHERE r.receipt_id=$1 AND r.version=2 AND a.storage_state='compacted' AND a.identity_kind='mcp'
|
||||
AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id
|
||||
AND r.binding_proof=a.binding_proof AND r.context_proof=a.context_proof
|
||||
AND r.original_activity_at=a.original_activity_at AND r.original_expires_at=a.authority_expires_at
|
||||
AND r.ciphertext_bytes=a.archive_bytes AND r.ciphertext_bytes=octet_length(r.ciphertext)
|
||||
FOR UPDATE OF r`
|
||||
|
||||
async function collect(deps:PollStorageDependencies, openRetired:typeof openServerPollRetiredArchive){
|
||||
return runPollStorageTransaction(deps,{deadlineAt:deps.now()+20000},async()=>{},async sql=>{
|
||||
// Hold the authoritative setting against concurrent updates until COMMIT.
|
||||
// Each DELETE guard additionally checks its current value independently.
|
||||
const setting=(await sql.query("SELECT requests_retention_days FROM mcp_settings WHERE id=1 FOR SHARE")).rows
|
||||
if(setting.length!==1||!Number.isSafeInteger(setting[0].requests_retention_days)||Number(setting[0].requests_retention_days)<1||Number(setting[0].requests_retention_days)>365)fail()
|
||||
await sql.query("SET LOCAL mm.poll_storage_gc='on'")
|
||||
const rows=(await sql.query("SELECT receipt_id,owner_id FROM server_poll_storage_admissions WHERE gc_not_before<clock_timestamp() AND server_poll_storage_gc_allowed(receipt_id) ORDER BY gc_not_before,receipt_id LIMIT 8 FOR UPDATE")).rows
|
||||
if(rows.length>8||rows.some(row=>typeof row.receipt_id!=="string"||!uuid.test(row.receipt_id)||typeof row.owner_id!=="string")||new Set(rows.map(row=>row.receipt_id)).size!==rows.length)fail()
|
||||
const rows=(await sql.query(`SELECT receipt_id,owner_id,
|
||||
(SELECT version FROM server_poll_archives WHERE receipt_id=a.receipt_id) AS archive_version
|
||||
FROM server_poll_storage_admissions a WHERE gc_not_before<clock_timestamp()
|
||||
AND server_poll_storage_gc_allowed(receipt_id) ORDER BY gc_not_before,receipt_id LIMIT 8 FOR UPDATE`)).rows
|
||||
if(rows.length>8||rows.some(row=>typeof row.receipt_id!=="string"||!uuid.test(row.receipt_id)||typeof row.owner_id!=="string"
|
||||
|| (row.archive_version!==null&&row.archive_version!==1&&row.archive_version!==2))||new Set(rows.map(row=>row.receipt_id)).size!==rows.length)fail()
|
||||
if(!rows.length)return {collected:0,automaticReplay:false as const}
|
||||
await assertServerPollArchiveAccounting(sql)
|
||||
for(const candidate of rows){
|
||||
if(candidate.archive_version!==2)continue
|
||||
const locked=(await sql.query(retiredCandidate,[candidate.receipt_id])).rows
|
||||
if(locked.length!==1||locked[0].version!==2||locked[0].receipt_id!==candidate.receipt_id||locked[0].owner_id!==candidate.owner_id)fail()
|
||||
const {history}=await openRetired(locked[0])
|
||||
if(history.gcNotBefore!==locked[0].admission_gc_not_before)fail()
|
||||
}
|
||||
const ids=rows.map(row=>row.receipt_id),owners=[...new Set(rows.map(row=>row.owner_id))]
|
||||
// No payload-only shortcut: all dependent rows, admission and its exact
|
||||
// trigger-owned refund commit together. Any missing guard/refund aborts.
|
||||
@@ -35,6 +64,7 @@ async function collect(deps:PollStorageDependencies){
|
||||
return {collected:ids.length,automaticReplay:false as const}
|
||||
})
|
||||
}
|
||||
export const collectExpiredServerPollStorage=()=>sharedDatabaseGate.request(()=>collect(production))
|
||||
export const collectExpiredServerPollStorage=()=>sharedDatabaseGate.request(()=>collect(production,openRetiredArchive))
|
||||
/** Explicit owned SQL composition; does not start a worker. */
|
||||
export const collectExpiredServerPollStorageForTests=(deps:PollStorageDependencies)=>sharedDatabaseGate.request(()=>collect(deps))
|
||||
export const collectExpiredServerPollStorageForTests=(deps:PollStorageDependencies,openRetired=openRetiredArchive)=>
|
||||
sharedDatabaseGate.request(()=>collect(deps,openRetired))
|
||||
@@ -77,7 +77,7 @@ async function main() {
|
||||
}
|
||||
try {
|
||||
await applySqlMigrations(pool)
|
||||
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0044_mcp_routing_manual_inverse_charges.sql")
|
||||
assert.equal(MIGRATION_MANIFEST.at(-1)?.file, "0053_server_probe_terminal_proofs.sql")
|
||||
assert.equal(MIGRATION_MANIFEST.find(pin=>pin.file==="0041_server_poll_terminal_archives.sql")?.sha256,"fbb8935ce977cfc507801f01beb8f303d4817ed63c6bdd4c05799768dfaac9d4")
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM schema_migrations")).rows[0].n, MIGRATION_MANIFEST.length)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM server_poll_storage_admissions")).rows[0].n, 0)
|
||||
@@ -242,6 +242,12 @@ async function main() {
|
||||
const archivedRows = (await pool.query(SERVER_POLL_ARCHIVE_READ_SQL, [receiptId])).rows
|
||||
assert.equal(archivedRows.length, 1)
|
||||
const archive = archivedRows[0]
|
||||
const { retireCompletedServerPollOperation } = await import("./server-poll-operation-retirement.js")
|
||||
const beforeRetirement = await ledger()
|
||||
await assert.rejects(retireCompletedServerPollOperation({ receiptId, oldArchiveProof: archive.archive_proof }))
|
||||
assert.equal(await ledger(), beforeRetirement, "Authenticated V1 younger than 180 days retains exact charge")
|
||||
assert.equal((await pool.query("SELECT version FROM server_poll_archives WHERE receipt_id=$1", [receiptId])).rows[0].version, 1)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operation_replay_tombstones WHERE operation_id=$1", [a.execution_id])).rows[0].n, 0)
|
||||
assert.equal(Number(archive.actual_ciphertext_bytes), Number(archive.ciphertext_bytes))
|
||||
assert(Number(archive.ciphertext_bytes) > 0 && Number(archive.ciphertext_bytes) <= SERVER_POLL_ARCHIVE_MAX_BYTES)
|
||||
assert.equal(Number(archive.ciphertext_bytes), Buffer.byteLength(archive.ciphertext))
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { assertOwnedBandwidthSql, prepareOwnedBandwidthState } from "./bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_PROBE_OPERATION_RETIREMENT, "true")
|
||||
await prepareOwnedBandwidthState()
|
||||
const { pool, closePool } = await import("../db/index.js")
|
||||
const { applySqlMigrations } = await import("../db/migrate.js")
|
||||
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
||||
const { authenticateMcpToken, createMcpConnection } = await import("../modules/mcp/store.js")
|
||||
const { applyPlan, invokeMcpTool } = await import("../modules/mcp/runtime.js")
|
||||
const { encryptSecret, McpError } = await import("../modules/mcp/security.js")
|
||||
const { serverProbeRetiredFixture } = await import("./server-probe-retired-source.fixture.js")
|
||||
const { captureServerProbeTerminalProof, collectExpiredRetiredServerProbe, retireCompletedServerProbeOperation,
|
||||
retireExpiredServerProbeOperation } = await import("./server-probe-operation-retirement.js")
|
||||
const { createProbeJournalStorage } = await import("./server-probe-journal-maintenance.js")
|
||||
const { connectProbeStorage } = await import("./server-probe-storage-connection.js")
|
||||
const { PROBE_STORAGE_LIMITS } = await import("../modules/mcp/diagnostic-cancellation-storage.js")
|
||||
const { retainServerProbeStorage } = await import("./server-probe-retention.js")
|
||||
const { tickMcpWorkers } = await import("../modules/mcp/workers.js")
|
||||
|
||||
try {
|
||||
await applySqlMigrations(pool)
|
||||
await pool.query("UPDATE mcp_settings SET enabled=true WHERE id=1")
|
||||
await pool.query("INSERT INTO servers(id,name,host,enabled) VALUES(9,'owned retirement source','127.0.0.1',true)")
|
||||
async function access() {
|
||||
const actor = { id: randomUUID(), name: "Owned diagnostic retention", email: "[email protected]", apps: ["mm"],
|
||||
permissions: ["mm:servers:read", "mm:servers:write", "mm:mcp:approve"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 }
|
||||
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval", permissions: actor.permissions,
|
||||
expiresAt: new Date(actor.expiresAt).toISOString(), scope: { serverIds: [9],
|
||||
operationIds: ["servers.probes.run", "servers.probes.receipt.read", "operations.read", "changes.apply"], allowGlobal: false },
|
||||
limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor)
|
||||
return { actor, principal: await authenticateMcpToken(issued.token) }
|
||||
}
|
||||
async function verifyLifecycle(mode: "retained" | "purged" | "default-old" | "default-young" | "default-unproven" | "default-spent" | "scheduled-retained") {
|
||||
const { actor, principal } = await access(), foreign = (await access()).principal
|
||||
const historical = mode === "retained" ? undefined : {
|
||||
base: Date.now()-(mode === "default-young" ? 60 : 200)*86400000, retentionDays:mode === "scheduled-retained" ? 365 : 30,
|
||||
}
|
||||
const { raw } = await serverProbeRetiredFixture(false, { connectionId: principal.connectionId, credentialId: principal.credentialId, ownerId: actor.id }, historical)
|
||||
const { original: o, admission: a } = raw, key = randomUUID(), empty = await encryptSecret("[]")
|
||||
const target = { receiptId: a.receipt_id, operationId: o.operation_id, connectionId: o.connection_id }
|
||||
const request = { operationId: o.operation_id, receiptId: a.receipt_id, page: 0 }
|
||||
const own = await pool.connect()
|
||||
try {
|
||||
await own.query("BEGIN")
|
||||
// Only this disposable database gets synthetic historical SQL rows. The
|
||||
// actual codec signs the historical originals with its owned state key.
|
||||
await own.query(`INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at,approved_by,approved_at,created_at)
|
||||
VALUES($1,$2,$3,1,1,'owned historical test','{}',$4,'completed',$5,$6,$7,$8)`,
|
||||
[o.plan_id,o.connection_id,o.operation_name,o.evidence,o.plan_expires_at,o.owner_id,o.plan_approved_at,o.plan_created_at])
|
||||
await own.query(`INSERT INTO mcp_operations(id,plan_id,connection_id,idempotency_key,state,verification,result,created_at,finished_at)
|
||||
VALUES($1,$2,$3,$4,'applied','inconclusive',$5,$6,$7)`, [o.operation_id,o.plan_id,o.connection_id,key,o.operation_result,o.operation_created_at,o.operation_finished_at])
|
||||
await own.query(`INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state,result,created_at,finished_at)
|
||||
VALUES($1,0,'acknowledged',$2,$3,$4,$5)`, [o.operation_id,o.intended_state,o.step_result,o.step_created_at,o.step_finished_at])
|
||||
await own.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed,after_sealed) VALUES($1,0,$2,$2)", [o.operation_id,empty])
|
||||
await own.query("SET LOCAL session_replication_role='replica'")
|
||||
await own.query("INSERT INTO server_probe_storage_admissions SELECT * FROM jsonb_populate_record(NULL::server_probe_storage_admissions,$1)", [a])
|
||||
await own.query("INSERT INTO server_probe_executions SELECT * FROM jsonb_populate_record(NULL::server_probe_executions,$1)", [raw.anchor])
|
||||
await own.query("INSERT INTO server_probe_execution_payloads SELECT * FROM jsonb_populate_record(NULL::server_probe_execution_payloads,$1)", [raw.payload])
|
||||
await own.query("INSERT INTO server_probe_execution_stages SELECT * FROM jsonb_populate_record(NULL::server_probe_execution_stages,$1)", [raw.stages[0]])
|
||||
await own.query("UPDATE server_probe_storage_usage SET admissions=1,reserved_files=1,reserved_bytes=294912 WHERE scope='global'")
|
||||
await own.query("INSERT INTO server_probe_storage_usage VALUES($1,1,1,294912)", ["owner:"+actor.id])
|
||||
await own.query("SET LOCAL session_replication_role='origin'")
|
||||
await own.query("COMMIT")
|
||||
} finally { own.release(true) }
|
||||
const live = await invokeMcpTool(principal, "mm_server_probe_receipt_get", request)
|
||||
const storage = createProbeJournalStorage({ connect:connectProbeStorage, policy:{ ...PROBE_STORAGE_LIMITS,retentionDays:30 }, safetyWindowMs:60000,
|
||||
async authorize(ownerId,operation,scope) {
|
||||
assert.equal(ownerId,actor.id); assert.equal(operation,"purge"); assert.equal(scope,undefined)
|
||||
return "a".repeat(64)
|
||||
} })
|
||||
if (mode === "default-young" || mode === "default-old" || mode === "default-unproven" || mode === "default-spent") {
|
||||
await assert.rejects(retireExpiredServerProbeOperation(target))
|
||||
if (mode === "default-unproven") {
|
||||
// This original still qualifies for normal payload expiry, but cannot
|
||||
// be authenticated as a dependency-free retirement source.
|
||||
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1",
|
||||
[o.operation_id,await encryptSecret('["held"]')])
|
||||
}
|
||||
assert.deepEqual(await retainServerProbeStorage(), { skipped:false,owners:1,collected:1 })
|
||||
assert.equal((await pool.query("SELECT reserved_bytes::int AS bytes FROM server_probe_storage_usage WHERE scope='global'")).rows[0].bytes,0)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1",[o.operation_id])).rows.length,
|
||||
mode === "default-unproven" ? 0 : 1)
|
||||
await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request))
|
||||
await assert.rejects(captureServerProbeTerminalProof(target))
|
||||
await assert.rejects(retireCompletedServerProbeOperation(target))
|
||||
if (mode === "default-spent") {
|
||||
const spent = (await pool.query(`INSERT INTO server_probe_retirement_claims
|
||||
(receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token)
|
||||
SELECT receipt_id,operation_id,$2,$3,source_proof,$4 FROM server_probe_terminal_proofs WHERE operation_id=$1
|
||||
RETURNING claim_token`,[o.operation_id,o.plan_id,o.connection_id,randomUUID()])).rows[0]
|
||||
assert.ok(spent)
|
||||
await tickMcpWorkers()
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual((await pool.query("SELECT claim_token FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows,[spent])
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,1)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operation_replay_tombstones WHERE operation_id=$1",[o.operation_id])).rows.length,0)
|
||||
return
|
||||
}
|
||||
if (mode !== "default-old") {
|
||||
await tickMcpWorkers()
|
||||
await assert.rejects(retireExpiredServerProbeOperation(target))
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,1)
|
||||
return
|
||||
}
|
||||
await pool.query("UPDATE mcp_operations SET result=jsonb_set(result,'{confirmedWrites}','1') WHERE id=$1",[o.operation_id])
|
||||
await tickMcpWorkers()
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1",[a.receipt_id])).rows.length,0)
|
||||
await pool.query("UPDATE mcp_operations SET result=$2 WHERE id=$1",[o.operation_id,o.operation_result])
|
||||
await tickMcpWorkers()
|
||||
const absence = (await pool.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS op,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint,
|
||||
NOT EXISTS(SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1) AS proof,
|
||||
NOT EXISTS(SELECT 1 FROM server_probe_retired_sources WHERE operation_id=$1) AS archive`,[o.operation_id,o.plan_id])).rows[0]
|
||||
assert.deepEqual(absence,{op:true,plan:true,step:true,checkpoint:true,proof:true,archive:true})
|
||||
await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED")
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:o.operation_id}),error=>error instanceof McpError&&error.code==="NOT_FOUND")
|
||||
return
|
||||
}
|
||||
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1", [o.operation_id,await encryptSecret('["held"]')])
|
||||
await assert.rejects(retireCompletedServerProbeOperation(target))
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_retirement_claims WHERE receipt_id=$1", [a.receipt_id])).rows.length, 0)
|
||||
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$2 WHERE operation_id=$1", [o.operation_id,empty])
|
||||
if (mode === "scheduled-retained") {
|
||||
await tickMcpWorkers()
|
||||
await tickMcpWorkers()
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[o.operation_id])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id])).rows.length,1)
|
||||
assert.deepEqual(await invokeMcpTool(principal,"mm_server_probe_receipt_get",request),live)
|
||||
await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED")
|
||||
return
|
||||
}
|
||||
assert.deepEqual(await retireCompletedServerProbeOperation(target), { retired:true,automaticReplay:false })
|
||||
const proof = (await pool.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operations WHERE id=$1) AS op,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint,
|
||||
(SELECT receipt_ciphertext=$4 FROM server_probe_execution_payloads WHERE receipt_id=$3) AS payload,
|
||||
(SELECT reserved_bytes=294912 FROM server_probe_storage_usage WHERE scope='global') AS quota`,
|
||||
[o.operation_id,o.plan_id,a.receipt_id,raw.payload.receipt_ciphertext])).rows[0]
|
||||
assert.deepEqual(proof, { op:true,plan:true,step:true,checkpoint:true,payload:true,quota:true })
|
||||
assert.deepEqual(await invokeMcpTool(principal, "mm_server_probe_receipt_get", request), live)
|
||||
await assert.rejects(invokeMcpTool(foreign, "mm_server_probe_receipt_get", request))
|
||||
await assert.rejects(applyPlan(principal,o.plan_id,key), error => error instanceof McpError && error.code==="OPERATION_HISTORY_EXPIRED" && error.statusCode===410)
|
||||
await assert.rejects(applyPlan(principal,randomUUID(),key), error => error instanceof McpError && error.code==="STATE_CONFLICT" && error.statusCode===409)
|
||||
await assert.rejects(retireCompletedServerProbeOperation(target))
|
||||
const gc = await pool.connect()
|
||||
try {
|
||||
await gc.query("BEGIN")
|
||||
await gc.query("SET LOCAL mm.probe_storage_gc='on'")
|
||||
assert.equal((await gc.query("SELECT server_probe_storage_gc_allowed($1) AS allowed", [a.receipt_id])).rows[0].allowed, false)
|
||||
await gc.query("ROLLBACK")
|
||||
} finally { gc.release(true) }
|
||||
const protectedWrite = await pool.connect()
|
||||
try {
|
||||
await protectedWrite.query("BEGIN")
|
||||
const {rows:[archive]} = await protectedWrite.query("SELECT proof FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id])
|
||||
await protectedWrite.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true),
|
||||
set_config('mm.probe_retired_gc_proof',$2,true)`, [a.receipt_id,archive.proof])
|
||||
await assert.rejects(protectedWrite.query("DELETE FROM server_probe_retired_sources WHERE receipt_id=$1",[a.receipt_id]),/immutable/)
|
||||
await protectedWrite.query("ROLLBACK")
|
||||
} finally { protectedWrite.release(true) }
|
||||
const raced = await pool.connect()
|
||||
try {
|
||||
await raced.query("BEGIN")
|
||||
await raced.query("SELECT receipt_id FROM server_probe_retired_sources WHERE receipt_id=$1 FOR UPDATE",[a.receipt_id])
|
||||
await assert.rejects(collectExpiredRetiredServerProbe(target))
|
||||
await raced.query("ROLLBACK")
|
||||
} finally { raced.release(true) }
|
||||
assert.deepEqual(await invokeMcpTool(principal, "mm_server_probe_receipt_get", request),live)
|
||||
if (mode === "purged") {
|
||||
await pool.query("UPDATE mcp_settings SET requests_retention_days=365 WHERE id=1")
|
||||
await assert.rejects(collectExpiredRetiredServerProbe(target))
|
||||
assert.deepEqual(await invokeMcpTool(principal,"mm_server_probe_receipt_get",request),live)
|
||||
await pool.query("UPDATE mcp_settings SET requests_retention_days=30 WHERE id=1")
|
||||
assert.deepEqual(await storage.purge(actor.id,1,{ deadlineAt:Date.now()+10000 }),{ purged:1,automaticReplay:false })
|
||||
await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request))
|
||||
assert.equal((await pool.query("SELECT reserved_bytes::int AS bytes FROM server_probe_storage_usage WHERE scope='global'")).rows[0].bytes,32768)
|
||||
}
|
||||
if (mode === "retained") await tickMcpWorkers()
|
||||
else assert.deepEqual(await collectExpiredRetiredServerProbe(target), {collected:true,automaticReplay:false})
|
||||
const collected = (await pool.query(`SELECT
|
||||
(SELECT count(*)::int FROM server_probe_retired_sources WHERE receipt_id=$1) AS archive,
|
||||
(SELECT count(*)::int FROM server_probe_storage_admissions WHERE receipt_id=$1) AS admission,
|
||||
(SELECT count(*)::int FROM server_probe_executions WHERE receipt_id=$1) AS anchor,
|
||||
(SELECT count(*)::int FROM server_probe_execution_stages WHERE receipt_id=$1) AS stages,
|
||||
(SELECT count(*)::int FROM server_probe_execution_payloads WHERE receipt_id=$1) AS payload,
|
||||
(SELECT reserved_bytes::int FROM server_probe_storage_usage WHERE scope='global') AS quota,
|
||||
(SELECT count(*)::int FROM server_probe_storage_usage WHERE scope=$2) AS owner_counter`,[a.receipt_id,"owner:"+actor.id])).rows[0]
|
||||
assert.deepEqual(collected,{archive:0,admission:0,anchor:0,stages:0,payload:0,quota:0,owner_counter:0})
|
||||
await assert.rejects(collectExpiredRetiredServerProbe(target))
|
||||
await assert.rejects(invokeMcpTool(principal,"mm_server_probe_receipt_get",request))
|
||||
await assert.rejects(applyPlan(principal,o.plan_id,key),error=>error instanceof McpError&&error.code==="OPERATION_HISTORY_EXPIRED")
|
||||
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1", [principal.connectionId])
|
||||
await assert.rejects(invokeMcpTool(principal, "mm_server_probe_receipt_get", request))
|
||||
await assert.rejects(applyPlan(principal,o.plan_id,key), error => error instanceof McpError && error.code==="NOT_FOUND" && error.statusCode===404)
|
||||
}
|
||||
await verifyLifecycle("retained")
|
||||
await verifyLifecycle("purged")
|
||||
await verifyLifecycle("default-young")
|
||||
await verifyLifecycle("default-old")
|
||||
await verifyLifecycle("default-unproven")
|
||||
await verifyLifecycle("default-spent")
|
||||
await verifyLifecycle("scheduled-retained")
|
||||
console.log("Owned diagnostic retirement: real worker, retained/expired paths, spent claims and holds, public replay/read and exact GC refunds")
|
||||
} finally { await closePool() }
|
||||
@@ -0,0 +1,370 @@
|
||||
/** Exact internal maintenance entry. No scheduler, public registration, or native work. */
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { z } from "zod"
|
||||
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
|
||||
import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js"
|
||||
import { PROBE_PAYLOAD_MAX_BYTES, PROBE_PERMANENT_RESERVATION_BYTES, ProbeStorageRefused, ProbeStorageUnknown, type ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
||||
import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js"
|
||||
import { matchesServerProbeRetiredTerminal, openServerProbeRetiredSource, projectServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js"
|
||||
import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
||||
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
||||
|
||||
const targetSchema = z.object({ receiptId: z.string().uuid(), operationId: z.string().uuid(), connectionId: z.string().uuid() }).strict()
|
||||
type Target = z.infer<typeof targetSchema>
|
||||
function requireThat(value: unknown): asserts value { if (!value) throw new ProbeStorageRefused() }
|
||||
|
||||
const ORIGINAL = `jsonb_build_object(
|
||||
'operation_id',o.id,'connection_id',o.connection_id,'owner_id',c.owner_id,'plan_id',p.id,
|
||||
'operation_name',p.operation_id,'operation_version',p.operation_version,'operation_state',o.state,
|
||||
'operation_verification',o.verification,'operation_error_code',o.error_code,'operation_result',o.result,
|
||||
'plan_state',p.state,'approved_by',p.approved_by,'evidence',p.evidence,'step_index',s.step_index,
|
||||
'step_state',s.state,'intended_state',s.intended_state,'step_result',s.result,
|
||||
'operation_created_at',o.created_at,'operation_finished_at',o.finished_at,'plan_created_at',p.created_at,
|
||||
'plan_approved_at',p.approved_at,'plan_expires_at',p.expires_at,'step_created_at',s.created_at,'step_finished_at',s.finished_at
|
||||
) AS original,o.idempotency_key,k.before_sealed,k.after_sealed`
|
||||
const ORIGINAL_JOINS = `JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id
|
||||
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
||||
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=s.step_index`
|
||||
const ORIGINAL_ELIGIBLE = ` AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()
|
||||
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
||||
AND isfinite(p.expires_at)
|
||||
AND s.state='acknowledged' AND s.server_id IS NULL AND s.before_state IS NULL
|
||||
AND octet_length(k.before_sealed)<=1024 AND octet_length(k.after_sealed)<=1024
|
||||
AND octet_length(p.evidence::text)+octet_length(s.intended_state::text)+octet_length(s.result::text)+octet_length(o.result::text)<=2097152
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=o.id AND step_index<>0)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id AND step_index<>0)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
|
||||
`
|
||||
const SOURCE = `SELECT ${ORIGINAL},to_jsonb(a) AS admission,to_jsonb(j) AS anchor,to_jsonb(b) AS payload
|
||||
FROM public.server_probe_storage_admissions a
|
||||
JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id
|
||||
JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id
|
||||
JOIN public.mcp_operations o ON o.id=a.execution_id
|
||||
${ORIGINAL_JOINS}
|
||||
WHERE a.receipt_id=$1 AND o.id=$2 AND c.id=$3 AND c.owner_id=a.owner_id AND a.identity_kind='mcp'
|
||||
AND a.claim_token IS NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
|
||||
AND (($5::timestamptz IS NULL AND a.authority_expires_at<transaction_timestamp() AND p.expires_at<transaction_timestamp())
|
||||
OR o.finished_at=$5::timestamptz)
|
||||
AND (NOT $4::boolean OR o.finished_at<transaction_timestamp()-interval '180 days')
|
||||
${ORIGINAL_ELIGIBLE} FOR UPDATE OF a,j,b,o,p,s,k NOWAIT`
|
||||
const CLOSED_SOURCE = `SELECT ${ORIGINAL},f.source,f.source_proof,f.idempotency_digest,f.capture_proof
|
||||
FROM public.mcp_operations o ${ORIGINAL_JOINS}
|
||||
JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id AND f.receipt_id=$1
|
||||
WHERE o.id=$2 AND c.id=$3 AND o.finished_at<transaction_timestamp()-interval '180 days' AND p.expires_at<transaction_timestamp()
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_executions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_payloads WHERE receipt_id=$1)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_stages WHERE receipt_id=$1)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1 OR operation_id=o.id)
|
||||
${ORIGINAL_ELIGIBLE} FOR UPDATE OF f,o,p,s,k NOWAIT`
|
||||
|
||||
async function transaction<T>(deadlineAt: number, work: (sql: ProbeStorageSql) => Promise<T>): Promise<T> {
|
||||
const sql = await connectProbeStorage({ deadlineAt })
|
||||
let committing = false, committed = false
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
try {
|
||||
await sql.query("BEGIN")
|
||||
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
|
||||
// The same order as admission, publication, cancellation and payload GC.
|
||||
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)) AS locked")).rows[0]?.locked === true)
|
||||
// Bound non-SQL proof work as well. Any late continuation retains only
|
||||
// this permanently released checkout and cannot issue further SQL.
|
||||
const expired = new Promise<never>((_resolve,reject) => {
|
||||
timer = setTimeout(() => {
|
||||
sql.release(true)
|
||||
reject(new ProbeStorageRefused())
|
||||
}, Math.max(1,deadlineAt-Date.now()))
|
||||
})
|
||||
const result = await Promise.race([work(sql),expired])
|
||||
clearTimeout(timer)
|
||||
requireThat(Date.now()<deadlineAt)
|
||||
committing = true
|
||||
await sql.query("COMMIT")
|
||||
committed = true
|
||||
return result
|
||||
} catch (error) {
|
||||
if (committing) throw new ProbeStorageUnknown()
|
||||
try { await sql.query("ROLLBACK") } catch { /* Checkout is discarded. */ }
|
||||
throw error
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
sql.release(!committed)
|
||||
}
|
||||
}
|
||||
|
||||
async function lockOriginal(sql: ProbeStorageSql, { operationId, connectionId }: Target) {
|
||||
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT", [connectionId])).rows.length === 1)
|
||||
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + operationId])).rows[0]?.locked === true)
|
||||
}
|
||||
async function assertEmptyCheckpoints(row: Record<string, unknown>) {
|
||||
for (const sealed of [row.before_sealed, row.after_sealed]) {
|
||||
requireThat(typeof sealed === "string")
|
||||
let decoded: unknown
|
||||
try { decoded = JSON.parse(await decryptSecretExisting(sealed)) }
|
||||
catch { throw new ProbeStorageRefused() }
|
||||
requireThat(Array.isArray(decoded) && decoded.length === 0)
|
||||
}
|
||||
}
|
||||
async function prepare(sql: ProbeStorageSql, target: Target, requireOld = true, expectedFinishedAt?: string) {
|
||||
const { receiptId, operationId, connectionId } = target
|
||||
await lockOriginal(sql, target)
|
||||
const rows = (await sql.query(SOURCE, [receiptId, operationId, connectionId, requireOld, expectedFinishedAt ?? null])).rows
|
||||
requireThat(rows.length === 1)
|
||||
const { original, admission, anchor, payload, idempotency_key: key } = rows[0]
|
||||
requireThat(typeof key === "string")
|
||||
await assertEmptyCheckpoints(rows[0])
|
||||
const stages = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
|
||||
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows.map(row => row.stage)
|
||||
const signed = await projectServerProbeRetiredSource({ original, admission, anchor, payload, stages, quarantineCount: 0, cancellationCount: 0 })
|
||||
requireThat(signed.source.receiptId === receiptId && signed.source.executionId === operationId && signed.source.connectionId === connectionId)
|
||||
return { signed, key }
|
||||
}
|
||||
|
||||
const captureHash = z.string().regex(/^[a-f0-9]{64}$/)
|
||||
async function terminalCaptureProof(sourceProof: string, idempotencyDigest: string) {
|
||||
const sign = await privateFingerprinterExisting("server-probe-terminal-capture-v1")
|
||||
return sign(fingerprint({ sourceProof, idempotencyDigest }))
|
||||
}
|
||||
|
||||
function captureTerminalProof(target: Target, deadlineAt: number, expectedFinishedAt?: string) {
|
||||
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
|
||||
const { signed, key } = await prepare(sql, target, false, expectedFinishedAt)
|
||||
const digest = operationIdempotencyDigest(target.connectionId, key)
|
||||
const proof = await terminalCaptureProof(signed.proof, digest)
|
||||
const saved = (await sql.query(`INSERT INTO public.server_probe_terminal_proofs
|
||||
(operation_id,receipt_id,source,source_proof,idempotency_digest,capture_proof)
|
||||
VALUES($1,$2,$3::jsonb,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
|
||||
[target.operationId,target.receiptId,JSON.stringify(signed.source),signed.proof,digest,proof])).rows
|
||||
requireThat(saved.length === 1 && saved[0].receipt_id === target.receiptId)
|
||||
return { captured:true as const, automaticReplay:false as const }
|
||||
}))
|
||||
}
|
||||
/** Captures evidence only while the normal retention owner can still read the original. */
|
||||
export function captureServerProbeTerminalProof(raw: Target, deadlineAt = Date.now()+20000) {
|
||||
return captureTerminalProof(targetSchema.parse(raw), deadlineAt)
|
||||
}
|
||||
|
||||
/** Only the operation runner supplies the positively acknowledged terminal SQL timestamp. */
|
||||
export function capturePublishedServerProbeTerminalProof(raw: Target & { finishedAt: string }) {
|
||||
const { finishedAt, ...identity } = raw
|
||||
const stamp = z.string().regex(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/).parse(finishedAt)
|
||||
return captureTerminalProof(targetSchema.parse(identity), Date.now()+1200, stamp)
|
||||
}
|
||||
|
||||
async function prepareClosed(sql: ProbeStorageSql, target: Target) {
|
||||
const { receiptId, operationId, connectionId } = target
|
||||
await lockOriginal(sql, target)
|
||||
const rows = (await sql.query(CLOSED_SOURCE, [receiptId, operationId, connectionId])).rows
|
||||
requireThat(rows.length === 1)
|
||||
const row = rows[0], key = z.string().parse(row.idempotency_key)
|
||||
await assertEmptyCheckpoints(row)
|
||||
const proof = captureHash.parse(row.source_proof), digest = operationIdempotencyDigest(connectionId, key)
|
||||
requireThat(digest === row.idempotency_digest
|
||||
&& constantEqual(captureHash.parse(row.capture_proof), await terminalCaptureProof(proof, digest)))
|
||||
const source = await openServerProbeRetiredSource({ source:row.source, proof })
|
||||
requireThat(source.receiptId === receiptId && source.executionId === operationId && source.connectionId === connectionId
|
||||
&& matchesServerProbeRetiredTerminal(source, row.original))
|
||||
return { signed:{source,proof},key }
|
||||
}
|
||||
|
||||
function retireServerProbeOperation(raw: Target, kind: "retained" | "expired", deadlineAt = Date.now()+20000) {
|
||||
const target = targetSchema.parse(raw)
|
||||
const { receiptId, operationId, connectionId } = target
|
||||
return sharedDatabaseGate.request(async () => {
|
||||
const token = randomUUID()
|
||||
const read = kind === "retained" ? prepare : prepareClosed
|
||||
const claim = await transaction(deadlineAt, async sql => {
|
||||
const { signed, key } = await read(sql, target)
|
||||
const digest = operationIdempotencyDigest(connectionId, key)
|
||||
const result = await sql.query(`INSERT INTO public.server_probe_retirement_claims
|
||||
(receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token)
|
||||
VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
|
||||
[receiptId, operationId, signed.source.planId, connectionId, signed.proof, token])
|
||||
requireThat(result.rows.length === 1 && result.rows[0].receipt_id === receiptId)
|
||||
return { proof: signed.proof, planId: signed.source.planId, digest }
|
||||
})
|
||||
return transaction(deadlineAt, async sql => {
|
||||
const { signed, key } = await read(sql, target), { planId, proof, digest } = claim
|
||||
requireThat(signed.proof === proof && signed.source.planId === planId && operationIdempotencyDigest(connectionId, key) === digest)
|
||||
const consumed = await sql.query(`SELECT receipt_id FROM public.server_probe_retirement_claims
|
||||
WHERE receipt_id=$1 AND operation_id=$2 AND plan_id=$3 AND connection_id=$4 AND source_proof=$5
|
||||
AND claim_token=$6 AND claim_xid<>pg_current_xact_id()`, [receiptId, operationId, planId, connectionId, proof, token])
|
||||
requireThat(consumed.rows.length === 1 && consumed.rows[0].receipt_id === receiptId)
|
||||
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,'servers.probes.run',$5::bigint[],false) RETURNING operation_id`,
|
||||
[operationId, planId, connectionId, digest, signed.source.serverIds])
|
||||
requireThat(tombstone.rows.length === 1 && tombstone.rows[0].operation_id === operationId)
|
||||
await sql.query("SELECT set_config('mm.probe_retirement_claim',$1,true)", [token])
|
||||
if (kind === "retained") {
|
||||
const archive = await sql.query(`INSERT INTO public.server_probe_retired_sources(receipt_id,operation_id,source,proof)
|
||||
VALUES($1,$2,$3::jsonb,$4) RETURNING receipt_id`, [receiptId, operationId, JSON.stringify(signed.source), proof])
|
||||
requireThat(archive.rows.length === 1 && archive.rows[0].receipt_id === receiptId)
|
||||
}
|
||||
const released = await sql.query("DELETE FROM public.server_probe_terminal_proofs WHERE operation_id=$1 RETURNING receipt_id", [operationId])
|
||||
requireThat(released.rows.length <= 1 && released.rows.every(row => row.receipt_id === receiptId)
|
||||
&& (kind !== "expired" || released.rows.length === 1))
|
||||
const checkpoint = await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
|
||||
requireThat(checkpoint.rows.length === 1 && checkpoint.rows[0].operation_id === operationId)
|
||||
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
|
||||
requireThat(step.rows.length === 1 && step.rows[0].operation_id === operationId)
|
||||
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id", [operationId, connectionId, planId])
|
||||
requireThat(operation.rows.length === 1 && operation.rows[0].id === operationId)
|
||||
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id", [planId, connectionId])
|
||||
requireThat(plan.rows.length === 1 && plan.rows[0].id === planId)
|
||||
if (kind === "retained") {
|
||||
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
|
||||
{ ...target, ownerId: signed.source.ownerId, scopeServerIds: signed.source.serverIds })
|
||||
requireThat(observed && !observed.payloadPurged && fingerprint(observed.source) === fingerprint(signed.source))
|
||||
}
|
||||
return { retired: true as const, automaticReplay: false as const }
|
||||
})
|
||||
})
|
||||
}
|
||||
export function retireCompletedServerProbeOperation(raw: Target) {
|
||||
return retireServerProbeOperation(raw, "retained")
|
||||
}
|
||||
export function retireExpiredServerProbeOperation(raw: Target) {
|
||||
return retireServerProbeOperation(raw, "expired")
|
||||
}
|
||||
|
||||
/** Exact internal GC. This entry has no worker or public registration. */
|
||||
export function collectExpiredRetiredServerProbe(raw: Target, deadlineAt = Date.now()+20000) {
|
||||
const target = targetSchema.parse(raw)
|
||||
const { receiptId, operationId, connectionId } = target
|
||||
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
|
||||
const settings = (await sql.query("SELECT requests_retention_days FROM public.mcp_settings WHERE id=1 FOR SHARE NOWAIT")).rows
|
||||
const retentionDays = settings[0]?.requests_retention_days
|
||||
requireThat(settings.length === 1 && typeof retentionDays === "number" && Number.isInteger(retentionDays)
|
||||
&& retentionDays >= 1 && retentionDays <= 365)
|
||||
const pinned = (await sql.query(`SELECT c.owner_id,r.proof,t.target_server_ids::text[] AS server_ids FROM public.server_probe_retired_sources r
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
|
||||
JOIN public.mcp_connections c ON c.id=t.connection_id
|
||||
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.owner_id=c.owner_id
|
||||
JOIN public.server_probe_executions j ON j.receipt_id=r.receipt_id
|
||||
WHERE r.receipt_id=$1 AND r.operation_id=$2 AND c.id=$3 FOR UPDATE OF c,r,t,a,j NOWAIT`,
|
||||
[receiptId, operationId, connectionId])).rows
|
||||
const identity = z.object({ owner_id: z.string().min(1).max(200), proof: z.string().regex(/^[a-f0-9]{64}$/),
|
||||
server_ids: z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(z.number().int().positive().safe())).min(1).max(2) }).strict().parse(pinned[0])
|
||||
requireThat(pinned.length === 1)
|
||||
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
|
||||
{ ...target, ownerId: identity.owner_id, scopeServerIds: identity.server_ids })
|
||||
requireThat(observed)
|
||||
const stageRows = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
|
||||
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows
|
||||
requireThat(stageRows.length >= 1 && stageRows.length <= 11
|
||||
&& fingerprint(stageRows.map(row => row.stage)) === observed.source.stageSnapshotDigest)
|
||||
await sql.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true),
|
||||
set_config('mm.probe_retired_gc_proof',$2,true)`, [receiptId, identity.proof])
|
||||
requireThat((await sql.query("SELECT public.server_probe_storage_gc_allowed($1) AS allowed", [receiptId])).rows[0]?.allowed === true)
|
||||
const deletedPayload = (await sql.query("DELETE FROM public.server_probe_execution_payloads WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
||||
const files = observed.payloadPurged ? 0 : 1
|
||||
requireThat(deletedPayload.length === files && deletedPayload.every(row => row.receipt_id === receiptId))
|
||||
const deletedStages = (await sql.query("DELETE FROM public.server_probe_execution_stages WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
||||
requireThat(deletedStages.length === stageRows.length && deletedStages.every(row => row.receipt_id === receiptId))
|
||||
const deletedAnchor = (await sql.query("DELETE FROM public.server_probe_executions WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
||||
requireThat(deletedAnchor.length === 1 && deletedAnchor[0].receipt_id === receiptId)
|
||||
// Admission deletion cascades to the exact authenticated archive only.
|
||||
const deletedAdmission = (await sql.query("DELETE FROM public.server_probe_storage_admissions WHERE receipt_id=$1 AND owner_id=$2 RETURNING receipt_id",
|
||||
[receiptId, identity.owner_id])).rows
|
||||
requireThat(deletedAdmission.length === 1 && deletedAdmission[0].receipt_id === receiptId)
|
||||
const scopes = ["global", "owner:" + identity.owner_id], bytes = PROBE_PERMANENT_RESERVATION_BYTES + files * PROBE_PAYLOAD_MAX_BYTES
|
||||
const released = (await sql.query(`UPDATE public.server_probe_storage_usage SET admissions=admissions-1,
|
||||
reserved_files=reserved_files-$2,reserved_bytes=reserved_bytes-$3 WHERE scope=ANY($1::text[])
|
||||
AND admissions>=1 AND reserved_files>=$2 AND reserved_bytes>=$3 RETURNING scope`, [scopes, files, bytes])).rows
|
||||
requireThat(released.length === 2 && scopes.every(scope => released.some(row => row.scope === scope)))
|
||||
await sql.query("DELETE FROM public.server_probe_storage_usage WHERE scope=$1 AND admissions=0 AND reserved_files=0 AND reserved_bytes=0", [scopes[1]])
|
||||
const state = (await sql.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1) AS archive_gone,
|
||||
EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones WHERE operation_id=$2 AND connection_id=$3) AS replay_retained,
|
||||
EXISTS(SELECT 1 FROM public.server_probe_retirement_claims WHERE receipt_id=$1 AND operation_id=$2 AND connection_id=$3) AS claim_retained`,
|
||||
[receiptId, operationId, connectionId])).rows[0]
|
||||
requireThat(state?.archive_gone === true && state.replay_retained === true && state.claim_retained === true)
|
||||
return { collected: true as const, automaticReplay: false as const }
|
||||
}))
|
||||
}
|
||||
|
||||
const RETIRE_CANDIDATES = `SELECT COALESCE(a.receipt_id,f.receipt_id) AS receipt_id,o.id AS operation_id,o.connection_id,
|
||||
o.finished_at::text AS ordered_at,CASE WHEN a.receipt_id IS NULL THEN 'expired' ELSE 'retained' END AS kind
|
||||
FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id
|
||||
LEFT JOIN public.server_probe_storage_admissions a ON a.identity_kind='mcp' AND a.execution_id=o.id AND a.owner_id=c.owner_id
|
||||
LEFT JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id
|
||||
WHERE o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
||||
AND ((a.receipt_id IS NULL AND f.receipt_id IS NOT NULL)
|
||||
OR (a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
|
||||
AND EXISTS(SELECT 1 FROM public.server_probe_execution_payloads b WHERE b.receipt_id=a.receipt_id)))
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retirement_claims x WHERE x.operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
|
||||
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
|
||||
ORDER BY o.finished_at,o.id LIMIT 8`
|
||||
const GC_CANDIDATES = `SELECT r.receipt_id,r.operation_id,t.connection_id,a.gc_not_before::text AS ordered_at,'gc' AS kind
|
||||
FROM public.server_probe_retired_sources r
|
||||
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
|
||||
JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id
|
||||
WHERE isfinite(a.gc_not_before) AND a.gc_not_before<transaction_timestamp()
|
||||
AND t.action_id='servers.probes.run' AND NOT t.target_global
|
||||
AND ($1::timestamptz IS NULL OR (a.gc_not_before,r.operation_id)>($1::timestamptz,$2::uuid))
|
||||
ORDER BY a.gc_not_before,r.operation_id LIMIT 8`
|
||||
|
||||
type Cursor = { orderedAt: string; operationId: string }
|
||||
let retireCursor: Cursor | undefined, gcCursor: Cursor | undefined
|
||||
const workerRow = z.object({ receipt_id:z.string().uuid(),operation_id:z.string().uuid(),connection_id:z.string().uuid(),
|
||||
ordered_at:z.string().refine(value=>Number.isFinite(Date.parse(value))),kind:z.enum(["retained","expired","gc"]) }).strict()
|
||||
|
||||
/** Invoked only under the existing scheduler owner. Keep expiry and one-shot
|
||||
* retirement independent; a claimed operation is never selected again. */
|
||||
export function maintainServerProbeOperationRetirement() {
|
||||
return sharedDatabaseGate.request(async () => {
|
||||
const deadlineAt = Date.now()+20000
|
||||
const scan = async (lane: "retire" | "gc") => {
|
||||
const cursor = lane === "retire" ? retireCursor : gcCursor
|
||||
const setCursor = (value: Cursor | undefined) => { if (lane === "retire") retireCursor=value; else gcCursor=value }
|
||||
const rows = await transaction(deadlineAt,async sql => (await sql.query(lane === "retire" ? RETIRE_CANDIDATES : GC_CANDIDATES,
|
||||
[cursor?.orderedAt ?? null,cursor?.operationId ?? null])).rows)
|
||||
requireThat(rows.length<=8)
|
||||
let scanned=0, completed=0
|
||||
for (const raw of rows) {
|
||||
if (Date.now()>=deadlineAt-2000 || completed>=2) break
|
||||
const row=workerRow.parse(raw)
|
||||
requireThat((row.kind === "gc") === (lane === "gc"))
|
||||
const target={receiptId:row.receipt_id,operationId:row.operation_id,connectionId:row.connection_id}
|
||||
setCursor({orderedAt:row.ordered_at,operationId:row.operation_id}); scanned++
|
||||
try {
|
||||
if (row.kind === "gc") await collectExpiredRetiredServerProbe(target,deadlineAt)
|
||||
else await retireServerProbeOperation(target,row.kind,deadlineAt)
|
||||
completed++
|
||||
} catch (error) {
|
||||
if (!(error instanceof ProbeStorageRefused || error instanceof ServerProbeRetirementSourceRefused || error instanceof z.ZodError)) throw error
|
||||
}
|
||||
}
|
||||
if (scanned === rows.length && rows.length < 8) setCursor(undefined)
|
||||
return completed
|
||||
}
|
||||
const collected=await scan("gc")
|
||||
const retired=Date.now()<deadlineAt-2000 ? await scan("retire") : 0
|
||||
return {collected,retired,automaticReplay:false as const}
|
||||
})
|
||||
}
|
||||
@@ -5,14 +5,55 @@ import { PROBE_STORAGE_LIMITS } from "../modules/mcp/diagnostic-cancellation-sto
|
||||
import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
||||
import { createProbeJournalStorage, ProbeStorageRefused, type ProbeStorageConnection, type ProbeStorageOptions } from "./server-probe-journal-maintenance.js"
|
||||
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
||||
import { captureServerProbeTerminalProof } from "./server-probe-operation-retirement.js"
|
||||
|
||||
function createRetention(connect: typeof connectProbeStorage) {
|
||||
/** Independent from the expiry transaction. A busy, malformed, or uncertain
|
||||
* source cannot extend retention of its ciphertext or reuse a checkout. */
|
||||
async function captureBeforeExpiry() {
|
||||
const deadlineAt = Date.now() + 2200
|
||||
let candidates: Record<string, unknown>[]
|
||||
try {
|
||||
const sql = await connectProbeStorage({ deadlineAt: Math.min(deadlineAt, Date.now()+500) })
|
||||
let read = false
|
||||
try {
|
||||
candidates = (await sql.query(`/* probe-retention:capture-candidates */
|
||||
SELECT a.receipt_id,o.id AS operation_id,o.connection_id
|
||||
FROM public.server_probe_storage_admissions a
|
||||
JOIN public.server_probe_execution_payloads b USING(receipt_id)
|
||||
JOIN public.mcp_operations o ON o.id=a.execution_id
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_connections c ON c.id=o.connection_id AND c.owner_id=a.owner_id
|
||||
WHERE a.identity_kind='mcp' AND a.purged_at IS NULL AND a.cancelled_at IS NULL AND a.claim_token IS NULL
|
||||
AND a.claim_attempted_at IS NOT NULL AND a.authority_expires_at<clock_timestamp()
|
||||
AND a.gc_not_before<clock_timestamp()+interval '1 day'
|
||||
AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<clock_timestamp()
|
||||
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
||||
AND isfinite(p.expires_at) AND p.expires_at<clock_timestamp()
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_terminal_proofs f WHERE f.operation_id=o.id OR f.receipt_id=a.receipt_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources r WHERE r.operation_id=o.id OR r.receipt_id=a.receipt_id)
|
||||
ORDER BY a.gc_not_before,a.receipt_id LIMIT 8`, [], {timeoutMs:450})).rows
|
||||
read = true
|
||||
} finally { sql.release(!read) }
|
||||
} catch { return }
|
||||
for (const row of candidates) {
|
||||
if (Date.now() >= deadlineAt || candidates.length > 8) break
|
||||
if (typeof row.receipt_id !== "string" || typeof row.operation_id !== "string" || typeof row.connection_id !== "string") continue
|
||||
try {
|
||||
await captureServerProbeTerminalProof({ receiptId:row.receipt_id, operationId:row.operation_id, connectionId:row.connection_id },
|
||||
Math.min(deadlineAt, Date.now()+550))
|
||||
} catch { /* Never hold the payload or retry a capture in this window. */ }
|
||||
}
|
||||
}
|
||||
|
||||
function createRetention(connect: typeof connectProbeStorage, preCapture?: () => Promise<void>) {
|
||||
let running = false, cursor = ""
|
||||
return async () => {
|
||||
if (running) return { skipped: true, owners: 0, collected: 0 }
|
||||
running = true
|
||||
try {
|
||||
return await sharedDatabaseGate.request(async () => {
|
||||
await preCapture?.()
|
||||
const options: ProbeStorageOptions = { deadlineAt: Date.now() + 20000 }
|
||||
const physical = await connect(options)
|
||||
let active = true, broken = false, released = false, currentOwner = ""
|
||||
@@ -69,6 +110,6 @@ function createRetention(connect: typeof connectProbeStorage) {
|
||||
}
|
||||
}
|
||||
|
||||
export const retainServerProbeStorage = createRetention(connectProbeStorage)
|
||||
export const retainServerProbeStorage = createRetention(connectProbeStorage, captureBeforeExpiry)
|
||||
/** Trusted owned-test composition only; never an HTTP/MCP parameter. */
|
||||
export const createServerProbeRetentionForTests = createRetention
|
||||
@@ -0,0 +1,67 @@
|
||||
/** Fixed private source lookup. The action owns the current authority and
|
||||
* protection fence, and the existing history reader authenticates payload. */
|
||||
import { z } from "zod"
|
||||
import { fingerprint } from "../modules/mcp/security.js"
|
||||
import { openServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js"
|
||||
|
||||
type Query=(sql:string,values:unknown[])=>Promise<Record<string,unknown>[]>
|
||||
export interface ServerProbeRetiredLookup {
|
||||
receiptId:string
|
||||
operationId:string
|
||||
connectionId:string
|
||||
ownerId:string
|
||||
scopeServerIds:readonly number[]
|
||||
}
|
||||
|
||||
const SQL=`/* probe-history:retired-source */ SELECT r.source,r.proof,t.plan_id,t.target_server_ids::text[] AS tombstone_server_ids,
|
||||
j.source_server_id,j.destination_server_id,j.tool,j.mcp_credential_id,j.authority_issued_at,
|
||||
j.receipt_proof,a.binding_proof,a.retention_days,a.safety_ms,
|
||||
to_char(a.authority_expires_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS authority_expires_at,
|
||||
to_char(a.gc_not_before AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS gc_not_before,
|
||||
a.purged_at IS NOT NULL AS payload_purged,p.receipt_proof AS payload_proof,
|
||||
CASE WHEN p.receipt_ciphertext IS NOT NULL THEN pg_catalog.encode(pg_catalog.sha256(pg_catalog.convert_to(p.receipt_ciphertext,'UTF8')),'hex') END AS payload_sha256
|
||||
FROM public.server_probe_retired_sources r
|
||||
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id AND t.connection_id=$3
|
||||
JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=$4
|
||||
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id
|
||||
JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id AND j.execution_id=a.execution_id AND j.identity_kind=a.identity_kind
|
||||
LEFT JOIN public.server_probe_execution_payloads p ON p.receipt_id=a.receipt_id
|
||||
WHERE r.receipt_id=$1 AND r.operation_id=$2 AND a.execution_id=r.operation_id AND a.identity_kind='mcp'
|
||||
AND a.owner_id=c.owner_id AND a.claim_token IS NULL AND isfinite(a.claim_attempted_at) AND a.cancelled_at IS NULL
|
||||
AND (a.purged_at IS NULL OR isfinite(a.purged_at))
|
||||
AND j.actor_id=c.owner_id AND j.mcp_connection_id=c.id AND j.authentication='mcp' AND j.request_id IS NULL
|
||||
AND j.binding_proof=a.binding_proof AND j.authority_expires_at=a.authority_expires_at AND j.state='completed'
|
||||
AND t.action_id='servers.probes.run' AND NOT t.target_global AND cardinality(t.target_server_ids) BETWEEN 1 AND 2
|
||||
AND j.source_server_id=ANY($5::int[]) AND (j.destination_server_id IS NULL OR j.destination_server_id=ANY($5::int[]))
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE id=r.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_plans WHERE id=t.plan_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=r.operation_id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines q
|
||||
WHERE q.receipt_id=r.receipt_id OR(q.identity_kind='mcp' AND q.execution_id=r.operation_id))
|
||||
LIMIT 2`
|
||||
const sid=z.number().int().positive().safe()
|
||||
const ids=z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(sid)).min(1).max(2)
|
||||
|
||||
export async function findRetiredServerProbeHistory(query:Query,input:ServerProbeRetiredLookup) {
|
||||
const rows=await query(SQL,[input.receiptId,input.operationId,input.connectionId,input.ownerId,input.scopeServerIds])
|
||||
if(rows.length!==1)return
|
||||
const row=rows[0]
|
||||
const source=await openServerProbeRetiredSource({source:row.source,proof:row.proof}).catch(error=>{
|
||||
if(error instanceof ServerProbeRetirementSourceRefused)return
|
||||
throw error
|
||||
})
|
||||
const parsedScope=ids.safeParse(row.tombstone_server_ids)
|
||||
if(!source||!parsedScope.success)return
|
||||
const scope=parsedScope.data.sort((a,b)=>a-b)
|
||||
if(source.receiptId!==input.receiptId||source.executionId!==input.operationId||source.connectionId!==input.connectionId||
|
||||
source.ownerId!==input.ownerId||source.planId!==row.plan_id||fingerprint(source.serverIds)!==fingerprint(scope)||
|
||||
source.serverIds.some(id=>!input.scopeServerIds.includes(id))||source.sourceServerId!==row.source_server_id||
|
||||
(source.destinationServerId??null)!==row.destination_server_id||source.tool!==row.tool||
|
||||
source.originalCredentialId!==row.mcp_credential_id||String(source.issuedAt)!==String(row.authority_issued_at)||
|
||||
source.bindingProof!==row.binding_proof||source.receiptProof!==row.receipt_proof||source.retentionDays!==row.retention_days||
|
||||
source.safetyMs!==row.safety_ms||source.authorityExpiresAt!==row.authority_expires_at||source.gcNotBefore!==row.gc_not_before)return
|
||||
if(row.payload_purged===true){
|
||||
if(row.payload_proof!==null||row.payload_sha256!==null)return
|
||||
}else if(row.payload_purged!==false||row.payload_proof!==source.receiptProof||row.payload_sha256!==source.ciphertextSha256)return
|
||||
return {source,payloadPurged:row.payload_purged}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { encryptSecret,fingerprint,privateSecretFingerprint,redact } from "../modules/mcp/security.js"
|
||||
import { serverProbeReceiptMetadata } from "../modules/mcp/server-probe-actions.js"
|
||||
import type { ServerProbeReceipt } from "./server-probe-execution.js"
|
||||
import type { ServerProbeHistoryBinding } from "./server-probe-journal-history.js"
|
||||
export async function serverProbeRetiredFixture(twoSides=false, identity?:{connectionId:string;credentialId:string;ownerId:string},
|
||||
historical={base:Date.parse("2025-01-02T00:00:00Z"),retentionDays:365}){
|
||||
const operationId=randomUUID(),receiptId=randomUUID(),connectionId=identity?.connectionId??randomUUID(),credentialId=identity?.credentialId??randomUUID(),ownerId=identity?.ownerId??randomUUID()
|
||||
const {base,retentionDays}=historical,stamp=(ms:number)=>new Date(base+ms).toISOString()
|
||||
const originalTimes={issuedAt:base,authorityExpiresAt:base+15000}
|
||||
const tool=twoSides?"bandwidth":"ping",sourceServerId=9,destination=twoSides?{destinationServerId:11}:{}
|
||||
const prepared={receiptId,tool,serverId:sourceServerId,...destination,expiresAt:stamp(15000),bindingCiphertext:"inert-private-inner-binding"}
|
||||
const binding={version:1,operation:"servers.probes.run",principal:{connectionId,credentialId,credentialExpiresAt:stamp(30000)},
|
||||
ownerId,...originalTimes,prepared,bindingProof:"a".repeat(64),stageBudget:twoSides?1:2}
|
||||
const proof=(domain:string,value:unknown)=>privateSecretFingerprint(fingerprint(value),domain)
|
||||
const input={serverId:sourceServerId,receiptId,inputDigest:await proof("server-probe-mcp-binding-v1",binding),
|
||||
bindingCiphertext:await encryptSecret(JSON.stringify(binding))}
|
||||
const step={kind:"service",service:binding.operation,input}
|
||||
const serverIds=twoSides?[9,11]:[9]
|
||||
const evidence={steps:[step],requiresHuman:true,verificationScope:"acknowledgement",mutationCost:binding.stageBudget,
|
||||
target:{scope:"servers",serverIds},validUntil:prepared.expiresAt}
|
||||
const context:ServerProbeHistoryBinding["context"]={identity:{kind:"mcp",operationName:"servers.probes.run",ownerId,
|
||||
connectionId,credentialId,executionId:operationId,issuedAt:base,expiresAt:base+15000},sourceServerId,...destination,tool}
|
||||
const receipt:ServerProbeReceipt={receiptId,serverId:sourceServerId,...destination,tool,state:"completed",httpStatus:200,
|
||||
attemptedStages:1,completedStages:1,output:"INERT_RESULT_ONLY_IN_ORIGINAL_PAYLOAD",data:{kind:tool},automaticReplay:false,forwardingVerified:false}
|
||||
const metadata=serverProbeReceiptMetadata(receipt,operationId,binding.stageBudget)
|
||||
const plaintext=JSON.stringify({version:1,context,bindingProof:binding.bindingProof,receipt})
|
||||
const payload={receipt_id:receiptId,receipt_proof:await proof("server-probe-journal-v1:receipt",plaintext),receipt_ciphertext:await encryptSecret(plaintext)}
|
||||
const anchor={receipt_id:receiptId,binding_proof:binding.bindingProof,identity_kind:"mcp",execution_id:operationId,
|
||||
actor_id:ownerId,authentication:"mcp",mcp_connection_id:connectionId,mcp_credential_id:credentialId,request_id:null,
|
||||
authority_issued_at:String(base),authority_expires_at:stamp(15000),source_server_id:sourceServerId,
|
||||
destination_server_id:destination.destinationServerId??null,tool,state:"completed",receipt_proof:payload.receipt_proof,
|
||||
attempted_stages:1,completed_stages:1,created_at:stamp(500),finished_at:stamp(1000)}
|
||||
const raw={
|
||||
original:{operation_id:operationId,connection_id:connectionId,owner_id:ownerId,plan_id:randomUUID(),
|
||||
operation_name:binding.operation,operation_version:1,operation_state:"applied",operation_verification:"inconclusive",operation_error_code:null,
|
||||
operation_result:{confirmedWrites:0,verificationScope:"acknowledgement",completedServiceSteps:1,diagnosticReceipt:metadata},
|
||||
plan_state:"completed",approved_by:ownerId,evidence:{sealed:await encryptSecret(JSON.stringify(evidence))},
|
||||
step_index:0,step_state:"acknowledged",intended_state:redact(step),step_result:metadata,
|
||||
operation_created_at:stamp(100),operation_finished_at:stamp(1500),plan_created_at:stamp(10),plan_approved_at:stamp(50),
|
||||
plan_expires_at:stamp(16000),step_created_at:stamp(200),step_finished_at:stamp(1400)},
|
||||
admission:{receipt_id:receiptId,owner_id:ownerId,identity_kind:"mcp",execution_id:operationId,binding_proof:binding.bindingProof,
|
||||
authority_expires_at:stamp(15000),claim_token:null,claim_attempted_at:stamp(400),purged_at:null,cancelled_at:null,created_at:stamp(300),
|
||||
retention_days:retentionDays,safety_ms:60000,gc_not_before:stamp(15000+retentionDays*86400000+60000)},
|
||||
anchor,payload,stages:[{receipt_id:receiptId,stage_index:1,kind:twoSides?"native-bandwidth":"native-ping",timeout_ms:1000,
|
||||
state:"received",outcome:"received",created_at:stamp(600),received_at:stamp(900)}],quarantineCount:0,cancellationCount:0,
|
||||
}
|
||||
return {raw,context,receipt,serverIds,stamp}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { mkdtemp,rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import path from "node:path"
|
||||
import { test } from "node:test"
|
||||
import { serverProbeRetiredFixture as fixture } from "./server-probe-retired-source.fixture.js"
|
||||
import { createServerProbeJournalHistoryReader } from "./server-probe-journal-history.js"
|
||||
import { openServerProbeRetiredSource,projectServerProbeRetiredSource,ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js"
|
||||
|
||||
test("production proofs authenticate the minimal projection and keep the original payload readable only with the current fence",async()=>{
|
||||
const directory=await mkdtemp(path.join(tmpdir(),"owned-probe-retirement-"))
|
||||
const previous=process.env.MCP_STATE_DIR
|
||||
process.env.MCP_STATE_DIR=directory
|
||||
try{
|
||||
for(const twoSides of [false,true]){
|
||||
const {raw,context,receipt,serverIds}=await fixture(twoSides)
|
||||
const exact=structuredClone(raw),signed=await projectServerProbeRetiredSource(raw)
|
||||
assert.deepEqual(raw,exact)
|
||||
const opened=await openServerProbeRetiredSource(JSON.parse(JSON.stringify(signed)))
|
||||
assert.deepEqual(opened,signed.source)
|
||||
assert.deepEqual(opened.serverIds,serverIds)
|
||||
assert.equal(opened.authorityExpiresAt,"2025-01-02T00:00:15.000000Z")
|
||||
assert.equal(opened.terminalAt,"2025-01-02T00:00:16.000000Z")
|
||||
assert.equal(opened.activityAt,opened.terminalAt)
|
||||
assert.equal(opened.retentionDays,365)
|
||||
assert.equal(opened.gcNotBefore,"2026-01-02T00:01:15.000000Z")
|
||||
const lookup={operationId:opened.executionId,receiptId:opened.receiptId,connectionId:opened.connectionId,ownerId:opened.ownerId,scopeServerIds:serverIds}
|
||||
const stored={...signed,...raw.anchor,plan_id:raw.original.plan_id,tombstone_server_ids:serverIds.map(String),
|
||||
retention_days:raw.admission.retention_days,safety_ms:raw.admission.safety_ms,
|
||||
gc_not_before:raw.admission.gc_not_before.replace("Z","000Z"),authority_expires_at:raw.anchor.authority_expires_at.replace("Z","000Z"),
|
||||
payload_purged:false,payload_proof:raw.payload.receipt_proof,payload_sha256:opened.ciphertextSha256}
|
||||
const read=(row:Record<string,unknown>)=>findRetiredServerProbeHistory(async (_sql,values)=>{
|
||||
assert.deepEqual(values,[lookup.receiptId,lookup.operationId,lookup.connectionId,lookup.ownerId,lookup.scopeServerIds])
|
||||
return [row]
|
||||
},lookup)
|
||||
assert.deepEqual(await read(stored),{source:opened,payloadPurged:false})
|
||||
assert.deepEqual(await read({...stored,payload_purged:true,payload_proof:null,payload_sha256:null}),{source:opened,payloadPurged:true})
|
||||
for(const alteration of [{tombstone_server_ids:["800"]},{plan_id:randomUUID()},{gc_not_before:raw.admission.authority_expires_at},
|
||||
{mcp_credential_id:randomUUID()},{payload_sha256:"d".repeat(64)},{payload_purged:true}])
|
||||
assert.equal(await read({...stored,...alteration}),undefined)
|
||||
assert.equal(await read({...stored,proof:"d".repeat(64)}),undefined)
|
||||
const envelope=JSON.stringify(signed)
|
||||
for(const secret of [raw.payload.receipt_ciphertext,raw.original.evidence.sealed,receipt.output!,"inert-private-inner-binding"])
|
||||
assert(!envelope.includes(secret))
|
||||
let permitted=true
|
||||
const reader=createServerProbeJournalHistoryReader({
|
||||
assertReadAuthority:async()=>{if(!permitted)throw Error("current scope or protection denied")},
|
||||
loadArchive:async()=>[{...raw.anchor,receipt_ciphertext:raw.payload.receipt_ciphertext,quarantine_count:0}],
|
||||
})
|
||||
const binding={context,receiptId:opened.receiptId,bindingProof:opened.bindingProof}
|
||||
assert.deepEqual(await reader.read(binding),receipt)
|
||||
permitted=false
|
||||
await assert.rejects(reader.read(binding),/current scope or protection denied/)
|
||||
for(const alteration of [{retentionDays:364},{activityAt:"2025-01-02T00:00:17.000000Z"},{ownerId:randomUUID()}])
|
||||
await assert.rejects(openServerProbeRetiredSource({...signed,source:{...signed.source,...alteration}}),ServerProbeRetirementSourceRefused)
|
||||
}
|
||||
const {raw}=await fixture()
|
||||
for(const alter of [
|
||||
(copy:typeof raw)=>{copy.original.evidence.sealed="broken"},
|
||||
(copy:typeof raw)=>{copy.payload.receipt_proof="b".repeat(64)},
|
||||
(copy:typeof raw)=>{copy.original.step_result.sha256="b".repeat(64)},
|
||||
(copy:typeof raw)=>{copy.stages[0].received_at="2025-01-02T00:00:00.000001Z"},
|
||||
(copy:typeof raw)=>{copy.admission.authority_expires_at="2025-01-02T00:00:15.000001Z"},
|
||||
(copy:typeof raw)=>{copy.quarantineCount=1},
|
||||
(copy:typeof raw)=>{copy.cancellationCount=1},
|
||||
]){
|
||||
const copy=structuredClone(raw);alter(copy)
|
||||
await assert.rejects(projectServerProbeRetiredSource(copy),ServerProbeRetirementSourceRefused)
|
||||
}
|
||||
}finally{
|
||||
if(previous===undefined)delete process.env.MCP_STATE_DIR
|
||||
else process.env.MCP_STATE_DIR=previous
|
||||
await rm(directory,{recursive:true,force:true})
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,147 @@
|
||||
/** Private codec only. A future caller must own a bounded, locked SQL
|
||||
* snapshot, and readers must still prove current scope and protections. */
|
||||
import { createHash } from "node:crypto"
|
||||
import { z } from "zod"
|
||||
import { authenticateServerProbeExecutionSourceRow } from "../modules/mcp/diagnostic-cancellation-source.js"
|
||||
import { serverProbeReceiptMetadata } from "../modules/mcp/server-probe-actions.js"
|
||||
import { serviceDiagnosticReceipt } from "../modules/mcp/service-diagnostic-progress.js"
|
||||
import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js"
|
||||
import { createLegacyServerProbeVerifier } from "./server-probe-legacy-verifier.js"
|
||||
import { validateServerProbeJournalReceipt } from "./server-probe-journal.js"
|
||||
import type { ServerProbeReceipt } from "./server-probe-execution.js"
|
||||
import { canonicalPollArchiveTimestamp } from "./server-poll-terminal-archive.js"
|
||||
|
||||
const purpose="server-probe-retired-source-v1"
|
||||
const uuid=z.string().uuid(),hash=z.string().regex(/^[0-9a-f]{64}$/),sid=z.number().int().positive().safe()
|
||||
const time=z.string().transform(canonicalPollArchiveTimestamp)
|
||||
const tool=z.enum(["ping","traceroute","bandwidth","dns","route","mtu"])
|
||||
const record=z.record(z.string(),z.unknown())
|
||||
const originalSchema=z.object({
|
||||
operation_id:uuid,connection_id:uuid,owner_id:z.string().min(1).max(200),plan_id:uuid,
|
||||
operation_name:z.literal("servers.probes.run"),operation_version:z.literal(1),operation_state:z.literal("applied"),
|
||||
operation_verification:z.literal("inconclusive"),operation_error_code:z.null(),operation_result:z.unknown(),
|
||||
plan_state:z.literal("completed"),approved_by:z.string().min(1).max(200),evidence:record,
|
||||
step_index:z.literal(0),step_state:z.literal("acknowledged"),intended_state:record,step_result:record,
|
||||
operation_created_at:time,operation_finished_at:time,plan_created_at:time,plan_approved_at:time,
|
||||
plan_expires_at:time,step_created_at:time,step_finished_at:time,
|
||||
}).strict()
|
||||
const admissionSchema=z.object({
|
||||
receipt_id:uuid,owner_id:z.string().min(1).max(200),identity_kind:z.literal("mcp"),execution_id:uuid,binding_proof:hash,
|
||||
authority_expires_at:time,claim_token:z.null(),claim_attempted_at:time,purged_at:z.null(),cancelled_at:z.null(),created_at:time,
|
||||
retention_days:z.number().int().min(1).max(365),safety_ms:z.number().int().min(60000).max(86400000),gc_not_before:time,
|
||||
}).strict()
|
||||
const sourceSchema=z.object({
|
||||
original:originalSchema,admission:admissionSchema,anchor:record,
|
||||
payload:z.object({receipt_id:uuid,receipt_proof:hash,receipt_ciphertext:z.string().min(1).max(262144)}).strict(),
|
||||
stages:z.array(record).min(1).max(11),quarantineCount:z.literal(0),cancellationCount:z.literal(0),
|
||||
}).strict()
|
||||
const signedSchema=z.object({
|
||||
version:z.literal(1),receiptId:uuid,executionId:uuid,connectionId:uuid,ownerId:z.string().min(1).max(200),planId:uuid,
|
||||
operation:z.literal("servers.probes.run"),operationVersion:z.literal(1),originalCredentialId:uuid,
|
||||
sourceServerId:sid,destinationServerId:sid.optional(),tool,serverIds:z.array(sid).min(1).max(2),
|
||||
issuedAt:z.number().int().nonnegative().safe(),authorityExpiresAt:time,terminalAt:time,activityAt:time,gcNotBefore:time,
|
||||
retentionDays:z.number().int().min(1).max(365),safetyMs:z.number().int().min(60000).max(86400000),
|
||||
bindingProof:hash,sourceDigest:hash,terminalDigest:hash,stageSnapshotDigest:hash,receiptProof:hash,ciphertextSha256:hash,
|
||||
}).strict().refine(value=>{
|
||||
const expected=[...new Set([value.sourceServerId,...(value.destinationServerId===undefined?[]:[value.destinationServerId])])].sort((a,b)=>a-b)
|
||||
return fingerprint(expected)===fingerprint(value.serverIds)
|
||||
&& (value.tool==="bandwidth" ? value.destinationServerId!==undefined&&value.destinationServerId!==value.sourceServerId : value.destinationServerId===undefined)
|
||||
&& value.issuedAt<Date.parse(value.authorityExpiresAt) && value.terminalAt<=value.activityAt
|
||||
&& value.authorityExpiresAt<=value.activityAt
|
||||
&& micros(value.gcNotBefore)>=micros(value.authorityExpiresAt)+BigInt(value.retentionDays*86400000+value.safetyMs)*1000n
|
||||
})
|
||||
|
||||
export type ServerProbeRetiredSource = z.output<typeof signedSchema>
|
||||
/** Match the exact canonical live terminal projection after payload expiry. */
|
||||
export function matchesServerProbeRetiredTerminal(source: ServerProbeRetiredSource, raw: unknown): boolean {
|
||||
const parsed = originalSchema.safeParse(raw)
|
||||
return parsed.success && parsed.data.operation_id === source.executionId && parsed.data.plan_id === source.planId
|
||||
&& parsed.data.connection_id === source.connectionId && parsed.data.owner_id === source.ownerId
|
||||
&& fingerprint(parsed.data) === source.terminalDigest
|
||||
}
|
||||
export interface SignedServerProbeRetiredSource { source: ServerProbeRetiredSource; proof: string }
|
||||
export class ServerProbeRetirementSourceRefused extends Error {
|
||||
readonly automaticReplay=false
|
||||
constructor(){super("Diagnostic retirement source cannot be authenticated");this.name="ServerProbeRetirementSourceRefused"}
|
||||
}
|
||||
function requireThat(value:unknown):asserts value {if(!value)throw new ServerProbeRetirementSourceRefused()}
|
||||
function micros(value:string){return BigInt(Date.parse(value))*1000n+BigInt(value.slice(23,26))}
|
||||
const maximum=(...values:string[])=>values.sort().at(-1)!
|
||||
|
||||
/** Produces no operation/payload/cancellation mutation or public authorization.
|
||||
* Only settled successful sources with the original retained payload qualify. */
|
||||
export async function projectServerProbeRetiredSource(raw:unknown):Promise<SignedServerProbeRetiredSource>{
|
||||
try{
|
||||
requireThat(Buffer.byteLength(JSON.stringify(raw))<=3*1048576)
|
||||
const source=sourceSchema.parse(structuredClone(raw)),{original,admission,anchor,payload,stages}=source
|
||||
const authenticated=await authenticateServerProbeExecutionSourceRow(original,original.operation_id,
|
||||
{connectionId:original.connection_id,ownerId:original.owner_id})
|
||||
requireThat(authenticated.terminal&&authenticated.sourceState==="applied")
|
||||
const context=authenticated.originalContext,id=context.identity
|
||||
requireThat(admission.receipt_id===authenticated.scope.receiptId&&admission.execution_id===id.executionId&&
|
||||
admission.owner_id===id.ownerId&&admission.binding_proof===authenticated.scope.bindingProof&&
|
||||
micros(admission.authority_expires_at)===BigInt(id.expiresAt)*1000n&&time.parse(anchor.authority_expires_at)===admission.authority_expires_at&&
|
||||
payload.receipt_id===admission.receipt_id&&
|
||||
payload.receipt_proof===anchor.receipt_proof&&anchor.state==="completed")
|
||||
const verify=createLegacyServerProbeVerifier({loadMcpSource:async operationId=>{
|
||||
requireThat(operationId===id.executionId)
|
||||
return authenticated
|
||||
}})
|
||||
const verified=await verify({identityKind:"mcp",executionId:id.executionId,
|
||||
anchor:{...anchor,receipt_ciphertext:payload.receipt_ciphertext},stages})
|
||||
requireThat(verified.status==="verified"&&verified.authentication==="receipt"&&verified.bindingAuthenticated&&
|
||||
verified.progressAuthenticated&&verified.terminal&&!verified.quarantined&&!verified.cancelled)
|
||||
// The same ciphertext has already passed the existing receipt MAC and
|
||||
// original-context checks above. Match its actual public effect as well.
|
||||
const envelope=JSON.parse(await decryptSecretExisting(payload.receipt_ciphertext)) as {receipt:ServerProbeReceipt}
|
||||
const receipt=validateServerProbeJournalReceipt(context,admission.receipt_id,envelope.receipt)
|
||||
requireThat(receipt.state==="completed")
|
||||
const metadata=serverProbeReceiptMetadata(receipt,id.executionId,authenticated.stageBudget)
|
||||
const confirmed=serviceDiagnosticReceipt(metadata,true,id.executionId,authenticated.stageBudget)
|
||||
requireThat(confirmed&&fingerprint(original.step_result)===fingerprint(metadata)&&fingerprint(original.operation_result)===fingerprint({
|
||||
confirmedWrites:0,verificationScope:"acknowledgement",completedServiceSteps:1,diagnosticReceipt:confirmed,
|
||||
}))
|
||||
// The original receipt authenticates context and progress, not SQL row
|
||||
// timestamps. Sign the exact observed locked snapshot and its chronology,
|
||||
// without upgrading the historical journal's stage or timestamp proofs.
|
||||
const anchorCreated=time.parse(anchor.created_at),anchorFinished=time.parse(anchor.finished_at)
|
||||
requireThat(original.plan_created_at<=original.plan_approved_at&&original.plan_approved_at<=original.operation_created_at&&
|
||||
original.operation_created_at<original.plan_expires_at&&original.operation_created_at<=original.step_created_at&&
|
||||
original.step_created_at<=admission.created_at&&admission.created_at<=admission.claim_attempted_at&&
|
||||
admission.claim_attempted_at<=anchorCreated&&anchorCreated<=anchorFinished&&anchorFinished<admission.authority_expires_at&&
|
||||
anchorFinished<=original.step_finished_at&&original.step_finished_at<=original.operation_finished_at)
|
||||
let stageActivity=anchorCreated
|
||||
for(const stage of stages){
|
||||
const created=time.parse(stage.created_at),received=time.parse(stage.received_at)
|
||||
requireThat(stage.state==="received"&&stageActivity<=created&&created<=received&&received<=anchorFinished)
|
||||
stageActivity=received
|
||||
}
|
||||
const terminalAt=maximum(original.operation_finished_at,original.plan_expires_at)
|
||||
const result=signedSchema.parse({
|
||||
version:1,receiptId:admission.receipt_id,executionId:id.executionId,connectionId:id.connectionId,ownerId:id.ownerId,
|
||||
planId:original.plan_id,operation:"servers.probes.run",operationVersion:1,originalCredentialId:id.credentialId,
|
||||
sourceServerId:context.sourceServerId,...(context.destinationServerId===undefined?{}:{destinationServerId:context.destinationServerId}),
|
||||
tool:context.tool,serverIds:authenticated.serverIds,
|
||||
issuedAt:id.issuedAt,authorityExpiresAt:admission.authority_expires_at,terminalAt,
|
||||
activityAt:maximum(terminalAt,admission.authority_expires_at,admission.created_at,anchorFinished,stageActivity),
|
||||
gcNotBefore:admission.gc_not_before,retentionDays:admission.retention_days,safetyMs:admission.safety_ms,
|
||||
bindingProof:authenticated.scope.bindingProof,sourceDigest:authenticated.sourceDigest,terminalDigest:fingerprint(original),
|
||||
stageSnapshotDigest:fingerprint(stages),receiptProof:payload.receipt_proof,
|
||||
ciphertextSha256:createHash("sha256").update(payload.receipt_ciphertext).digest("hex"),
|
||||
})
|
||||
const sign=await privateFingerprinterExisting(purpose)
|
||||
return {source:result,proof:sign(fingerprint(result))}
|
||||
}catch{throw new ServerProbeRetirementSourceRefused()}
|
||||
}
|
||||
|
||||
/** Authenticates the private projection. It never grants current access or
|
||||
* authorizes the retained ciphertext without a fresh SQL identity read. */
|
||||
export async function openServerProbeRetiredSource(raw:unknown):Promise<ServerProbeRetiredSource>{
|
||||
try{
|
||||
requireThat(Buffer.byteLength(JSON.stringify(raw))<=8192)
|
||||
const {source,proof}=z.object({source:signedSchema,proof:hash}).strict().parse(structuredClone(raw))
|
||||
const sign=await privateFingerprinterExisting(purpose)
|
||||
requireThat(constantEqual(proof,sign(fingerprint(source))))
|
||||
return source
|
||||
}catch{throw new ServerProbeRetirementSourceRefused()}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { pool } from "../db/index.js"
|
||||
import { getServerRowById, type ServerRow } from "../modules/servers/repository/servers-repository.js"
|
||||
import { fingerprint, McpError, privateFingerprinter } from "../modules/mcp/security.js"
|
||||
import { readWireguardPeerDeletionState, wireguardPeerDeletionTarget, wireguardConfigurationRows, type WireguardPeerDeletionState } from "./wireguard-peer-delete.js"
|
||||
import { greDependencyPaths, greStableRows, assertGreDeleteDependencies } from "./gre-mutation-state.js"
|
||||
import { greStableRows, assertGreDeleteDependencies } from "./gre-mutation-state.js"
|
||||
import { observedBoolean } from "./ospf-observed-values.js"
|
||||
import { MikrotikClient } from "./mikrotik.js"
|
||||
import { withApplicationMutationLock, withConfigMutationLock } from "./config-mutation-lock.js"
|
||||
@@ -12,6 +12,8 @@ import { executeGuardedApplicationWrite } from "./config-mutation-policy.js"
|
||||
import { captureAndAppendRevision } from "./config-revisions.js"
|
||||
import { captureWireguardSnapshot } from "./wireguard-live.js"
|
||||
import { sameWireguardEndpoint } from "./wireguard-source.js"
|
||||
import { wireguardInterfaceDependencyPaths, wireguardInterfaceRuntimeDependencyPaths } from "./routeros-native-paths.js"
|
||||
export { wireguardInterfaceDependencyPaths } from "./routeros-native-paths.js"
|
||||
|
||||
type Row = Record<string, string>
|
||||
export interface WireguardInterfaceDeleteState {
|
||||
@@ -19,8 +21,7 @@ export interface WireguardInterfaceDeleteState {
|
||||
filters: { id: number; gatewayTunnelId: string; gateway: string }[]; proof: string;
|
||||
}
|
||||
export interface WireguardInterfaceDeleteCommand { method: "PATCH" | "DELETE"; path: string; resource: "interfaces" | "peers" | "ipv4" | "ipv6"; id: string; body?: Record<string, string> }
|
||||
const runtimePaths = new Set(["/routing/bgp/session", "/routing/ospf/neighbor", "/routing/ospf/interface", "/routing/bfd/session"])
|
||||
export const wireguardInterfaceDependencyPaths = [...new Set<string>([...greDependencyPaths, "/interface/vxlan", "/interface/vrrp", "/interface/bonding", "/interface/eoip", "/interface/gre", "/interface/ipip", "/ip/dhcp-server", "/ip/dhcp-client", "/ipv6/dhcp-server", "/ipv6/dhcp-client", "/ip/ipsec/peer", "/ip/ipsec/policy", ...runtimePaths])]
|
||||
const runtimePaths = new Set<string>(wireguardInterfaceRuntimeDependencyPaths)
|
||||
const fail = (code: string, message: string) => new McpError(code, message, ["STATE_UNAVAILABLE", "MUTATION_OUTCOME_UNKNOWN", "VERIFICATION_FAILED"].includes(code) ? 502 : 409)
|
||||
const automaticLinkLocal = (row: Row) => observedBoolean(row.dynamic) === true && /^fe80:/i.test(row.address ?? "")
|
||||
const ordered = (path: string) => ["/ip/vrf", "/routing/rule", "/routing/filter/rule", "/routing/bfd/configuration"].includes(path) || /^\/(?:ip|ipv6)\/firewall\//.test(path)
|
||||
|
||||
@@ -66,3 +66,54 @@ test("same-tuple pre-refresh packet still decodes with previous template and ref
|
||||
assert.throws(() => checkChrCapture(input), /reused the previous template/)
|
||||
assert.equal(checkChrCapture(fixture()).captureReplayMatched, true)
|
||||
})
|
||||
|
||||
function concatenateSets(version: number, first: string, second: string, reversed = false) {
|
||||
const header = version === 9 ? 20 : 16
|
||||
const left = Buffer.from(first, "hex"), right = Buffer.from(second, "hex")
|
||||
const values = [left.subarray(header), right.subarray(header)]
|
||||
if (reversed) values.reverse()
|
||||
const packet = Buffer.concat([left.subarray(0, header), ...values])
|
||||
packet.writeUInt16BE(version === 9 ? 2 : packet.length, 2)
|
||||
return packet
|
||||
}
|
||||
|
||||
function bundledFixture() {
|
||||
const input = fixture()
|
||||
input.frames = [0, 1, 4, 5].map((index, ordinal) => {
|
||||
const first = input.frames[index], second = input.frames[index + 2]
|
||||
return { ...first, ordinal: ordinal + 1, expected: second.expected,
|
||||
payloadHex: concatenateSets(input.exporters[first.exporter].version, first.payloadHex, second.payloadHex).toString("hex") }
|
||||
})
|
||||
return input
|
||||
}
|
||||
|
||||
test("v9 and IPFIX template plus data datagrams prove both phases with the current product parser", () => {
|
||||
const result = checkChrCapture(bundledFixture())
|
||||
assert.equal(result.frames, 4)
|
||||
assert.deepEqual(result.exporters.map(row => [row.baselineRecords, row.refreshedRecords]), [[1, 1], [1, 1]])
|
||||
const multipleData = fixture()
|
||||
const row = multipleData.frames[3]
|
||||
row.payloadHex = concatenateSets(10, row.payloadHex, row.payloadHex).toString("hex")
|
||||
row.expected = [...row.expected, ...row.expected]
|
||||
assert.equal(checkChrCapture(multipleData).exporters[1].baselineRecords, 2)
|
||||
})
|
||||
|
||||
test("rejects stale data before a refresh, foreign sets, trailing bytes and v9 counts that hide a set", () => {
|
||||
const original = fixture()
|
||||
const reversed = bundledFixture()
|
||||
reversed.frames[2].payloadHex = concatenateSets(9, original.frames[4].payloadHex, original.frames[6].payloadHex, true).toString("hex")
|
||||
assert.throws(() => checkChrCapture(reversed), /template order/)
|
||||
const foreign = bundledFixture()
|
||||
const foreignPacket = Buffer.from(foreign.frames[1].payloadHex, "hex")
|
||||
foreignPacket.writeUInt16BE(257, 16 + foreignPacket.readUInt16BE(18))
|
||||
foreign.frames[1].payloadHex = foreignPacket.toString("hex")
|
||||
assert.throws(() => checkChrCapture(foreign), /data set ID/)
|
||||
const trailing = bundledFixture()
|
||||
trailing.frames[0].payloadHex += "00"
|
||||
assert.throws(() => checkChrCapture(trailing), /set count or header/)
|
||||
const hidden = bundledFixture()
|
||||
const hiddenPacket = Buffer.from(hidden.frames[0].payloadHex, "hex")
|
||||
hiddenPacket.writeUInt16BE(1, 2)
|
||||
hidden.frames[0].payloadHex = hiddenPacket.toString("hex")
|
||||
assert.throws(() => checkChrCapture(hidden), /v9 set coverage/)
|
||||
})
|
||||
@@ -6,7 +6,7 @@ import { resolve } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { isDeepStrictEqual } from "node:util"
|
||||
import { z } from "zod"
|
||||
import { clearFlowTemplateCache, FlowTemplateRefused, parseFlowPacket } from "../services/traffic-flow-parse.js"
|
||||
import { clearFlowTemplateCache, FLOW_TEMPLATE_LIMITS, FlowTemplateRefused, parseFlowPacket } from "../services/traffic-flow-parse.js"
|
||||
|
||||
const maxInput = 512 * 1024
|
||||
const ip = z.string().max(45).refine(value => isIP(value) !== 0)
|
||||
@@ -28,20 +28,65 @@ const capture = z.object({
|
||||
payloadHex: z.string().min(40).max(8192).regex(/^(?:[0-9a-f]{2})+$/),
|
||||
kind: z.enum(["template", "data", "refresh", "pre-refresh-probe"]),
|
||||
expected: z.array(flow).max(64),
|
||||
}).strict()).min(8).max(64),
|
||||
}).strict()).min(4).max(64),
|
||||
}).strict()
|
||||
|
||||
function requireEvidence(condition: unknown, reason: string): asserts condition {
|
||||
if (!condition) throw Error(reason)
|
||||
}
|
||||
|
||||
type Exporter = z.infer<typeof exporter>
|
||||
const zeroPadding = (bytes: Buffer) => bytes.length <= 3 && bytes.every(value => value === 0)
|
||||
|
||||
/** Check the entire datagram envelope without decoding record values. */
|
||||
function inspectSets(packet: Buffer, source: Exporter, changesTemplate: boolean, previousWidth: number) {
|
||||
const header = source.version === 9 ? 20 : 16
|
||||
requireEvidence(packet.length >= header + 4 && packet.readUInt16BE(0) === source.version &&
|
||||
packet.readUInt32BE(header - 4) === source.domain &&
|
||||
(source.version === 9 || packet.readUInt16BE(2) === packet.length), "Packet does not match the declared exporter")
|
||||
let width = previousWidth, templateSeen = false, records = 0, sets = 0
|
||||
for (let offset = header; offset < packet.length;) {
|
||||
requireEvidence(packet.length - offset >= 4 && ++sets <= 32, "Invalid set count or header")
|
||||
const id = packet.readUInt16BE(offset), length = packet.readUInt16BE(offset + 2)
|
||||
requireEvidence(length >= 4 && offset + length <= packet.length, "Invalid set length")
|
||||
const body = packet.subarray(offset + 4, offset + length)
|
||||
offset += length
|
||||
if (id === (source.version === 9 ? 0 : 2)) {
|
||||
requireEvidence(changesTemplate && !templateSeen && records === 0 && body.length >= 4 &&
|
||||
body.readUInt16BE(0) === source.templateId, "Unexpected template identity or order")
|
||||
const fieldCount = body.readUInt16BE(2)
|
||||
requireEvidence(fieldCount > 0 && fieldCount <= FLOW_TEMPLATE_LIMITS.perTemplateFields, "Invalid template field count")
|
||||
let at = 4
|
||||
width = 0
|
||||
for (let field = 0; field < fieldCount; field++) {
|
||||
requireEvidence(body.length - at >= 4, "Truncated template field")
|
||||
const type = body.readUInt16BE(at), size = body.readUInt16BE(at + 2)
|
||||
requireEvidence(size > 0 && width + size <= 4096, "Variable or oversized replay template")
|
||||
width += size
|
||||
at += source.version === 10 && (type & 0x8000) !== 0 ? 8 : 4
|
||||
requireEvidence(at <= body.length, "Truncated enterprise template field")
|
||||
}
|
||||
requireEvidence(zeroPadding(body.subarray(at)), "Unexpected additional template or padding")
|
||||
templateSeen = true
|
||||
} else {
|
||||
requireEvidence(id === source.templateId && (!changesTemplate || templateSeen), "Unexpected data set ID or template order")
|
||||
requireEvidence(width > 0 && body.length >= width && zeroPadding(body.subarray(Math.floor(body.length / width) * width)), "Invalid data set records or padding")
|
||||
records += Math.floor(body.length / width)
|
||||
requireEvidence(records <= 64, "Replay record budget exceeded")
|
||||
}
|
||||
}
|
||||
requireEvidence(templateSeen === changesTemplate && (source.version !== 9 || packet.readUInt16BE(2) >= sets),
|
||||
"Template transition or v9 set coverage differs")
|
||||
return { width, records }
|
||||
}
|
||||
|
||||
/** Synchronous, isolated replay. Call in an offline process with no live parser. */
|
||||
export function checkChrCapture(input: unknown) {
|
||||
const parsed = capture.safeParse(input)
|
||||
requireEvidence(parsed.success, "Invalid capture projection")
|
||||
const { destination, exporters, frames } = parsed.data
|
||||
requireEvidence(exporters[0].address !== exporters[1].address && exporters.every(row => row.address !== destination), "Expected two distinct exporter addresses and one destination")
|
||||
const state = exporters.map(() => ({ phase: 0, baselineRecords: 0, refreshedRecords: 0, missingTemplateObserved: false }))
|
||||
const state = exporters.map(() => ({ phase: 0, width: 0, baselineRecords: 0, refreshedRecords: 0, missingTemplateObserved: false }))
|
||||
let lastOrdinal = -1, lastReceivedMs = -1
|
||||
clearFlowTemplateCache()
|
||||
try {
|
||||
@@ -51,19 +96,13 @@ export function checkChrCapture(input: unknown) {
|
||||
lastReceivedMs = frame.receivedMs
|
||||
const source = exporters[frame.exporter], current = state[frame.exporter]
|
||||
const packet = Buffer.from(frame.payloadHex, "hex")
|
||||
const header = source.version === 9 ? 20 : 16
|
||||
requireEvidence(packet.length >= header + 8 && packet.readUInt16BE(0) === source.version &&
|
||||
packet.readUInt32BE(header - 4) === source.domain &&
|
||||
(source.version === 9 || packet.readUInt16BE(2) === packet.length) &&
|
||||
packet.readUInt16BE(header + 2) === packet.length - header, "Packet does not match the declared exporter or single-set envelope")
|
||||
const isTemplate = frame.kind === "template" || frame.kind === "refresh"
|
||||
requireEvidence(packet.readUInt16BE(header) === (isTemplate ? source.version === 9 ? 0 : 2 : source.templateId), "Unexpected set ID")
|
||||
if (isTemplate) {
|
||||
requireEvidence(packet.readUInt16BE(header + 4) === source.templateId && frame.expected.length === 0, "Unexpected template ID or template records")
|
||||
requireEvidence(frame.kind === "template" ? current.phase === 0 : current.phase === 1 && current.baselineRecords > 0, "Unexpected template transition")
|
||||
} else {
|
||||
requireEvidence(current.phase > 0, "Data preceding baseline template")
|
||||
}
|
||||
const envelope = inspectSets(packet, source, isTemplate, current.width)
|
||||
const transport = { sourcePort: source.port, collectorSocketId: "offline_capture_socket" }
|
||||
if (frame.kind === "pre-refresh-probe") {
|
||||
requireEvidence(current.phase === 1 && current.baselineRecords > 0 && !current.missingTemplateObserved && frame.expected.length === 0, "Unexpected pre-refresh probe")
|
||||
@@ -77,10 +116,13 @@ export function checkChrCapture(input: unknown) {
|
||||
continue
|
||||
}
|
||||
const actual = parseFlowPacket(packet, source.address, transport).map(({ src, dst, bytes, packets, inIface, outIface }) => ({ src, dst, bytes, packets, inIface, outIface }))
|
||||
requireEvidence(isDeepStrictEqual(actual, frame.expected), "Decoded flow records differ from the expected capture projection")
|
||||
if (isTemplate) current.phase++
|
||||
else {
|
||||
requireEvidence(actual.length > 0 && actual.every(row => row.bytes > 0 && row.packets > 0), "Expected positive traffic after the template")
|
||||
requireEvidence(actual.length === envelope.records && isDeepStrictEqual(actual, frame.expected), "Decoded flow records differ from the expected capture projection")
|
||||
if (isTemplate) {
|
||||
current.phase++
|
||||
current.width = envelope.width
|
||||
}
|
||||
if (actual.length) {
|
||||
requireEvidence(actual.every(row => row.bytes > 0 && row.packets > 0), "Expected positive traffic after the template")
|
||||
if (current.phase === 1) current.baselineRecords += actual.length
|
||||
else current.refreshedRecords += actual.length
|
||||
}
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { EventEmitter } from "node:events"
|
||||
import { PassThrough } from "node:stream"
|
||||
import { test } from "node:test"
|
||||
import { captureOwnedChrPacketsForTests } from "./traffic-flow-chr-capture-process.js"
|
||||
|
||||
const scope = { interfaceName: "owned0", sources: ["192.0.2.1", "192.0.2.2"] as const, destination: "192.0.2.100", port: 4739 }
|
||||
function fixturePcap(linkType: "RAW" | "EN10MB" = "RAW") {
|
||||
const link = linkType === "EN10MB" ? 14 : 0, frame = 60 + link
|
||||
const bytes = Buffer.alloc(24 + 2 * frame)
|
||||
bytes.writeUInt32LE(0xa1b2c3d4)
|
||||
bytes.writeUInt16LE(2, 4); bytes.writeUInt16LE(4, 6)
|
||||
bytes.writeUInt32LE(4124 + link, 16); bytes.writeUInt32LE(link ? 1 : 101, 20)
|
||||
for (const [index, at] of [24, 24 + frame].entries()) {
|
||||
bytes.writeUInt32LE(1700000000, at); bytes.writeUInt32LE(44 + link, at + 8); bytes.writeUInt32LE(44 + link, at + 12)
|
||||
if (link) bytes.writeUInt16BE(0x0800, at + 16 + 12)
|
||||
const ip = bytes.subarray(at + 16 + link, at + frame)
|
||||
ip[0] = 0x45; ip.writeUInt16BE(44, 2); ip[9] = 17
|
||||
Buffer.from([192, 0, 2, index + 1, 192, 0, 2, 100]).copy(ip, 12)
|
||||
ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(scope.port, 22); ip.writeUInt16BE(24, 24)
|
||||
ip.writeUInt16BE(10, 28); ip.writeUInt16BE(16, 30)
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
const ready = "tcpdump: listening on owned0, link-type RAW (Raw IP), snapshot length 4124 bytes\n"
|
||||
type Link = "RAW" | "EN10MB"
|
||||
function fixture({ mode = "normal", drop = 0, linkType = "RAW", reportedLink = linkType, fileLink = linkType }:
|
||||
{ mode?: "normal" | "early" | "wrong-interface"; drop?: number; linkType?: Link; reportedLink?: Link; fileLink?: Link } = {}) {
|
||||
const child = Object.assign(new EventEmitter(), { stdout: new PassThrough(), stderr: new PassThrough(),
|
||||
kill(signal: "SIGINT" | "SIGKILL") {
|
||||
events.push(signal)
|
||||
if (signal === "SIGINT") {
|
||||
child.stdout.end(fixturePcap(fileLink))
|
||||
child.stderr.end("2 packets captured\n2 packets received by filter\n" + drop + " packets dropped by kernel\n")
|
||||
}
|
||||
queueMicrotask(() => child.emit("close", signal === "SIGINT" ? 0 : null, signal === "SIGINT" ? null : signal))
|
||||
return true
|
||||
},
|
||||
})
|
||||
const events: string[] = []
|
||||
const launch = (args: string[]) => {
|
||||
events.push("launch")
|
||||
assert.deepEqual(args, ["-n", "-p", "-U", "-i", "owned0", "-y", linkType, "-s", linkType === "RAW" ? "4124" : "4138", "-c", "64", "-w", "-",
|
||||
"ip and udp and dst host 192.0.2.100 and dst port 4739 and (src host 192.0.2.1 or src host 192.0.2.2)"])
|
||||
queueMicrotask(() => {
|
||||
if (mode === "early") child.emit("close", 0, null)
|
||||
else {
|
||||
const line = reportedLink === "RAW" ? ready : ready.replace("RAW (Raw IP)", "EN10MB (Ethernet)").replace("4124", "4138")
|
||||
child.stderr.write(mode === "normal" ? line : line.replace("owned0", "foreign0"))
|
||||
}
|
||||
})
|
||||
return child
|
||||
}
|
||||
return { child, launch, events }
|
||||
}
|
||||
|
||||
test("confined capture waits for exact readiness, then checks complete acknowledged two-source pcap", async () => {
|
||||
const f = fixture()
|
||||
const result = await captureOwnedChrPacketsForTests(scope, {
|
||||
deadlineAt: Date.now() + 1000, assertCurrent: async () => { f.events.push("current") },
|
||||
exercise: async signal => { assert.equal(signal.aborted, false); f.events.push("exercise") },
|
||||
}, f.launch)
|
||||
assert.deepEqual(f.events, ["current", "launch", "current", "exercise", "current", "SIGINT"])
|
||||
assert.deepEqual(result.counts, { captured: 2, received: 2, dropped: 0 })
|
||||
assert.deepEqual(result.bytes, fixturePcap())
|
||||
})
|
||||
|
||||
test("wrong-interface readiness and early exit never dispatch exercise", async () => {
|
||||
for (const mode of ["wrong-interface", "early"] as const) {
|
||||
const f = fixture({ mode })
|
||||
await assert.rejects(captureOwnedChrPacketsForTests(scope, {
|
||||
deadlineAt: Date.now() + 100, assertCurrent: async () => {},
|
||||
exercise: async () => { f.events.push("exercise") },
|
||||
}, f.launch))
|
||||
assert.deepEqual(f.events, mode === "early" ? ["launch"] : ["launch", "SIGKILL"])
|
||||
}
|
||||
})
|
||||
|
||||
test("capture drops refuse evidence and scope revocation prevents exercise", async () => {
|
||||
const dropped = fixture({ drop: 1 })
|
||||
await assert.rejects(captureOwnedChrPacketsForTests(scope, {
|
||||
deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {},
|
||||
}, dropped.launch), /completeness/)
|
||||
const revoked = fixture()
|
||||
let checks = 0
|
||||
await assert.rejects(captureOwnedChrPacketsForTests(scope, {
|
||||
deadlineAt: Date.now() + 1000, assertCurrent: async () => { if (++checks === 2) throw Error("revoked") },
|
||||
exercise: async () => { revoked.events.push("exercise") },
|
||||
}, revoked.launch), /revoked/)
|
||||
assert.deepEqual(revoked.events, ["launch", "SIGKILL"])
|
||||
})
|
||||
|
||||
test("EN10MB launch, readiness, and file must agree independently", async () => {
|
||||
const ethScope = { ...scope, linkType: "EN10MB" } as const
|
||||
const f = fixture({ linkType: "EN10MB" })
|
||||
const result = await captureOwnedChrPacketsForTests(ethScope, {
|
||||
deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {},
|
||||
}, f.launch)
|
||||
assert.deepEqual(result.bytes, fixturePcap("EN10MB"))
|
||||
|
||||
const wrongReady = fixture({ linkType: "EN10MB", reportedLink: "RAW" })
|
||||
await assert.rejects(captureOwnedChrPacketsForTests(ethScope, {
|
||||
deadlineAt: Date.now() + 100, assertCurrent: async () => {}, exercise: async () => { wrongReady.events.push("exercise") },
|
||||
}, wrongReady.launch))
|
||||
assert.deepEqual(wrongReady.events, ["launch", "SIGKILL"])
|
||||
|
||||
const wrongFile = fixture({ linkType: "EN10MB", fileLink: "RAW" })
|
||||
await assert.rejects(captureOwnedChrPacketsForTests(ethScope, {
|
||||
deadlineAt: Date.now() + 1000, assertCurrent: async () => {}, exercise: async () => {},
|
||||
}, wrongFile.launch), /pinned link type EN10MB/)
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
/** Owns a single confined tcpdump child. Used only by an admitted native runner. */
|
||||
import { spawn } from "node:child_process"
|
||||
import type { Readable } from "node:stream"
|
||||
import { isIPv4 } from "node:net"
|
||||
import { chrCaptureLayout, projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
import { prepareChrReplay } from "./traffic-flow-chr-offline-replay.js"
|
||||
|
||||
type Scope = Parameters<typeof projectChrWire>[1] & { interfaceName: string }
|
||||
type Child = {
|
||||
stdout: Pick<Readable, "on">
|
||||
stderr: Pick<Readable, "on">
|
||||
once(event: "error", listener: (error: Error) => void): unknown
|
||||
once(event: "close", listener: (code: number | null, signal: NodeJS.Signals | null) => void): unknown
|
||||
kill(signal: "SIGINT" | "SIGKILL"): boolean
|
||||
}
|
||||
type Launch = (args: string[]) => Child
|
||||
type Options = {
|
||||
deadlineAt: number
|
||||
assertCurrent: () => Promise<void>
|
||||
exercise: (signal: AbortSignal) => Promise<void>
|
||||
}
|
||||
const maxCapture = 512 * 1024
|
||||
const maxDiagnostics = 8192
|
||||
const packetCap = 64
|
||||
function must(value: unknown, reason: string): asserts value {
|
||||
if (!value) throw Error(reason)
|
||||
}
|
||||
function argsFor(scope: Scope, layout: ReturnType<typeof chrCaptureLayout>) {
|
||||
must(/^[A-Za-z0-9][A-Za-z0-9_.-]{0,14}$/.test(scope.interfaceName) && scope.interfaceName !== "any" && scope.sources.length === 2 &&
|
||||
[...scope.sources, scope.destination].every(isIPv4) && new Set([...scope.sources, scope.destination]).size === 3 &&
|
||||
Number.isInteger(scope.port) && scope.port >= 1 && scope.port <= 65535, "Invalid confined capture scope")
|
||||
const [first, second] = scope.sources
|
||||
return ["-n", "-p", "-U", "-i", scope.interfaceName, "-y", layout.linkType, "-s", String(layout.snapLength), "-c", String(packetCap), "-w", "-",
|
||||
"ip and udp and dst host " + scope.destination + " and dst port " + scope.port + " and (src host " + first + " or src host " + second + ")"]
|
||||
}
|
||||
function counters(stderr: string) {
|
||||
const exact = (label: string) => {
|
||||
const matches = stderr.split(/\r?\n/).map(line => new RegExp("^(\\d+) packets? " + label + "$").exec(line)).filter(match => match !== null)
|
||||
must(matches.length === 1, "Capture counters are missing or ambiguous")
|
||||
const value = Number(matches[0][1])
|
||||
must(Number.isSafeInteger(value) && value >= 0, "Capture counter exceeds integer range")
|
||||
return value
|
||||
}
|
||||
return { captured: exact("captured"), received: exact("received by filter"), dropped: exact("dropped by kernel") }
|
||||
}
|
||||
|
||||
async function run(scope: Scope, options: Options, launch: Launch) {
|
||||
const pinned = structuredClone(scope), layout = chrCaptureLayout(pinned.linkType)
|
||||
const args = argsFor(pinned, layout), deadline = options.deadlineAt
|
||||
must(Number.isSafeInteger(deadline) && deadline > Date.now() && deadline - Date.now() <= 90000, "Invalid capture deadline")
|
||||
const controller = new AbortController()
|
||||
const live = () => must(!controller.signal.aborted && Date.now() < deadline, "Capture deadline or ownership ended")
|
||||
await options.assertCurrent()
|
||||
live()
|
||||
const child = launch(args), chunks: Buffer[] = []
|
||||
let bytes = 0, diagnosticBytes = 0, stderr = "", requestedStop = false, closed = false, failed = false
|
||||
let ready!: () => void, terminate!: () => void, rejectWork!: (error: Error) => void
|
||||
const readiness = new Promise<void>(resolve => { ready = resolve })
|
||||
const workFailed = new Promise<never>((_resolve, reject) => { rejectWork = reject })
|
||||
// The promise can reject during cleanup after an earlier error was observed.
|
||||
void workFailed.catch(() => {})
|
||||
const stopped = new Promise<void>(resolve => { terminate = resolve })
|
||||
const fail = (reason: string) => {
|
||||
if (failed) return
|
||||
failed = true
|
||||
controller.abort()
|
||||
rejectWork(Error(reason))
|
||||
}
|
||||
child.stdout.on("data", (part: Buffer) => {
|
||||
bytes += part.length
|
||||
if (bytes > maxCapture) { fail("Capture exceeded byte limit"); return }
|
||||
chunks.push(Buffer.from(part))
|
||||
})
|
||||
child.stderr.on("data", (part: Buffer) => {
|
||||
diagnosticBytes += part.length
|
||||
if (diagnosticBytes > maxDiagnostics) { fail("Capture diagnostics exceeded byte limit"); return }
|
||||
stderr += part.toString("utf8")
|
||||
const expected = "tcpdump: listening on " + pinned.interfaceName + ", link-type " + layout.linkType
|
||||
if (stderr.split(/\r?\n/).some(line => line.startsWith(expected + " (") && line.endsWith("snapshot length " + layout.snapLength + " bytes"))) ready()
|
||||
})
|
||||
child.once("error", () => fail("Capture child failed to start or signal"))
|
||||
child.once("close", (code, signal) => {
|
||||
closed = true
|
||||
if (!requestedStop || code !== 0 || signal !== null) fail("Capture closed outside its acknowledged stop")
|
||||
terminate()
|
||||
})
|
||||
const timer = setTimeout(() => fail("Capture deadline expired"), Math.max(1, deadline - Date.now()))
|
||||
let cleanupTimer: ReturnType<typeof setTimeout> | undefined
|
||||
try {
|
||||
await Promise.race([readiness, workFailed])
|
||||
live()
|
||||
await Promise.race([options.assertCurrent(), workFailed])
|
||||
live()
|
||||
await Promise.race([options.exercise(controller.signal), workFailed])
|
||||
live()
|
||||
await Promise.race([options.assertCurrent(), workFailed])
|
||||
live()
|
||||
requestedStop = true
|
||||
must(child.kill("SIGINT"), "Capture stop signal was not delivered")
|
||||
await Promise.race([stopped, workFailed])
|
||||
live()
|
||||
const counts = counters(stderr), capture = Buffer.concat(chunks, bytes)
|
||||
const wire = projectChrWire(capture, { sources: pinned.sources, destination: pinned.destination, port: pinned.port, ...(pinned.linkType === undefined ? {} : { linkType: pinned.linkType }) })
|
||||
must(counts.dropped === 0 && counts.captured === counts.received && counts.captured === wire.frames.length && counts.captured < packetCap,
|
||||
"Capture completeness was not established")
|
||||
return { bytes: capture, sha256: wire.captureSha256, dropped: counts.dropped, complete: true as const, counts, wire }
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
controller.abort()
|
||||
if (!closed) {
|
||||
child.kill("SIGKILL")
|
||||
const exited = await Promise.race([stopped.then(() => true), new Promise<false>(resolve => { cleanupTimer = setTimeout(() => resolve(false), 2000) })])
|
||||
if (cleanupTimer) clearTimeout(cleanupTimer)
|
||||
must(exited, "Capture child exit could not be confirmed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const captureOwnedChrPackets = (scope: Scope, options: Options) => {
|
||||
must(process.platform === "linux", "Confined tcpdump capture requires Linux")
|
||||
return run(scope, options, args =>
|
||||
spawn("tcpdump", args, { shell: false, stdio: ["ignore", "pipe", "pipe"], env: { ...process.env, LC_ALL: "C" } }))
|
||||
}
|
||||
|
||||
/** Adapter for the capture callback of executeOwnedChrCapture. */
|
||||
export function createConfinedChrCapture(scope: Scope, options: Omit<Options, "exercise">) {
|
||||
const pinned = structuredClone(scope)
|
||||
return async (exercise: (signal?: AbortSignal) => Promise<void>) => {
|
||||
const capture = await captureOwnedChrPackets(pinned, { ...options, exercise })
|
||||
return { ...capture, ...prepareChrReplay(capture.wire) }
|
||||
}
|
||||
}
|
||||
|
||||
/** Explicit inert transport seam. */
|
||||
export const captureOwnedChrPacketsForTests = run
|
||||
@@ -0,0 +1,65 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { assessChrCaptureTiming, planChrMeasuredCaptureWindow } from "./traffic-flow-chr-capture-timing.js"
|
||||
import { CHR_TOPOLOGY_CATALOG_PAIR_GETS, CHR_TOPOLOGY_CATALOG_READ_LIMIT } from "./traffic-flow-chr-topology-inventory.js"
|
||||
import { CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS } from "./traffic-flow-chr-target-harness.js"
|
||||
|
||||
const current = { pairGets: CHR_TOPOLOGY_CATALOG_PAIR_GETS, catalogueReadLimit: CHR_TOPOLOGY_CATALOG_READ_LIMIT,
|
||||
captureMs: 90000, targetMinimumRemainingMs: CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS }
|
||||
|
||||
test("reduced exact catalogue count fits the existing cap but does not authorize writes without measurement", () => {
|
||||
assert.equal(current.pairGets, 86)
|
||||
const result = assessChrCaptureTiming(current)
|
||||
assert.equal(result.status, "measurement-required")
|
||||
assert.equal(result.minimumNodeCatalogueGets, 9460)
|
||||
const max = assessChrCaptureTiming({ ...current, forwardWrites: 6 })
|
||||
assert.equal(max.minimumNodeCatalogueGets, 10492)
|
||||
assert.equal(max.pairsBeforeGenerator, 22)
|
||||
assert.equal(result.pairsBeforeFirstTargetWindow, 2)
|
||||
assert.equal(result.setupBudgetMs, 26000)
|
||||
assert.equal(assessChrCaptureTiming({ ...current, catalogueReadLimit: 8000 }).status, "catalogue-read-limit")
|
||||
})
|
||||
|
||||
const measured = {
|
||||
childPairMs: 12023, parentPairMs: 12023, directSetupMs: 4000,
|
||||
collectorBeforeCaptureMs: 45000, collectorAfterCaptureMs: 55000,
|
||||
beforeCaptureMs: 1400000, afterCaptureMs: 1500000, counterSpanMs: 120000, remainingAuthorityMs: 3600000,
|
||||
}
|
||||
|
||||
test("proposes a finite measured capture window and reserves authority through inverse", () => {
|
||||
assert.deepEqual(planChrMeasuredCaptureWindow(measured), {
|
||||
status: "candidate-window", captureMs: 205000, requiredAtCaptureStartMs: 1810000, requiredBeforeFirstWriteMs: 3315000,
|
||||
requiredCollectorMs: 515000,
|
||||
})
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, remainingAuthorityMs: 3315000 }).status, "authority-window-exceeded")
|
||||
assert.deepEqual(planChrMeasuredCaptureWindow({ ...measured, childPairMs: 100, parentPairMs: 100 }), {
|
||||
status: "candidate-window", captureMs: 90000, requiredAtCaptureStartMs: 1695000, requiredBeforeFirstWriteMs: 3200000,
|
||||
requiredCollectorMs: 400000,
|
||||
})
|
||||
})
|
||||
|
||||
test("refuses missing or excessive measurements and preserves the counter evidence horizon", () => {
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, parentPairMs: undefined }).status, "measurement-required")
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, directSetupMs: undefined }).status, "measurement-required")
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, childPairMs: 30000 }).status, "capture-window-exceeded")
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, counterSpanMs: 180001 }).status, "capture-window-exceeded")
|
||||
assert.throws(() => planChrMeasuredCaptureWindow({ ...measured, childPairMs: Infinity }), /Invalid CHR measured capture inputs/)
|
||||
assert.throws(() => planChrMeasuredCaptureWindow({ ...measured, remainingAuthorityMs: 3600001 }), /Invalid CHR measured capture inputs/)
|
||||
})
|
||||
|
||||
test("a long fixture lease still requires measured work within the short MCP credential", () => {
|
||||
assert.equal(planChrMeasuredCaptureWindow({ ...measured, collectorAfterCaptureMs: undefined }).status, "collector-measurement-required")
|
||||
assert.deepEqual(planChrMeasuredCaptureWindow({ ...measured, collectorAfterCaptureMs: 140000 }), {
|
||||
status: "collector-window-exceeded", requiredCollectorMs: 600000,
|
||||
})
|
||||
})
|
||||
|
||||
test("a larger catalogue budget still requires real representative measurement and refuses observed latency", () => {
|
||||
const options = { ...current, catalogueReadLimit: 20000 }
|
||||
assert.equal(assessChrCaptureTiming(options).status, "measurement-required")
|
||||
// Minimum recorded GET on CHR A/B times 43 each. This is a projection,
|
||||
// not an assertion about the next live relay transaction.
|
||||
assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 43 * (108 + 263) }).status, "capture-setup-window")
|
||||
assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 100 }).status, "parent-measurement-required")
|
||||
assert.equal(assessChrCaptureTiming({ ...options, pairedReadMs: 100, parentPairedReadMs: 100 }).status, "budget-check-passed")
|
||||
})
|
||||
@@ -0,0 +1,88 @@
|
||||
import { CHR_COLLECTOR_RESULT_POLL_MS, CHR_STAGED_COLLECTOR_APPROVAL_MS, CHR_STAGED_TARGET_CAPTURE_MAX_MS,
|
||||
CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS } from "./traffic-flow-chr-target-timing-profile.js"
|
||||
|
||||
/** This model is a lower bound for the current serial read schedule. It can
|
||||
* refuse admission. Its success never substitutes for live scope or authority. */
|
||||
export function assessChrCaptureTiming(options: {
|
||||
pairGets: number
|
||||
catalogueReadLimit: number
|
||||
captureMs: number
|
||||
targetMinimumRemainingMs: number
|
||||
forwardWrites?: number
|
||||
pairedReadMs?: number
|
||||
parentPairedReadMs?: number
|
||||
}) {
|
||||
const { pairGets, catalogueReadLimit, captureMs, targetMinimumRemainingMs, forwardWrites = 4, pairedReadMs, parentPairedReadMs } = options
|
||||
if (![pairGets, catalogueReadLimit, captureMs, targetMinimumRemainingMs].every(value => Number.isSafeInteger(value) && value > 0) ||
|
||||
![4, 5, 6].includes(forwardWrites) || [pairedReadMs, parentPairedReadMs].some(value =>
|
||||
value !== undefined && (!Number.isFinite(value) || value <= 0))) {
|
||||
throw Error("Invalid CHR timing inputs")
|
||||
}
|
||||
// Each flow setting/target write takes three fresh reads and a fresh
|
||||
// topology proof after each. Eight outer pairs include the timing sample.
|
||||
const pairsBeforeFirstTargetWindow = 2
|
||||
const pairsBeforeGenerator = pairsBeforeFirstTargetWindow + 3 * forwardWrites + 2
|
||||
const topologyAcquireAndRestorePairs = 2 * (1 + 2 * 18)
|
||||
const minimumNodePairs = topologyAcquireAndRestorePairs + pairsBeforeGenerator + 3 * forwardWrites + 8
|
||||
const minimumNodeCatalogueGets = minimumNodePairs * pairGets
|
||||
const minimumParentPairsBeforeGenerator = 2 * forwardWrites
|
||||
const minimumParentCatalogueGetsBeforeGenerator = minimumParentPairsBeforeGenerator * pairGets
|
||||
const setupBudgetMs = captureMs - targetMinimumRemainingMs
|
||||
// A measured pair omits parent identity/catalogue extras. Use it only to
|
||||
// veto an already underbudget capture, with margin for local scheduling.
|
||||
const projectedSetupMs = pairedReadMs === undefined ? undefined :
|
||||
Math.ceil((pairsBeforeGenerator * pairedReadMs + minimumParentPairsBeforeGenerator * (parentPairedReadMs ?? 0)) * 1.25)
|
||||
const status = minimumNodeCatalogueGets > catalogueReadLimit ? "catalogue-read-limit" :
|
||||
setupBudgetMs <= 0 || (projectedSetupMs !== undefined && projectedSetupMs >= setupBudgetMs) ? "capture-setup-window" :
|
||||
pairedReadMs === undefined ? "measurement-required" :
|
||||
parentPairedReadMs === undefined ? "parent-measurement-required" : "budget-check-passed"
|
||||
return { status, minimumNodeCatalogueGets, minimumParentCatalogueGetsBeforeGenerator,
|
||||
pairsBeforeFirstTargetWindow, pairsBeforeGenerator, setupBudgetMs, projectedSetupMs } as const
|
||||
}
|
||||
|
||||
/** Offline proposal for the staged disabled-baseline path. Before/after phase
|
||||
* budgets cover the complete reviewed schedule outside capture, with one
|
||||
* separate bounded human approval in each phase. */
|
||||
export function planChrMeasuredCaptureWindow(options: {
|
||||
childPairMs?: number
|
||||
parentPairMs?: number
|
||||
directSetupMs?: number
|
||||
collectorBeforeCaptureMs?: number
|
||||
collectorAfterCaptureMs?: number
|
||||
beforeCaptureMs: number
|
||||
afterCaptureMs: number
|
||||
counterSpanMs: number
|
||||
remainingAuthorityMs: number
|
||||
}) {
|
||||
const { childPairMs, parentPairMs, directSetupMs, collectorBeforeCaptureMs, collectorAfterCaptureMs, beforeCaptureMs, afterCaptureMs,
|
||||
counterSpanMs, remainingAuthorityMs } = options
|
||||
const valid = (value: number) => Number.isSafeInteger(value) && value > 0 && value <= 3600000
|
||||
if (![beforeCaptureMs, afterCaptureMs, counterSpanMs, remainingAuthorityMs].every(valid) ||
|
||||
[childPairMs, parentPairMs, directSetupMs, collectorBeforeCaptureMs, collectorAfterCaptureMs]
|
||||
.some(value => value !== undefined && !valid(value))) {
|
||||
throw Error("Invalid CHR measured capture inputs")
|
||||
}
|
||||
if (childPairMs === undefined || parentPairMs === undefined || directSetupMs === undefined) {
|
||||
return { status: "measurement-required" } as const
|
||||
}
|
||||
// Two enables, each with full pre-dispatch and post-ACK snapshots on both
|
||||
// independent paths, plus the child's complete pre-generation fence.
|
||||
const setupMs = (5 * childPairMs + 4 * parentPairMs + directSetupMs) * 1.25
|
||||
const captureMs = Math.max(90000,
|
||||
(Math.floor((CHR_TARGET_MINIMUM_CAPTURE_REMAINING_MS + setupMs) / 1000) + 1) * 1000)
|
||||
if (captureMs > CHR_STAGED_TARGET_CAPTURE_MAX_MS || counterSpanMs > 180000) {
|
||||
return { status: "capture-window-exceeded", captureMs } as const
|
||||
}
|
||||
const operationMs = CHR_STAGED_COLLECTOR_APPROVAL_MS + CHR_COLLECTOR_RESULT_POLL_MS
|
||||
if (collectorBeforeCaptureMs === undefined || collectorAfterCaptureMs === undefined) {
|
||||
return { status: "collector-measurement-required" } as const
|
||||
}
|
||||
const requiredCollectorMs = captureMs + 2 * operationMs + collectorBeforeCaptureMs + collectorAfterCaptureMs
|
||||
if (requiredCollectorMs >= 600000) return { status: "collector-window-exceeded", requiredCollectorMs } as const
|
||||
const requiredAtCaptureStartMs = captureMs + afterCaptureMs + operationMs
|
||||
const requiredBeforeFirstWriteMs = beforeCaptureMs + operationMs + requiredAtCaptureStartMs
|
||||
if (requiredBeforeFirstWriteMs >= remainingAuthorityMs) {
|
||||
return { status: "authority-window-exceeded", requiredBeforeFirstWriteMs } as const
|
||||
}
|
||||
return { status: "candidate-window", captureMs, requiredAtCaptureStartMs, requiredBeforeFirstWriteMs, requiredCollectorMs } as const
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js"
|
||||
|
||||
function fixture(lose = false, idless = false) {
|
||||
const baseline = {
|
||||
deviceId: "fixture", flow: { ...(idless ? {} : { ".id": "*F" }), enabled: "false", interfaces: "all", untouched: "1m" },
|
||||
ipfix: { ...(idless ? {} : { ".id": "*A" }), bytes: "false", packets: "true", "src-address": "true", "dst-address": "true",
|
||||
protocol: "true", "src-port": "true", "dst-port": "true", "in-interface": "true", "out-interface": "true" },
|
||||
targets: [] as Record<string, unknown>[],
|
||||
}
|
||||
const state = structuredClone(baseline)
|
||||
const writes: Record<string, string>[] = []
|
||||
const journal: string[] = []
|
||||
const dispatches: string[] = []
|
||||
async function change<T>(method: string, path: string, body: Record<string, string | string[]>,
|
||||
fence?: () => Promise<unknown>): Promise<T> {
|
||||
await fence?.()
|
||||
dispatches.push(method + " " + path)
|
||||
assert.ok(Object.values(body).every(value => typeof value === "string"))
|
||||
writes.push(body as Record<string, string>)
|
||||
const row = path.includes("/ipfix/") ? state.ipfix : state.flow
|
||||
Object.assign(row, Object.fromEntries(Object.entries(body).map(([key, value]) =>
|
||||
[key, key === "interfaces" ? value : value === "yes" ? "true" : "false"])))
|
||||
if (lose) throw Error("lost ACK")
|
||||
return undefined as T
|
||||
}
|
||||
const harness = createDisabledChrFlowHarness({
|
||||
baseline, transitInterface: "transit-test", read: async () => state, authorize: async () => {},
|
||||
checkpoint: async entry => { journal.push(entry.step + "." + entry.phase) },
|
||||
native: {
|
||||
patch: (path, body, _ms, fence) => change("PATCH", path, body, fence),
|
||||
post: (path, body, _ms, _signal, fence) => change("POST", path, body, fence),
|
||||
},
|
||||
})
|
||||
return { baseline, state, writes, journal, harness, dispatches }
|
||||
}
|
||||
|
||||
test("changes only required fields and one transit interface, then restores the exact disabled inventory", async () => {
|
||||
const { baseline, state, writes, journal, harness } = fixture()
|
||||
assert.equal((await harness.enable()).flow.enabled, "true")
|
||||
assert.deepEqual(await harness.restore(), { restored: true, automaticReplay: false })
|
||||
assert.deepEqual(state, baseline)
|
||||
assert.deepEqual(writes, [{ bytes: "yes" }, { enabled: "yes", interfaces: "transit-test" },
|
||||
{ enabled: "no", interfaces: "all" }, { bytes: "no" }])
|
||||
assert.deepEqual(journal.filter(value => value.endsWith(".verified")),
|
||||
["ipfix.enable.verified", "flow.enable.verified", "flow.restore.verified", "ipfix.restore.verified"])
|
||||
})
|
||||
|
||||
test("missing singleton IDs use only fixed POST /set and require ACK before inverse", async () => {
|
||||
const { baseline, state, dispatches, harness } = fixture(false, true)
|
||||
await harness.enable()
|
||||
await harness.restore()
|
||||
assert.deepEqual(state, baseline)
|
||||
assert.deepEqual(dispatches, ["POST /ip/traffic-flow/ipfix/set", "POST /ip/traffic-flow/set",
|
||||
"POST /ip/traffic-flow/set", "POST /ip/traffic-flow/ipfix/set"])
|
||||
const lost = fixture(true, true)
|
||||
await assert.rejects(lost.harness.enable(), /requires reconciliation/)
|
||||
await assert.rejects(lost.harness.restore(), /No verified/)
|
||||
assert.deepEqual(lost.dispatches, ["POST /ip/traffic-flow/ipfix/set"])
|
||||
})
|
||||
|
||||
test("refuses restore while a target remains and refuses replay after unknown acknowledgement", async () => {
|
||||
const known = fixture()
|
||||
await known.harness.enable()
|
||||
known.state.targets.push({ ".id": "*1" })
|
||||
await assert.rejects(known.harness.restore(), /baseline changed/)
|
||||
assert.equal(known.writes.length, 2)
|
||||
const unknown = fixture(true)
|
||||
await assert.rejects(unknown.harness.enable(), /requires reconciliation/)
|
||||
await assert.rejects(unknown.harness.enable(), /already attempted/)
|
||||
await assert.rejects(unknown.harness.restore(), /No verified/)
|
||||
assert.equal(unknown.writes.length, 1)
|
||||
})
|
||||
@@ -0,0 +1,116 @@
|
||||
import { runChrWrite, type ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js"
|
||||
import { isDeepStrictEqual } from "node:util"
|
||||
import type { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { encodeRosId } from "../services/routeros-native-paths.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
type Inventory = { deviceId: string; flow: Row; ipfix: Row; targets: Row[] }
|
||||
type Step = "ipfix.enable" | "flow.enable" | "flow.restore" | "ipfix.restore"
|
||||
type Options = {
|
||||
baseline: Inventory
|
||||
relayWrite?: { side: "A" | "B"; admission: ChrWriteAdmission }
|
||||
transitInterface: string
|
||||
read: () => Promise<Inventory>
|
||||
authorize: () => Promise<void>
|
||||
verifyContext?: () => Promise<void>
|
||||
native: Pick<MikrotikClient, "patch" | "post">
|
||||
checkpoint: (entry: { step: Step; phase: "intent" | "acknowledged" | "verified";
|
||||
method: "PATCH" | "POST"; path: string; id?: string }) => Promise<void>
|
||||
}
|
||||
const fields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"]
|
||||
const yes = (value: unknown) => value === true || value === "true" || value === "yes"
|
||||
const no = (value: unknown) => value === false || value === "false" || value === "no"
|
||||
const base = "/ip/traffic-flow"
|
||||
function must(value: unknown, message: string): asserts value {
|
||||
if (!value) throw Error(message)
|
||||
}
|
||||
function idOf(row: Row) {
|
||||
if (!Object.hasOwn(row, ".id")) return undefined
|
||||
const id = row[".id"]
|
||||
must(typeof id === "string" && /^\*[a-f0-9]{1,16}$/i.test(id), "Exact Traffic Flow singleton ID required")
|
||||
return id
|
||||
}
|
||||
|
||||
export function chrDisabledFlowPatchCount(ipfix: Row) {
|
||||
must(fields.every(key => yes(ipfix[key]) || no(ipfix[key])), "Exact restorable IPFIX fields required")
|
||||
return fields.some(key => no(ipfix[key])) ? 2 : 1
|
||||
}
|
||||
|
||||
/** Scope is an already proven isolated transit interface. This component owns
|
||||
* singleton changes only. The separate target owner must remove its ACKed row
|
||||
* and conserve both singletons before restore is admitted. */
|
||||
export function createDisabledChrFlowHarness(options: Options) {
|
||||
const baseline = structuredClone(options.baseline)
|
||||
const { transitInterface, authorize, verifyContext, read, native, checkpoint, relayWrite } = options
|
||||
must(baseline.deviceId && baseline.targets.length === 0 && no(baseline.flow.enabled), "Disabled target-free baseline required")
|
||||
must(/^[a-zA-Z0-9_.-]{1,64}$/.test(transitInterface) && transitInterface !== "all", "Exact transit interface required")
|
||||
const flowId = idOf(baseline.flow), ipfixId = idOf(baseline.ipfix)
|
||||
must(typeof baseline.flow.interfaces === "string" && baseline.flow.interfaces.length > 0, "Exact restorable settings required")
|
||||
chrDisabledFlowPatchCount(baseline.ipfix)
|
||||
const missing = fields.filter(key => no(baseline.ipfix[key]))
|
||||
const enableFields = Object.fromEntries(missing.map(key => [key, "yes"]))
|
||||
const restoreFields = Object.fromEntries(missing.map(key => [key, "no"]))
|
||||
let phase: "new" | "blocked" | "enabled" | "restored" = "new"
|
||||
let active: Inventory | undefined
|
||||
async function current() {
|
||||
await authorize()
|
||||
const result = structuredClone(await read())
|
||||
await verifyContext?.()
|
||||
await authorize()
|
||||
return result
|
||||
}
|
||||
async function change(step: Step, kind: "flow" | "ipfix", before: Inventory, body: Record<string, string>) {
|
||||
const id = kind === "flow" ? flowId : ipfixId
|
||||
const path = (kind === "flow" ? base : base + "/ipfix") + "/" + (id ? encodeRosId(id) : "set")
|
||||
const method = id ? "PATCH" : "POST"
|
||||
const record = { step, path, method, ...(id && { id }) } as const
|
||||
must(isDeepStrictEqual(await current(), before), "Traffic Flow baseline changed")
|
||||
await checkpoint({ ...record, phase: "intent" })
|
||||
let dispatched = false
|
||||
try {
|
||||
await runChrWrite(relayWrite?.admission, relayWrite && { step: "flow:" + relayWrite.side + ":" + step,
|
||||
side: relayWrite.side, deviceId: baseline.deviceId, method, path, body }, async arm => {
|
||||
const beforeDispatch = async () => {
|
||||
must(!dispatched && isDeepStrictEqual(await current(), before) && !dispatched, "Traffic Flow dispatch refused")
|
||||
dispatched = true
|
||||
await arm()
|
||||
}
|
||||
if (id) await native.patch(path, body, 15000, beforeDispatch)
|
||||
else await native.post(path, body, 15000, undefined, beforeDispatch)
|
||||
})
|
||||
must(dispatched, "Traffic Flow dispatch fence omitted")
|
||||
await checkpoint({ ...record, phase: "acknowledged" })
|
||||
const after = await current()
|
||||
const changed = after[kind]
|
||||
must(Object.entries(body).every(([key, value]) => key === "interfaces" ? changed[key] === value :
|
||||
value === "yes" ? yes(changed[key]) : no(changed[key])), "Traffic Flow settings differ")
|
||||
const conserved = structuredClone(after)
|
||||
for (const key of Object.keys(body)) conserved[kind][key] = before[kind][key]
|
||||
must(isDeepStrictEqual(conserved, before), "Unrelated Traffic Flow settings changed")
|
||||
await checkpoint({ ...record, phase: "verified" })
|
||||
return after
|
||||
} catch (error) {
|
||||
if (dispatched) throw Error("Traffic Flow write requires reconciliation. Automatic replay forbidden.")
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return {
|
||||
async enable() {
|
||||
must(phase === "new", "Traffic Flow enable already attempted")
|
||||
phase = "blocked"
|
||||
const before = missing.length ? await change("ipfix.enable", "ipfix", baseline, enableFields) : baseline
|
||||
active = await change("flow.enable", "flow", before, { enabled: "yes", interfaces: transitInterface })
|
||||
phase = "enabled"
|
||||
return structuredClone(active)
|
||||
},
|
||||
async restore() {
|
||||
must(phase === "enabled" && active, "No verified enabled state to restore")
|
||||
phase = "blocked"
|
||||
const after = await change("flow.restore", "flow", active, { enabled: "no", interfaces: String(baseline.flow.interfaces) })
|
||||
const restored = missing.length ? await change("ipfix.restore", "ipfix", after, restoreFields) : after
|
||||
must(isDeepStrictEqual(restored, baseline), "Original Traffic Flow baseline not restored")
|
||||
phase = "restored"
|
||||
return { restored: true as const, automaticReplay: false as const }
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises"
|
||||
import { tmpdir } from "node:os"
|
||||
import { join } from "node:path"
|
||||
import { test } from "node:test"
|
||||
import { chrExportDrainMs, executeDisabledChrCapture, type DisabledChrCaptureOptions } from "./traffic-flow-chr-disabled-runner.js"
|
||||
|
||||
test("export drain requires a bounded inactive-flow timeout plus two seconds", () => {
|
||||
assert.equal(chrExportDrainMs("15s"), 17000)
|
||||
assert.equal(chrExportDrainMs("00:00:30"), 32000)
|
||||
for (const value of ["", "0s", "1m1s", "300", undefined]) {
|
||||
assert.throws(() => chrExportDrainMs(value), /Inactive flow timeout/)
|
||||
}
|
||||
})
|
||||
|
||||
const cases = {
|
||||
"create ACK loss": /Target creation outcome unknown/,
|
||||
"unsealed direct capture": /capture producer stopped/,
|
||||
"staged capture ACK missing": /capture acknowledgement unavailable/,
|
||||
"staged enable ACK loss": /Target enable outcome unknown/,
|
||||
"unsealed staged capture": /capture producer stopped/,
|
||||
} as const
|
||||
|
||||
for (const mode of Object.keys(cases) as (keyof typeof cases)[]) test(mode, async () => {
|
||||
const staged = mode.includes("staged")
|
||||
const dir = await mkdtemp(join(tmpdir(), "mm-chr-disabled-"))
|
||||
const actions: string[] = []
|
||||
const cleanup: boolean[] = []
|
||||
let topologyChecks = 0, topologyChecksAtCapture = 0
|
||||
function router(index: number) {
|
||||
const name = "nfa012bcde" + (index === 0 ? "ea" : "eb")
|
||||
const baseline = { deviceId: String(index), flow: { ".id": "*F", enabled: "false", interfaces: "all", "inactive-flow-timeout": "15s" },
|
||||
ipfix: { ".id": "*A", ...Object.fromEntries(["bytes", "packets", "src-address", "dst-address", "protocol", "src-port",
|
||||
"dst-port", "in-interface", "out-interface"].map(key => [key, "true"])) }, targets: [] as Record<string, unknown>[] }
|
||||
const state = structuredClone(baseline)
|
||||
return { baseline, transitInterface: name, interfaces: [{ id: "*1", name, ifIndex: "1" },
|
||||
{ id: "*2", name: "nfa012bcde" + (index === 0 ? "ma" : "mb"), ifIndex: "2" }] as const,
|
||||
read: async () => state, counters: async () => {
|
||||
if (staged) actions.push("counters:" + index)
|
||||
return { startedAt: 1, observedAt: 1, rows: [] }
|
||||
},
|
||||
native: { async post() { throw Error("unexpected POST") },
|
||||
async patch<T>(_path: string, body: Record<string, string>, _timeout?: number, fence?: () => Promise<unknown>) {
|
||||
await fence?.()
|
||||
actions.push("flow:" + index)
|
||||
Object.assign(state.flow, body, { enabled: "true" })
|
||||
return undefined as T
|
||||
} },
|
||||
targetNative: { async put<T>(_path: string, body: Record<string, string>, _timeout?: number, fence?: () => Promise<unknown>): Promise<T> {
|
||||
await fence?.()
|
||||
actions.push("target:" + index)
|
||||
if (mode === "create ACK loss") throw Error("lost ACK")
|
||||
assert.equal(body.disabled, staged ? "yes" : undefined)
|
||||
state.targets.push({ ...body, ".id": "*C", disabled: body.disabled ?? "false" })
|
||||
return { ".id": "*C" } as T
|
||||
}, async patch<T>(path: string, body: Record<string, string>, _timeout?: number, fence?: () => Promise<unknown>) {
|
||||
await fence?.()
|
||||
assert.equal(path, "/ip/traffic-flow/target/*C")
|
||||
assert.deepEqual(body, { disabled: "no" })
|
||||
assert.equal(state.targets[0]?.disabled, "yes")
|
||||
state.targets[0].disabled = "false"
|
||||
actions.push("enable:" + index)
|
||||
if (mode === "staged enable ACK loss") throw Error("lost ACK")
|
||||
return { ".id": "*C" } as T
|
||||
}, async delete() { actions.push("unexpected inverse") } } }
|
||||
}
|
||||
const journalPath = join(dir, "owned.jsonl")
|
||||
try {
|
||||
const options: DisabledChrCaptureOptions = {
|
||||
journalPath, deadlineAt: Date.now() + 180000, ...(staged ? { stagedTarget: true as const } : {}),
|
||||
scope: { sources: ["192.0.2.1", "192.0.2.2"], destination: "192.0.2.3", port: 50000 },
|
||||
expected: [], serverIds: [1, 2], routers: [router(0), router(1)],
|
||||
assertTopology: async () => { topologyChecks++ }, authorize: async () => {},
|
||||
capture: async exercise => {
|
||||
if (staged) actions.push("capture")
|
||||
topologyChecksAtCapture = topologyChecks
|
||||
await exercise(new AbortController().signal, staged && mode !== "staged capture ACK missing"
|
||||
? { kind: "staged-ipfix-timeout-20s", captureDeadlineAt: Date.now() + 120000 } : undefined)
|
||||
throw Error("unexpected capture")
|
||||
},
|
||||
generate: async () => {
|
||||
if (staged) {
|
||||
assert.equal(topologyChecks - topologyChecksAtCapture, 5) // Two per enable, one before generation.
|
||||
actions.push("generate")
|
||||
}
|
||||
throw Error("capture producer stopped before seal")
|
||||
},
|
||||
drainCollector: async () => { actions.push("unexpected drain") },
|
||||
reconcile: async () => {}, reportRouterCleanup: clean => cleanup.push(clean),
|
||||
}
|
||||
await assert.rejects(executeDisabledChrCapture(options), cases[mode])
|
||||
await assert.rejects(executeDisabledChrCapture(options), { code: "EEXIST" })
|
||||
const prefix = ["flow:0", "target:0", "flow:1", "target:1"]
|
||||
assert.deepEqual(actions, {
|
||||
"create ACK loss": prefix.slice(0, 2),
|
||||
"unsealed direct capture": prefix,
|
||||
"staged capture ACK missing": [...prefix, "capture"],
|
||||
"staged enable ACK loss": [...prefix, "capture", "enable:0"],
|
||||
"unsealed staged capture": [...prefix, "capture", "enable:0", "enable:1", "counters:0", "counters:1", "generate"],
|
||||
}[mode])
|
||||
assert.deepEqual(cleanup, [false])
|
||||
const journal = (await readFile(journalPath, "utf8")).trim().split("\n").map(line => JSON.parse(line))
|
||||
if (staged && mode !== "staged capture ACK missing") {
|
||||
assert(journal.find(entry => entry.phase === "capture-ready"))
|
||||
}
|
||||
assert.deepEqual(journal.at(-1), {
|
||||
"create ACK loss": { device: 0, action: "create", phase: "intent" },
|
||||
"unsealed direct capture": { device: 1, action: "create", phase: "verified", id: "*C" },
|
||||
"staged capture ACK missing": { phase: "targets-staged-disabled" },
|
||||
"staged enable ACK loss": { device: 0, action: "enable", phase: "intent", id: "*C" },
|
||||
"unsealed staged capture": { device: 1, action: "enable", phase: "verified", id: "*C" },
|
||||
}[mode])
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,224 @@
|
||||
import { open } from "node:fs/promises"
|
||||
import { dirname } from "node:path"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js"
|
||||
import { assertChrTargetTimerWindow, createChrTargetHarness, type ChrStagedTargetTimerProfile, type ChrTargetTimerProfile } from "./traffic-flow-chr-target-harness.js"
|
||||
import { verifyChrOwnedWgEvidence } from "./traffic-flow-chr-owned-wg-evidence.js"
|
||||
import { projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
|
||||
import type { ChrWriteAdmission } from "./traffic-flow-chr-write-admission.js"
|
||||
|
||||
type Flow = Parameters<typeof createDisabledChrFlowHarness>[0]
|
||||
type Target = Parameters<typeof createChrTargetHarness>[0]
|
||||
type Evidence = Parameters<typeof verifyChrOwnedWgEvidence>[0]
|
||||
type Counters = Evidence["exporters"][0]["before"]
|
||||
type InterfacePins = Evidence["exporters"][0]["interfaces"]
|
||||
type Router = Pick<Flow, "baseline" | "read" | "transitInterface" | "native"> & {
|
||||
targetNative: Target["native"]
|
||||
interfaces: InterfacePins
|
||||
counters: () => Promise<Counters>
|
||||
}
|
||||
type Capture = Pick<Evidence, "bytes" | "sha256" | "complete" | "dropped">
|
||||
export type DisabledChrCaptureOptions = {
|
||||
journalPath: string
|
||||
deadlineAt: number
|
||||
writeAdmission?: ChrWriteAdmission
|
||||
targetTimerProfile?: Omit<ChrTargetTimerProfile, "now">
|
||||
stagedTarget?: true
|
||||
scope: Evidence["scope"]
|
||||
expected: Evidence["expected"]
|
||||
serverIds: readonly [number, number]
|
||||
routers: readonly [Router, Router]
|
||||
assertTopology: () => Promise<void>
|
||||
authorize: () => Promise<void>
|
||||
capture: (exercise: (signal: AbortSignal, timer?: Omit<ChrStagedTargetTimerProfile, "now">) => Promise<void>) => Promise<Capture>
|
||||
generate: (signal: AbortSignal) => Promise<{ lastDatagramAt: number }>
|
||||
drainCollector: () => Promise<void>
|
||||
reconcile: (wire: ReturnType<typeof projectChrWire>, serverIds: readonly [number, number]) => Promise<void>
|
||||
reportRouterCleanup: (clean: boolean) => void
|
||||
}
|
||||
function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) }
|
||||
|
||||
export function chrExportDrainMs(value: unknown) {
|
||||
must(typeof value === "string", "Inactive flow timeout unavailable")
|
||||
const compact = /^(?:(\d{1,2})m)?(?:(\d{1,2})s)?$/.exec(value)
|
||||
const clock = /^(\d{2}):(\d{2}):(\d{2})$/.exec(value)
|
||||
const seconds = compact ? Number(compact[1] ?? 0) * 60 + Number(compact[2] ?? 0) :
|
||||
clock ? Number(clock[1]) * 3600 + Number(clock[2]) * 60 + Number(clock[3]) : 0
|
||||
must(seconds >= 1 && seconds <= 60, "Inactive flow timeout outside owned capture budget")
|
||||
return seconds * 1000 + 2000
|
||||
}
|
||||
|
||||
/** Trusted local runner composition. The caller owns topology and collector.
|
||||
* A post-dispatch ambiguity is never converted into a synthetic cleanup ACK. */
|
||||
export async function executeDisabledChrCapture(options: DisabledChrCaptureOptions) {
|
||||
const scope = structuredClone(options.scope), expected = structuredClone(options.expected)
|
||||
const serverIds = structuredClone(options.serverIds), deadline = options.deadlineAt
|
||||
const writeAdmission = options.writeAdmission
|
||||
const targetTimerProfile = options.targetTimerProfile && { ...options.targetTimerProfile }
|
||||
const staged = options.stagedTarget === true
|
||||
const routers = options.routers.map(router => ({ ...router, baseline: structuredClone(router.baseline),
|
||||
interfaces: structuredClone(router.interfaces) }))
|
||||
const live = () => must(Number.isSafeInteger(deadline) && Date.now() < deadline, "Disabled capture authority expired")
|
||||
const authorize = async () => {
|
||||
live()
|
||||
await options.authorize()
|
||||
live()
|
||||
}
|
||||
const verifyTopology = async () => {
|
||||
await authorize()
|
||||
await options.assertTopology()
|
||||
await authorize()
|
||||
}
|
||||
live()
|
||||
must(deadline - Date.now() <= 600000 && routers.length === 2 && routers[0].baseline.deviceId !== routers[1].baseline.deviceId &&
|
||||
serverIds.length === 2 && new Set(serverIds).size === 2 && serverIds.every(id => Number.isSafeInteger(id) && id > 0) &&
|
||||
routers.every(router => router.interfaces.some(pin => pin.name === router.transitInterface && /e[ab]$/.test(pin.name))),
|
||||
"Invalid disabled capture fixture")
|
||||
must(!staged || (!targetTimerProfile && routers.every(router => "patch" in router.targetNative && typeof router.targetNative.patch === "function" &&
|
||||
router.baseline.targets.length === 0 && ["15s", "00:00:15"].some(value => value === router.baseline.flow["inactive-flow-timeout"]))),
|
||||
"Invalid staged target fixture")
|
||||
const drainMs = Math.max(...routers.map(router => chrExportDrainMs(router.baseline.flow["inactive-flow-timeout"])))
|
||||
const assertTargetTimersFit = () => {
|
||||
if (!targetTimerProfile) return
|
||||
must(targetTimerProfile.captureDeadlineAt <= deadline, "IPFIX capture timer exceeds the owner deadline")
|
||||
for (const router of routers) assertChrTargetTimerWindow(targetTimerProfile, router.baseline.flow["inactive-flow-timeout"])
|
||||
}
|
||||
assertTargetTimersFit()
|
||||
const journal = await open(options.journalPath, "wx", 0o600)
|
||||
const append = async (record: object) => {
|
||||
await journal.writeFile(JSON.stringify(record) + "\n")
|
||||
await journal.sync()
|
||||
}
|
||||
const states = routers.map(() => ({ clean: true, inverse: undefined as (() => Promise<unknown>) | undefined }))
|
||||
let work: Promise<void> | undefined, closed = false
|
||||
let before: Counters[] | undefined, finalExport: { lastDatagramAt: number; completedAt: number } | undefined
|
||||
let captureSealed = false
|
||||
let captureSignal: AbortSignal | undefined
|
||||
const routerAuthority = async () => {
|
||||
must(captureSealed || (!closed && !captureSignal?.aborted), "Disabled capture cancelled")
|
||||
await authorize()
|
||||
must(captureSealed || (!closed && !captureSignal?.aborted), "Disabled capture cancelled")
|
||||
}
|
||||
const routerContext = async () => {
|
||||
await routerAuthority()
|
||||
await options.assertTopology()
|
||||
await routerAuthority()
|
||||
}
|
||||
const stageRouters = async () => {
|
||||
const targets: ReturnType<typeof createChrTargetHarness>[] = []
|
||||
for (const [index, router] of routers.entries()) {
|
||||
const state = states[index]
|
||||
const checkpoint = (entry: object) => append({ device: index, ...entry })
|
||||
const relayWrite = writeAdmission && { side: index === 0 ? "A" as const : "B" as const, admission: writeAdmission }
|
||||
const flow = createDisabledChrFlowHarness({ ...router, authorize: routerAuthority, verifyContext: routerContext, checkpoint, relayWrite })
|
||||
await routerAuthority()
|
||||
state.clean = false
|
||||
const active = await flow.enable()
|
||||
state.inverse = async () => { await flow.restore(); state.clean = true }
|
||||
const targetOptions = { baseline: active, read: router.read,
|
||||
source: scope.sources[index], destination: scope.destination, port: scope.port, authorize: routerAuthority,
|
||||
verifyContext: routerContext, checkpoint, relayWrite }
|
||||
const native = router.targetNative
|
||||
const target = staged ? (() => {
|
||||
must("patch" in native && typeof native.patch === "function", "Staged target enable transport unavailable")
|
||||
return createChrTargetHarness({ ...targetOptions, stagedDisabled: true, native })
|
||||
})() : createChrTargetHarness({ ...targetOptions, native, timerProfile: targetTimerProfile })
|
||||
await routerAuthority()
|
||||
state.inverse = undefined
|
||||
await target.create()
|
||||
state.inverse = async () => { await target.remove(); await flow.restore(); state.clean = true }
|
||||
targets.push(target)
|
||||
}
|
||||
return targets
|
||||
}
|
||||
try {
|
||||
if (process.platform !== "win32") {
|
||||
const dir = await open(dirname(options.journalPath), "r")
|
||||
try { await dir.sync() } finally { await dir.close() }
|
||||
}
|
||||
await verifyTopology()
|
||||
await append({ phase: "admitted", scope, serverIds })
|
||||
const stagedTargets = staged ? await stageRouters() : undefined
|
||||
if (stagedTargets) await append({ phase: "targets-staged-disabled" })
|
||||
const capture = await options.capture((signal, captureTimer) => {
|
||||
must(!work && !closed && !signal.aborted, "Disabled capture exercise already dispatched or cancelled")
|
||||
work = (async () => {
|
||||
captureSignal = signal
|
||||
const timer = captureTimer && { ...captureTimer }
|
||||
must(staged ? timer?.kind === "staged-ipfix-timeout-20s" && timer.captureDeadlineAt <= deadline : !timer,
|
||||
"Staged capture acknowledgement unavailable")
|
||||
if (timer) for (const router of routers) assertChrTargetTimerWindow(timer, router.baseline.flow["inactive-flow-timeout"])
|
||||
if (stagedTargets) await routerAuthority()
|
||||
else await routerContext()
|
||||
if (!stagedTargets) before = await Promise.all(routers.map(router => router.counters()))
|
||||
assertTargetTimersFit()
|
||||
const targets = stagedTargets ?? await stageRouters()
|
||||
if (timer) {
|
||||
await append({ phase: "capture-ready", captureDeadlineAt: timer.captureDeadlineAt })
|
||||
for (const target of targets) await target.enable(timer)
|
||||
}
|
||||
await routerAuthority()
|
||||
if (stagedTargets) {
|
||||
await routerContext()
|
||||
for (const target of stagedTargets) await target.assertStagedTimerReady()
|
||||
before = await Promise.all(routers.map(router => router.counters()))
|
||||
} else if (targetTimerProfile) {
|
||||
for (const target of targets) await target.assertTimerReady()
|
||||
}
|
||||
await routerAuthority()
|
||||
const generatedAfter = Date.now()
|
||||
const { lastDatagramAt } = await options.generate(signal)
|
||||
must(Number.isSafeInteger(lastDatagramAt) && lastDatagramAt >= generatedAfter && lastDatagramAt <= Date.now() &&
|
||||
lastDatagramAt + drainMs < Math.min(deadline, timer?.captureDeadlineAt ?? targetTimerProfile?.captureDeadlineAt ?? deadline),
|
||||
"Final export drain cannot fit the owned window")
|
||||
await append({ phase: "final-export-drain", lastDatagramAt, drainMs })
|
||||
await delay(Math.max(0, lastDatagramAt + drainMs - Date.now()), undefined, { signal })
|
||||
// The staged path takes its complete post-traffic snapshot after the
|
||||
// host seals capture, with no native write between the two points.
|
||||
if (stagedTargets) await routerAuthority()
|
||||
else await routerContext()
|
||||
const completedAt = Date.now()
|
||||
must(completedAt >= lastDatagramAt + drainMs && (!timer || completedAt < timer.captureDeadlineAt), "Final export inactivity unconfirmed")
|
||||
await append({ phase: "final-export-drain-complete", lastDatagramAt, drainMs, completedAt })
|
||||
finalExport = { lastDatagramAt, completedAt }
|
||||
})()
|
||||
return work
|
||||
})
|
||||
must(work, "Capture omitted the disabled exercise")
|
||||
await work
|
||||
if (!staged) await verifyTopology()
|
||||
await authorize()
|
||||
const after = await Promise.all(routers.map(router => router.counters()))
|
||||
if (staged) await verifyTopology()
|
||||
must(before?.length === 2 && after.length === 2 && finalExport && states.every(state => state.inverse), "Disabled capture evidence incomplete")
|
||||
const evidence = verifyChrOwnedWgEvidence({ ...capture, scope, expected, requireRefresh: true,
|
||||
...(staged ? { counterWindow: "traffic" as const } : {}), exporters: [
|
||||
{ interfaces: routers[0].interfaces, before: before[0], after: after[0] },
|
||||
{ interfaces: routers[1].interfaces, before: before[1], after: after[1] },
|
||||
] })
|
||||
await append({ phase: "wire-bound", captureSha256: evidence.captureSha256, ...finalExport })
|
||||
captureSealed = true
|
||||
// The parent independently proves export inactivity and sealed capture
|
||||
// before authorizing these exact one-shot inverses.
|
||||
for (const state of [...states].reverse()) {
|
||||
const inverse = state.inverse
|
||||
state.inverse = undefined
|
||||
must(inverse, "Verified router inverse unavailable")
|
||||
await inverse()
|
||||
}
|
||||
must(states.every(state => state.clean), "Disabled capture router cleanup requires reconciliation")
|
||||
await verifyTopology()
|
||||
await options.drainCollector()
|
||||
await verifyTopology()
|
||||
await options.reconcile(projectChrWire(capture.bytes, scope), serverIds)
|
||||
await append({ phase: "reconciled" })
|
||||
return { ...evidence, componentSequenceComplete: true as const }
|
||||
} finally {
|
||||
closed = true
|
||||
// Capture may report child failure before its exercise settles.
|
||||
if (work) await work.catch(() => undefined)
|
||||
try { options.reportRouterCleanup(states.every(state => state.clean)) }
|
||||
finally { await journal.close() }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { networkInterfaces } from "node:os"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import type { McpPrincipal } from "../modules/mcp/store.js"
|
||||
import { readChrLinuxNamespaceIdentity, readChrLinuxUdpOwner } from "./traffic-flow-chr-linux-udp-owner.js"
|
||||
import { CHR_COLLECTOR_RESULT_POLL_MS } from "./traffic-flow-chr-target-timing-profile.js"
|
||||
|
||||
type Namespace = Awaited<ReturnType<typeof readChrLinuxNamespaceIdentity>>
|
||||
type Options = {
|
||||
runId: string
|
||||
deadlineAt: number
|
||||
namespace: Namespace
|
||||
hostNamespace: Namespace
|
||||
port: number
|
||||
serverIds: readonly [number, number]
|
||||
principal: McpPrincipal
|
||||
database: { name: string; oid: string; role: string }
|
||||
assertCurrent: () => Promise<void>
|
||||
/** Waits for a separate operator's ordinary approval of this exact new plan. */
|
||||
awaitHumanApproval: (planId: string, enabled: boolean, deadlineAt: number) => Promise<void>
|
||||
}
|
||||
const address = "198.18.242.3"
|
||||
const uuid = (value: unknown): value is string =>
|
||||
typeof value === "string" && /^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i.test(value)
|
||||
function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) }
|
||||
|
||||
/** Runs inside the disposable backend process. The reviewed parent owns its
|
||||
* launch, admission, process exit and namespace removal. */
|
||||
export async function createChrDisposableCollector(options: Options) {
|
||||
const { runId, deadlineAt, port, assertCurrent, awaitHumanApproval } = options
|
||||
const { namespace, hostNamespace, serverIds, database, principal } = structuredClone({
|
||||
namespace: options.namespace, hostNamespace: options.hostNamespace, serverIds: options.serverIds,
|
||||
database: options.database, principal: options.principal,
|
||||
})
|
||||
const instance = "netflow-" + runId
|
||||
const interfaceName = "nf" + runId + "cv"
|
||||
must(process.platform === "linux" && process.env.MMAPP_OWN_CHR_NETFLOW_COLLECTOR === "true" &&
|
||||
/^[a-f0-9]{8}$/.test(runId) && Number.isSafeInteger(deadlineAt) &&
|
||||
Date.now() < deadlineAt && deadlineAt - Date.now() <= 600000 &&
|
||||
Number.isSafeInteger(port) && port >= 49152 && port <= 65535 &&
|
||||
[namespace.device, namespace.inode, hostNamespace.device, hostNamespace.inode, database.oid].every(value => /^[1-9][0-9]{0,19}$/.test(value)) &&
|
||||
(namespace.device !== hostNamespace.device || namespace.inode !== hostNamespace.inode) &&
|
||||
database.name === "mcp_netflow_" + runId && database.role === "mcp_test" &&
|
||||
serverIds.length === 2 && new Set(serverIds).size === 2 && serverIds.every(value => Number.isSafeInteger(value) && value > 0),
|
||||
"Invalid disposable collector boundary")
|
||||
const fixtureRoot = "/tmp/mcp-netflow-" + runId
|
||||
const dbUrl = new URL(process.env.DATABASE_URL ?? "about:blank")
|
||||
must(["postgres:", "postgresql:"].includes(dbUrl.protocol) && dbUrl.hostname === "localhost" &&
|
||||
dbUrl.username === database.role && /^[a-f0-9]{64}$/.test(dbUrl.password) && !dbUrl.port && !dbUrl.hash &&
|
||||
dbUrl.pathname === "/" + database.name && [...dbUrl.searchParams.keys()].join() === "host" &&
|
||||
dbUrl.searchParams.get("host") === fixtureRoot + "/pg-socket" &&
|
||||
process.env.MCP_STATE_DIR === fixtureRoot + "/state", "Disposable database or MCP state endpoint is unpinned")
|
||||
const local = async () => {
|
||||
must(Date.now() < deadlineAt, "Disposable collector deadline ended")
|
||||
await assertCurrent()
|
||||
const actual = await readChrLinuxNamespaceIdentity()
|
||||
must(Date.now() < deadlineAt && actual.device === namespace.device && actual.inode === namespace.inode &&
|
||||
(networkInterfaces()[interfaceName] ?? []).filter(row => row.family === "IPv4" && row.address === address).length === 1 &&
|
||||
(!process.env.FLOW_LISTEN_HOST?.trim() || process.env.FLOW_LISTEN_HOST.trim() === address) &&
|
||||
!process.env.FLOW_AGENT_SOCKET?.trim(), "Disposable namespace or listener source changed")
|
||||
}
|
||||
await local()
|
||||
const [{ pool }, { invokeMcpTool, applyPlan, loadPlan }, { validatePrincipal }, ingest, { createFlowSettingsStore },
|
||||
{ flowSettingsPrivateProof }] = await Promise.all([
|
||||
import("../db/index.js"), import("../modules/mcp/runtime.js"), import("../modules/mcp/store.js"),
|
||||
import("../services/traffic-flow-ingest.js"), import("../services/traffic-flow-settings-store.js"),
|
||||
import("../services/traffic-flow-host-keys-service.js"),
|
||||
])
|
||||
const authorize = async () => {
|
||||
await local()
|
||||
await validatePrincipal(principal)
|
||||
await local()
|
||||
}
|
||||
await authorize()
|
||||
const db = (await pool.query("SELECT current_database() AS name,current_user AS role,oid::text AS oid," +
|
||||
"inet_server_addr() IS NULL AS unix_socket FROM pg_database WHERE datname=current_database()")).rows
|
||||
const servers = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 3")).rows.map(row => Number(row.id))
|
||||
must(db.length === 1 && db[0].name === database.name && db[0].role === database.role && db[0].oid === database.oid &&
|
||||
db[0].unix_socket === true && servers.length === 2 && serverIds.every(id => servers.includes(id)), "Disposable database identity differs")
|
||||
const configStore = createFlowSettingsStore(pool, authorize, () => {})
|
||||
const config = await configStore.read()
|
||||
must(config.enabled === false && config.collectorIp === address && config.flowListenPort === port,
|
||||
"Expected disabled exact collector configuration")
|
||||
const proof = (value: typeof config) => flowSettingsPrivateProof({ ...value, enabled: false })
|
||||
const invariantProof = await proof(config)
|
||||
const observedUdp = () => readChrLinuxUdpOwner({ namespace, address, port })
|
||||
const read = async () => {
|
||||
await authorize()
|
||||
const configuration = await configStore.read()
|
||||
const udpOwners = await observedUdp()
|
||||
const listener = ingest.getFlowListenerState(), worker = ingest.getFlowWorkerHealth()
|
||||
const currentProof = await proof(configuration)
|
||||
must(currentProof === invariantProof && configuration.collectorIp === address &&
|
||||
configuration.flowListenPort === port && typeof configuration.enabled === "boolean", "Disposable collector settings changed")
|
||||
await authorize()
|
||||
return { instance, pid: process.pid, invariantProof: currentProof, listenAddress: configuration.collectorIp,
|
||||
listenPort: configuration.flowListenPort, enabled: configuration.enabled, workerAlive: worker.alive,
|
||||
boundAddress: listener.bound ? listener.address : null, udpOwners }
|
||||
}
|
||||
const initial = await read()
|
||||
must(initial.udpOwners.length === 0 && !initial.workerAlive && initial.boundAddress === null, "Disposable collector not idle")
|
||||
let enableAttempted = false, disableAttempted = false, enabledVerified = false, disableVerified = false
|
||||
const applyEnabled = async (enabled: boolean, beforeDispatch: () => Promise<void>) => {
|
||||
must(enabled ? !enableAttempted : enabledVerified && !disableAttempted, "Collector mutation already attempted or unknown")
|
||||
if (enabled) enableAttempted = true
|
||||
else disableAttempted = true
|
||||
await authorize()
|
||||
const current = await read()
|
||||
must(current.enabled !== enabled, "Unexpected collector enable state")
|
||||
const prepared = await invokeMcpTool(principal, "mm_netflow_settings_update_plan", { settings: { enabled } }) as {
|
||||
state?: string; planId?: string
|
||||
}
|
||||
must(prepared.state === "awaiting-approval" && uuid(prepared.planId), "Collector plan did not await human approval")
|
||||
await awaitHumanApproval(prepared.planId, enabled, deadlineAt)
|
||||
await authorize()
|
||||
const approved = await loadPlan(prepared.planId)
|
||||
must(approved.connection_id === principal.connectionId && approved.operation_id === "netflow.settings.update" &&
|
||||
approved.state === "approved" && approved.approved_by && approved.expires_at.getTime() > Date.now(), "Collector approval unconfirmed")
|
||||
await beforeDispatch()
|
||||
await authorize()
|
||||
const operation = await applyPlan(principal, prepared.planId, randomUUID())
|
||||
must(uuid(operation.id), "Collector operation acknowledgement missing")
|
||||
const pollUntil = Math.min(deadlineAt, Date.now() + CHR_COLLECTOR_RESULT_POLL_MS)
|
||||
while (Date.now() < pollUntil) {
|
||||
await authorize()
|
||||
const rows = (await pool.query("SELECT o.state,o.verification,s.state AS step_state,s.result " +
|
||||
"FROM mcp_operations o LEFT JOIN mcp_operation_steps s ON s.operation_id=o.id " +
|
||||
"WHERE o.id=$1 AND o.connection_id=$2 AND o.plan_id=$3 LIMIT 2",
|
||||
[operation.id, principal.connectionId, prepared.planId])).rows
|
||||
const [row] = rows
|
||||
if (rows.length === 1 && ["pending", "running"].includes(row.state)) { await delay(100); continue }
|
||||
must(rows.length === 1 && row.state === "applied" && row.verification === "passed" && row.step_state === "verified" &&
|
||||
row.result?.configurationConfirmed === true && row.result.receiptConfirmed === true &&
|
||||
row.result.listener === "request-acknowledged" && row.result.automaticReplay === false &&
|
||||
uuid(row.result.receiptId), "Collector operation result requires reconciliation")
|
||||
const receipt = (await pool.query("SELECT phase,state FROM flow_settings_intents " +
|
||||
"WHERE receipt_id=$1 ORDER BY phase", [row.result.receiptId])).rows
|
||||
must(receipt.length === 2 && receipt[0].phase === "configuration" && receipt[0].state === "completed" &&
|
||||
receipt[1].phase === "listener" && receipt[1].state === "completed", "Collector durable listener acknowledgement missing")
|
||||
await authorize()
|
||||
if (enabled) enabledVerified = true
|
||||
else disableVerified = true
|
||||
return { operationId: operation.id, acknowledged: true }
|
||||
}
|
||||
throw Error("Collector operation outcome unknown. Automatic replay forbidden.")
|
||||
}
|
||||
const assertDrained = async () => {
|
||||
const observed = await read()
|
||||
must(disableVerified && !observed.enabled && !observed.workerAlive &&
|
||||
observed.boundAddress === null && observed.udpOwners.length === 0, "Collector retirement unconfirmed")
|
||||
}
|
||||
return { interfaceName, pins: { instance, pid: process.pid, invariantProof }, collector: { authorize, read, applyEnabled, assertDrained } }
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { verifyChrFirewallPreflight } from "./traffic-flow-chr-firewall-preflight.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
const rule = { ".id": "*A", chain: "forward", action: "accept", disabled: "false", protocol: "udp",
|
||||
"src-address": "198.18.240.0/24", "dst-address": "198.18.240.2/32",
|
||||
"in-interface": "edge-a", "out-interface": "middle-a", "dst-port": "50003,50101-50103",
|
||||
"connection-state": "new,established", log: "false", "log-prefix": "" }
|
||||
|
||||
test("owned WG policy admits complete empty snapshots only and requires matching path profile", async () => {
|
||||
const { config, rules } = fixture()
|
||||
rules.filter.length = 0
|
||||
const profile = "owned-wg-empty-firewall" as const
|
||||
const owned = { ...config, profile, readPathEvidence: async () => ({
|
||||
...await config.readPathEvidence(), profile, ownedInterfaces: ["edge-a", "middle-a"],
|
||||
ownedRoutes: [{ destination: "198.18.240.1", interfaceName: "edge-a" }, { destination: "198.18.240.2", interfaceName: "middle-a" }],
|
||||
udpPairs: ["198.18.240.1:50100/198.18.240.2:50101"],
|
||||
}) }
|
||||
assert.equal((await verifyChrFirewallPreflight(owned)).policyVerified, true)
|
||||
await assert.rejects(verifyChrFirewallPreflight({ ...owned, packets: [{ ...config.packets[0], destinationPort: 50102 }] }), /only owned WireGuard/)
|
||||
await assert.rejects(verifyChrFirewallPreflight({ ...owned, readPathEvidence: config.readPathEvidence }), /reachability/)
|
||||
rules.raw.push({ ".id": "*A", chain: "output", action: "accept", disabled: "true" })
|
||||
await assert.rejects(verifyChrFirewallPreflight(owned), /requires empty firewall/)
|
||||
})
|
||||
function fixture() {
|
||||
const calls: string[] = []
|
||||
const rules = { raw: [] as Row[], mangle: [] as Row[], nat: [] as Row[], filter: [rule] as Row[] }
|
||||
const config = {
|
||||
identity: "router-a", version: "7.24.5 (stable)",
|
||||
packets: [{ direction: "forward" as const, source: "198.18.240.1", destination: "198.18.240.2",
|
||||
sourcePort: 50100, destinationPort: 50101, ingress: "edge-a", egress: "middle-a" }],
|
||||
assertCurrent: async () => {},
|
||||
readPathEvidence: async () => ({ routesVerified: true, freshConntrackTuples: true, fingerprint: "a".repeat(64) }),
|
||||
client: { async get<T>(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise<unknown>) {
|
||||
assert.ok(timeout && timeout <= 2000)
|
||||
assert.equal(bytes, 128 * 1024)
|
||||
await fence?.()
|
||||
calls.push(path)
|
||||
if (path === "/system/identity") return [{ name: "router-a" }] as T
|
||||
if (path === "/system/resource") return [{ version: "7.24.5 (stable)" }] as T
|
||||
const table = path.replace("/ip/firewall/", "") as keyof typeof rules
|
||||
return rules[table].map(row => ({ ...row, packets: String(calls.length), bytes: "100" })) as T
|
||||
} },
|
||||
}
|
||||
return { config, rules, calls }
|
||||
}
|
||||
|
||||
test("reads only fixed bounded native inventories twice and accepts exact simple UDP policy", async () => {
|
||||
const { config, calls, rules } = fixture()
|
||||
rules.filter.unshift({ ".id": "*B", chain: "forward", action: "drop", protocol: "tcp", "in-interface-list": "any" })
|
||||
const paths = ["/system/identity", "/system/resource", "/ip/firewall/raw", "/ip/firewall/mangle",
|
||||
"/ip/firewall/nat", "/ip/firewall/filter"]
|
||||
assert.deepEqual(await verifyChrFirewallPreflight(config), { policyVerified: true, packetScopes: 1, actualIngressVerified: false })
|
||||
assert.deepEqual(calls, [...paths, ...paths])
|
||||
})
|
||||
|
||||
test("rejects matching FastTrack, NAT, jumps, unsupported matchers and incomplete filter admission", async () => {
|
||||
for (const [table, inserted] of [
|
||||
["filter", { chain: "forward", action: "fasttrack-connection", "connection-state": "established" }],
|
||||
["nat", { chain: "srcnat", action: "masquerade" }],
|
||||
["raw", { chain: "prerouting", action: "jump", "jump-target": "policy" }],
|
||||
["filter", { chain: "forward", action: "accept", "src-address-list": "unknown" }],
|
||||
] as const) {
|
||||
const { config, rules } = fixture()
|
||||
rules[table].unshift({ ".id": "*B", ...inserted })
|
||||
await assert.rejects(verifyChrFirewallPreflight(config), /matching|unsupported/)
|
||||
}
|
||||
const { config, rules } = fixture()
|
||||
rules.filter.length = 0
|
||||
await assert.rejects(verifyChrFirewallPreflight(config), /no explicit filter allow/)
|
||||
})
|
||||
|
||||
test("rejects inventory drift, unavailable route or conntrack proof, and lost authorization", async () => {
|
||||
const stale = fixture()
|
||||
stale.config.readPathEvidence = async () => {
|
||||
if (stale.calls.length === 6) stale.rules.filter[0] = { ...rule, comment: "changed" }
|
||||
return { routesVerified: true, freshConntrackTuples: true, fingerprint: "a".repeat(64) }
|
||||
}
|
||||
await assert.rejects(verifyChrFirewallPreflight(stale.config), /stale inventory/)
|
||||
const unknown = fixture()
|
||||
unknown.config.readPathEvidence = async () => ({ routesVerified: true, freshConntrackTuples: false, fingerprint: "a".repeat(64) })
|
||||
await assert.rejects(verifyChrFirewallPreflight(unknown.config), /unknown active reachability/)
|
||||
const expired = fixture()
|
||||
expired.config.assertCurrent = async () => { throw Error("ownership expired") }
|
||||
await assert.rejects(verifyChrFirewallPreflight(expired.config), /ownership expired/)
|
||||
assert.equal(expired.calls.length, 0)
|
||||
})
|
||||
@@ -0,0 +1,179 @@
|
||||
import { isIPv4 } from "node:net"
|
||||
import { performance } from "node:perf_hooks"
|
||||
import { isDeepStrictEqual } from "node:util"
|
||||
import type { MikrotikClient } from "../services/mikrotik.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
const tables = ["raw", "mangle", "nat", "filter"] as const
|
||||
type Table = (typeof tables)[number]
|
||||
type Packet = {
|
||||
direction: "input" | "forward" | "output"
|
||||
source: string
|
||||
destination: string
|
||||
sourcePort: number
|
||||
destinationPort: number
|
||||
ingress?: string
|
||||
egress?: string
|
||||
}
|
||||
type Options = {
|
||||
client: Pick<MikrotikClient, "get">
|
||||
identity: string
|
||||
version: string
|
||||
packets: readonly Packet[]
|
||||
assertCurrent: () => Promise<void>
|
||||
/** Trusted reader of exact active routes and address/interface bindings. */
|
||||
profile?: "owned-wg-empty-firewall"
|
||||
readPathEvidence: () => Promise<{ routesVerified: boolean; freshConntrackTuples: boolean; fingerprint: string;
|
||||
profile?: "owned-wg-empty-firewall"; ownedInterfaces?: readonly string[];
|
||||
ownedRoutes?: readonly { destination: string; interfaceName: string }[]; udpPairs?: readonly string[] }>
|
||||
}
|
||||
const paths: Record<Table, Record<Packet["direction"], readonly string[]>> = {
|
||||
raw: { input: ["prerouting"], forward: ["prerouting"], output: ["output"] },
|
||||
mangle: { input: ["prerouting", "input"], forward: ["prerouting", "forward", "postrouting"], output: ["output", "postrouting"] },
|
||||
nat: { input: ["dstnat", "input"], forward: ["dstnat", "srcnat"], output: ["output", "srcnat"] },
|
||||
filter: { input: ["input"], forward: ["forward"], output: ["output"] },
|
||||
}
|
||||
const meta = new Set([".id", "action", "chain", "comment", "disabled", "dynamic", "invalid"])
|
||||
const matchKeys = new Set(["protocol", "src-address", "dst-address", "src-port", "dst-port", "in-interface", "out-interface", "connection-state"])
|
||||
const states = ["new", "established", "related", "invalid", "untracked"] as const
|
||||
const deny = (reason: string): never => { throw Error("RouterOS fixture preflight refused: " + reason) }
|
||||
const validPort = (value: number) => Number.isInteger(value) && value > 0 && value <= 65535
|
||||
const flag = (value: unknown) => {
|
||||
if (value === undefined || value === false || value === "false" || value === "no") return false
|
||||
if (value === true || value === "true" || value === "yes") return true
|
||||
return deny("unsupported rule flag")
|
||||
}
|
||||
const ipv4Number = (address: string) => address.split(".").reduce((sum, octet) => sum * 256 + Number(octet), 0)
|
||||
function addressMatches(value: string, ip: string) {
|
||||
const [base, bits = "32", ...extra] = value.split("/")
|
||||
if (!isIPv4(base) || !/^(0|[1-9]|[12][0-9]|3[0-2])$/.test(bits) || extra.length) return undefined
|
||||
const size = 2 ** (32 - Number(bits))
|
||||
return Math.floor(ipv4Number(base) / size) === Math.floor(ipv4Number(ip) / size)
|
||||
}
|
||||
function portMatches(value: string, port: number) {
|
||||
const ranges = value.split(",").map(range => /^([1-9][0-9]{0,4})(?:-([1-9][0-9]{0,4}))?$/.exec(range))
|
||||
if (ranges.some(range => !range || !validPort(Number(range[1])) || !validPort(Number(range[2] ?? range[1])) ||
|
||||
Number(range[1]) > Number(range[2] ?? range[1]))) return undefined
|
||||
return ranges.some(range => range && Number(range[1]) <= port && port <= Number(range[2] ?? range[1]))
|
||||
}
|
||||
function matcher(key: string, value: unknown, packet: Packet, state: "new" | "established", chain: string) {
|
||||
if (typeof value !== "string" || !value) return undefined
|
||||
switch (key) {
|
||||
case "protocol":
|
||||
return /^(udp|tcp|icmp|gre|ipsec-esp|17|6|1|47|50)$/.test(value) ? value === "udp" || value === "17" : undefined
|
||||
case "src-address": return addressMatches(value, packet.source)
|
||||
case "dst-address": return addressMatches(value, packet.destination)
|
||||
case "src-port": return portMatches(value, packet.sourcePort)
|
||||
case "dst-port": return portMatches(value, packet.destinationPort)
|
||||
case "in-interface":
|
||||
return !["postrouting", "srcnat"].includes(chain) && packet.ingress && /^[\w.-]{1,64}$/.test(value)
|
||||
? value === packet.ingress : undefined
|
||||
case "out-interface":
|
||||
return !["prerouting", "dstnat"].includes(chain) && packet.egress && /^[\w.-]{1,64}$/.test(value)
|
||||
? value === packet.egress : undefined
|
||||
case "connection-state": {
|
||||
const values = value.split(",")
|
||||
return values.every(item => (states as readonly string[]).includes(item)) ? values.includes(state) : undefined
|
||||
}
|
||||
default: return undefined
|
||||
}
|
||||
}
|
||||
function matches(rule: Row, packet: Packet, state: "new" | "established", chain: string, table: Table) {
|
||||
const results = Object.entries(rule).filter(([key]) => !meta.has(key))
|
||||
.filter(([key, value]) => !(key === "log" && (value === false || value === "false" || value === "no") ||
|
||||
key === "log-prefix" && value === ""))
|
||||
.map(([key, value]) => matchKeys.has(key) && !(table === "raw" && key === "connection-state")
|
||||
? matcher(key, value, packet, state, chain) : undefined)
|
||||
if (results.includes(false)) return false
|
||||
if (results.includes(undefined)) deny("unsupported potentially matching rule")
|
||||
if (flag(rule.dynamic) || flag(rule.invalid)) deny("dynamic or invalid matching rule")
|
||||
return true
|
||||
}
|
||||
function verifyRules(inventory: Record<Table, Row[]>, packet: Packet) {
|
||||
for (const state of ["new", "established"] as const) {
|
||||
for (const table of tables) {
|
||||
for (const chain of paths[table][packet.direction]) {
|
||||
let accepted = false
|
||||
for (const rule of inventory[table]) {
|
||||
if (rule.chain !== chain || flag(rule.disabled) || !matches(rule, packet, state, chain, table)) continue
|
||||
if (rule.action !== "accept") deny("matching non-accept action")
|
||||
accepted = true
|
||||
break
|
||||
}
|
||||
// Require an explicit filter allow even though RouterOS may implicitly accept.
|
||||
if (table === "filter" && !accepted) deny("no explicit filter allow")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
function rows(value: unknown, limit: number): Row[] {
|
||||
const result = Array.isArray(value) ? value : [value]
|
||||
if (result.length > limit || result.some(row => !row || typeof row !== "object" || Array.isArray(row))) return deny("invalid inventory")
|
||||
return result
|
||||
}
|
||||
|
||||
/** Bounded double read via the existing RouterOS client. This proves a narrow
|
||||
* policy subset at observation time. It does not prove packet delivery. */
|
||||
export async function verifyChrFirewallPreflight(options: Options) {
|
||||
const { client, assertCurrent, readPathEvidence, profile } = options
|
||||
const { identity, version, packets } = structuredClone({ identity: options.identity, version: options.version, packets: options.packets })
|
||||
if (!identity || !version || !packets.length || packets.length > 32 ||
|
||||
packets.some(packet => !Object.hasOwn(paths.filter, packet.direction) ||
|
||||
!isIPv4(packet.source) || !isIPv4(packet.destination) || !validPort(packet.sourcePort) || !validPort(packet.destinationPort) ||
|
||||
(packet.direction !== "output" && !packet.ingress) || (packet.direction !== "input" && !packet.egress) ||
|
||||
[packet.ingress, packet.egress].some(name => name !== undefined && !/^[\w.-]{1,64}$/.test(name)))) deny("invalid packet scope")
|
||||
const deadline = performance.now() + 15000
|
||||
async function fence() {
|
||||
if (performance.now() >= deadline) deny("observation expired")
|
||||
await assertCurrent()
|
||||
if (performance.now() >= deadline) deny("observation expired")
|
||||
}
|
||||
async function get(path: string, limit: number) {
|
||||
await fence()
|
||||
const result = rows(await client.get<unknown>(path, Math.min(2000, Math.ceil(deadline - performance.now())),
|
||||
undefined, 128 * 1024, fence), limit)
|
||||
await fence()
|
||||
return result
|
||||
}
|
||||
async function snapshot() {
|
||||
const [device] = await get("/system/identity", 1)
|
||||
const [resource] = await get("/system/resource", 1)
|
||||
if (device?.name !== identity || resource?.version !== version) deny("device identity changed")
|
||||
const inventory = {} as Record<Table, Row[]>
|
||||
for (const table of tables) {
|
||||
inventory[table] = (await get("/ip/firewall/" + table, 256)).map(({ bytes: _bytes, packets: _packets, ...rule }) => {
|
||||
if (typeof rule.chain !== "string" || !rule.chain || typeof rule.action !== "string" ||
|
||||
typeof rule[".id"] !== "string" || !/^\*[a-f0-9]{1,16}$/i.test(rule[".id"])) deny("invalid rule identity")
|
||||
return structuredClone(rule)
|
||||
})
|
||||
if (new Set(inventory[table].map(rule => rule[".id"])).size !== inventory[table].length) deny("duplicate rule ID")
|
||||
}
|
||||
await fence()
|
||||
const evidence = await readPathEvidence()
|
||||
await fence()
|
||||
if (evidence.routesVerified !== true || evidence.freshConntrackTuples !== true || evidence.profile !== profile ||
|
||||
!/^[a-f0-9]{64}$/.test(evidence.fingerprint)) deny("unknown active reachability or existing connection")
|
||||
if (profile && (evidence.ownedInterfaces?.length !== 2 ||
|
||||
!packets.some(packet => packet.direction === "forward") ||
|
||||
packets.some(packet => packet.direction === "forward" &&
|
||||
(packet.ingress === packet.egress || !evidence.ownedInterfaces?.includes(packet.ingress!) ||
|
||||
!evidence.ownedInterfaces?.includes(packet.egress!) ||
|
||||
!evidence.ownedRoutes?.some(route => route.destination === packet.destination && route.interfaceName === packet.egress) ||
|
||||
!evidence.ownedRoutes?.some(route => route.destination === packet.source && route.interfaceName === packet.ingress) ||
|
||||
!evidence.udpPairs?.includes([packet.source + ":" + packet.sourcePort, packet.destination + ":" + packet.destinationPort].sort().join("/")))))) {
|
||||
deny("forwarding does not use only owned WireGuard")
|
||||
}
|
||||
return { inventory, fingerprint: evidence.fingerprint, ownedInterfaces: evidence.ownedInterfaces,
|
||||
ownedRoutes: evidence.ownedRoutes, udpPairs: evidence.udpPairs }
|
||||
}
|
||||
const before = await snapshot()
|
||||
const after = await snapshot()
|
||||
if (!isDeepStrictEqual(before, after)) deny("stale inventory or path")
|
||||
if (profile === "owned-wg-empty-firewall") {
|
||||
if (tables.some(table => after.inventory[table].length !== 0)) deny("owned WireGuard profile requires empty firewall")
|
||||
} else {
|
||||
for (const packet of packets) verifyRules(after.inventory, packet)
|
||||
}
|
||||
await fence()
|
||||
return { policyVerified: true as const, packetScopes: packets.length, actualIngressVerified: false as const }
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import { constants } from "node:fs"
|
||||
import { lstat, open } from "node:fs/promises"
|
||||
import { createConnection } from "node:net"
|
||||
import { CHR_STAGED_TARGET_CAPTURE_MAX_MS } from "./traffic-flow-chr-target-timing-profile.js"
|
||||
import type { ChrStagedTargetTimerProfile } from "./traffic-flow-chr-target-harness.js"
|
||||
|
||||
type Operation = "topology-ready" | "generate" | "capture-start" | "capture-stop" | "cancel"
|
||||
type Phase = "calibration" | "initial" | "refresh"
|
||||
type Identity = { device: string; inode: string }
|
||||
const limit = 4096
|
||||
const captureLimit = 512 * 1024
|
||||
function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) }
|
||||
|
||||
export function decodeChrHostReply(raw: Buffer, sequence: number, op: Operation, stagedStart = false) {
|
||||
must(raw.length > 0 && raw.length <= limit && raw.at(-1) === 10 && raw.indexOf(10) === raw.length - 1,
|
||||
"Invalid host IPC frame")
|
||||
const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(raw))
|
||||
must(value && value.sequence === sequence && value.op === op && value.ok === true, "Host IPC acknowledgement unavailable")
|
||||
const extra = op === "generate" ? ["lastDatagramAt"] : op === "capture-stop" ? ["sha256", "complete", "dropped"] :
|
||||
op === "capture-start" && stagedStart ? ["captureDeadlineAt"] : []
|
||||
must(Object.keys(value).sort().join() === ["sequence", "op", "ok", ...extra].sort().join(), "Invalid host IPC reply shape")
|
||||
if (op === "generate") must(Number.isSafeInteger(value.lastDatagramAt) && value.lastDatagramAt > 0, "Host UDP time unconfirmed")
|
||||
if (op === "capture-stop") must(value.complete === true && value.dropped === 0 && typeof value.sha256 === "string" &&
|
||||
/^[a-f0-9]{64}$/.test(value.sha256), "Host capture unconfirmed")
|
||||
if (op === "capture-start" && stagedStart) must(Number.isSafeInteger(value.captureDeadlineAt) && value.captureDeadlineAt > 0,
|
||||
"Host capture deadline unconfirmed")
|
||||
return value as { lastDatagramAt?: number; sha256?: string; captureDeadlineAt?: number }
|
||||
}
|
||||
|
||||
export function bindChrStagedHostCaptureTimer(captureDeadlineAt: number | undefined, captureMs: number,
|
||||
ownerDeadline: number, requestedAt: number, acknowledgedAt: number): Omit<ChrStagedTargetTimerProfile, "now"> {
|
||||
must(Number.isSafeInteger(captureMs) && captureMs >= 90000 && captureMs <= CHR_STAGED_TARGET_CAPTURE_MAX_MS &&
|
||||
[ownerDeadline, requestedAt, acknowledgedAt].every(value => Number.isSafeInteger(value) && value > 0) &&
|
||||
captureDeadlineAt !== undefined && Number.isSafeInteger(captureDeadlineAt) && requestedAt <= acknowledgedAt &&
|
||||
captureDeadlineAt > acknowledgedAt && captureDeadlineAt <= ownerDeadline &&
|
||||
captureDeadlineAt >= requestedAt + captureMs && captureDeadlineAt <= acknowledgedAt + captureMs,
|
||||
"Host capture deadline outside reviewed window")
|
||||
return { kind: "staged-ipfix-timeout-20s", captureDeadlineAt }
|
||||
}
|
||||
|
||||
export async function createChrHostControl(options: {
|
||||
runId: string; nonce: string; deadlineAt: number; socket: Identity; assertCurrent: () => Promise<void>
|
||||
}) {
|
||||
const { runId, nonce, deadlineAt, assertCurrent } = options
|
||||
const pin = structuredClone(options.socket)
|
||||
const gid = process.getegid?.()
|
||||
must(process.platform === "linux" && gid && /^[a-f0-9]{8}$/.test(runId) &&
|
||||
/^[a-f0-9]{32}$/.test(nonce) && nonce.startsWith(runId), "Host IPC ownership unavailable")
|
||||
const directory = "/tmp/mcp-netflow-" + runId + "/ipc"
|
||||
const socketPath = directory + "/control.sock"
|
||||
const fence = async () => {
|
||||
await assertCurrent()
|
||||
const [dir, socket] = await Promise.all([lstat(directory, { bigint: true }), lstat(socketPath, { bigint: true })])
|
||||
must(Date.now() < deadlineAt && dir.isDirectory() && dir.uid === 0n && dir.gid === BigInt(gid) &&
|
||||
(dir.mode & 0o777n) === 0o710n && socket.isSocket() && socket.uid === 0n && socket.gid === BigInt(gid) &&
|
||||
(socket.mode & 0o777n) === 0o660n && String(socket.dev) === pin.device && String(socket.ino) === pin.inode,
|
||||
"Host IPC ownership changed")
|
||||
}
|
||||
await fence()
|
||||
let sequence = 0, busy = false, failed = false, phase = "new"
|
||||
const reached = (expected: string) => phase === expected
|
||||
async function exchange(op: Operation, signal?: AbortSignal, generation?: Phase, stagedStart = false) {
|
||||
must(!failed && !busy && !signal?.aborted && sequence < 8, "Host IPC request refused")
|
||||
busy = true
|
||||
try {
|
||||
await fence()
|
||||
const current = ++sequence
|
||||
const started = Date.now()
|
||||
const raw = await new Promise<Buffer>((resolve, reject) => {
|
||||
const sock = createConnection(socketPath)
|
||||
let reply = Buffer.alloc(0)
|
||||
const fail = () => { sock.destroy(); reject(Error("Host IPC outcome unknown")) }
|
||||
const timer = setTimeout(fail, Math.max(1, Math.min(15000, deadlineAt - Date.now())))
|
||||
const cleanup = () => { clearTimeout(timer); signal?.removeEventListener("abort", fail) }
|
||||
signal?.addEventListener("abort", fail, { once: true })
|
||||
if (signal?.aborted) fail()
|
||||
sock.once("connect", () => sock.write(JSON.stringify({ runId, nonce, sequence: current, op,
|
||||
...(generation && { phase: generation }) }) + "\n"))
|
||||
sock.on("data", chunk => {
|
||||
if (reply.length + chunk.length > limit) return fail()
|
||||
reply = Buffer.concat([reply, chunk])
|
||||
if (reply.includes(10)) { cleanup(); resolve(reply); sock.destroy() }
|
||||
})
|
||||
sock.once("error", fail)
|
||||
sock.once("close", () => { cleanup(); reject(Error("Host IPC closed before acknowledgement")) })
|
||||
})
|
||||
const value = decodeChrHostReply(raw, current, op, stagedStart)
|
||||
await fence()
|
||||
if (op === "generate") must(value.lastDatagramAt! >= started && value.lastDatagramAt! <= Date.now(),
|
||||
"Host UDP receipt is outside request window")
|
||||
return value
|
||||
} catch (error) { failed = true; throw error }
|
||||
finally { busy = false }
|
||||
}
|
||||
async function capture(deadline: number, exercise: (signal: AbortSignal, timer?: Omit<ChrStagedTargetTimerProfile, "now">) => Promise<void>,
|
||||
stagedMs?: number) {
|
||||
const requestedAt = Date.now()
|
||||
must(phase === "calibration" && requestedAt < deadline && deadline <= deadlineAt, "Host capture phase differs")
|
||||
const startSignal = AbortSignal.timeout(deadline - requestedAt)
|
||||
const started = await exchange("capture-start", startSignal, undefined, stagedMs !== undefined)
|
||||
phase = "capturing"
|
||||
try {
|
||||
const acknowledgedAt = Date.now()
|
||||
const timer = stagedMs === undefined ? undefined :
|
||||
bindChrStagedHostCaptureTimer(started.captureDeadlineAt, stagedMs, deadline, requestedAt, acknowledgedAt)
|
||||
const signal = timer ? AbortSignal.timeout(timer.captureDeadlineAt - acknowledgedAt) : startSignal
|
||||
if (timer) await exercise(signal, timer)
|
||||
else await exercise(signal)
|
||||
must(reached("refresh") && !signal.aborted, "Host capture exercise incomplete")
|
||||
const { sha256 } = await exchange("capture-stop", signal)
|
||||
phase = "finished"
|
||||
const file = await open(directory + "/capture.pcap", constants.O_RDONLY | constants.O_NOFOLLOW)
|
||||
try {
|
||||
const row = await file.stat()
|
||||
must(row.isFile() && row.uid === 0 && row.gid === gid && (row.mode & 0o777) === 0o640 &&
|
||||
row.nlink === 1 && row.size > 0 && row.size <= captureLimit, "Host capture ownership unavailable")
|
||||
const bytes = Buffer.alloc(row.size + 1)
|
||||
let offset = 0
|
||||
while (offset < bytes.length) {
|
||||
const result = await file.read(bytes, offset, bytes.length - offset, null)
|
||||
if (!result.bytesRead) break
|
||||
offset += result.bytesRead
|
||||
}
|
||||
const capture = bytes.subarray(0, offset)
|
||||
must(offset === row.size && createHash("sha256").update(capture).digest("hex") === sha256, "Host capture bytes differ")
|
||||
return { bytes: capture, sha256: sha256!, complete: true, dropped: 0 }
|
||||
} finally { await file.close() }
|
||||
} finally {
|
||||
if (!failed && phase !== "finished") {
|
||||
phase = "cancelled"
|
||||
// Parent cancellation intentionally refuses the session. The
|
||||
// original capture failure remains the terminal cause.
|
||||
await exchange("cancel").catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
async topologyReady() {
|
||||
must(phase === "new", "Host topology acknowledgement already attempted")
|
||||
await exchange("topology-ready")
|
||||
phase = "topology"
|
||||
},
|
||||
async generate(generation: Phase, signal?: AbortSignal) {
|
||||
const expected = generation === "calibration" ? "topology" : generation === "initial" ? "capturing" : "initial"
|
||||
must(phase === expected, "Host UDP phase order differs")
|
||||
const value = await exchange("generate", signal, generation)
|
||||
phase = generation
|
||||
return { lastDatagramAt: value.lastDatagramAt! }
|
||||
},
|
||||
capture: (deadline: number, exercise: (signal: AbortSignal) => Promise<void>) => capture(deadline, exercise),
|
||||
captureStaged: (captureMs: number, exercise: Parameters<typeof capture>[1]) => {
|
||||
must(Number.isSafeInteger(captureMs) && captureMs >= 90000 && captureMs <= CHR_STAGED_TARGET_CAPTURE_MAX_MS &&
|
||||
Date.now() + captureMs < deadlineAt, "Staged host capture window unavailable")
|
||||
return capture(deadlineAt, exercise, captureMs)
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js"
|
||||
|
||||
const fields = [[1, 8], [2, 8], [4, 1], [7, 2], [8, 4], [10, 4], [11, 2], [12, 4], [14, 4]]
|
||||
function set(id: number, body: Buffer) {
|
||||
const header = Buffer.alloc(4)
|
||||
header.writeUInt16BE(id); header.writeUInt16BE(body.length + 4, 2)
|
||||
return Buffer.concat([header, body])
|
||||
}
|
||||
function packet(sets: Buffer[]) {
|
||||
const header = Buffer.alloc(16)
|
||||
header.writeUInt16BE(10)
|
||||
header.writeUInt16BE(16 + sets.reduce((total, value) => total + value.length, 0), 2)
|
||||
header.writeUInt32BE(7, 12)
|
||||
return Buffer.concat([header, ...sets]).toString("hex")
|
||||
}
|
||||
function fixture() {
|
||||
const template = Buffer.alloc(4 + fields.length * 4)
|
||||
template.writeUInt16BE(256); template.writeUInt16BE(fields.length, 2)
|
||||
fields.forEach(([field, size], index) => { template.writeUInt16BE(field, 4 + index * 4); template.writeUInt16BE(size, 6 + index * 4) })
|
||||
const data = Buffer.alloc(37)
|
||||
data.writeBigUInt64BE(9007199254740993n); data.writeBigUInt64BE(3n, 8); data[16] = 17
|
||||
data.writeUInt16BE(9000, 17); Buffer.from([192, 0, 2, 1]).copy(data, 19); data.writeUInt32BE(11, 23)
|
||||
data.writeUInt16BE(9001, 27); Buffer.from([198, 51, 100, 1]).copy(data, 29); data.writeUInt32BE(12, 33)
|
||||
const frames = [0, 1].map((exporter, index) => ({
|
||||
ordinal: index + 1, receivedMs: 1700000000000, receivedNs: "1700000000000000000", exporter, sourcePort: 2055,
|
||||
version: 10, domain: 7, payloadHex: packet([set(2, template), set(256, data)]),
|
||||
}))
|
||||
return { template, data, input: { captureSha256: "a".repeat(64), scope: { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4739 }, frames } }
|
||||
}
|
||||
|
||||
test("independent IPv4 IPFIX decoder preserves exact counters beyond JavaScript safe integer range", () => {
|
||||
const { input } = fixture()
|
||||
const result = decodeOwnedIpfix(input)
|
||||
assert.equal(result.captureSha256, input.captureSha256)
|
||||
assert.equal(result.nativeAcceptance, false)
|
||||
for (const row of result.frames) {
|
||||
assert.deepEqual(row.setIds, [2, 256])
|
||||
assert.deepEqual(row.flows, [{ src: "192.0.2.1", dst: "198.51.100.1", proto: 17, srcPort: 9000, dstPort: 9001,
|
||||
inIface: "11", outIface: "12", bytes: "9007199254740993", packets: "3" }])
|
||||
}
|
||||
})
|
||||
|
||||
test("refuses template borrowing across exporters and UDP source ports", () => {
|
||||
const { input, data } = fixture()
|
||||
input.frames[1].payloadHex = packet([set(256, data)])
|
||||
assert.throws(() => decodeOwnedIpfix(input), /Missing IPFIX template/)
|
||||
input.frames[1].exporter = 0
|
||||
input.frames[1].sourcePort = 2056
|
||||
assert.throws(() => decodeOwnedIpfix(input), /Missing IPFIX template/)
|
||||
})
|
||||
|
||||
test("refuses duplicate required fields, ambiguous padding and unsupported variable fields", () => {
|
||||
const duplicate = fixture()
|
||||
duplicate.template.writeUInt16BE(1, 8)
|
||||
duplicate.input.frames[0].payloadHex = packet([set(2, duplicate.template), set(256, duplicate.data)])
|
||||
assert.throws(() => decodeOwnedIpfix(duplicate.input), /Ambiguous/)
|
||||
const variable = fixture()
|
||||
variable.template.writeUInt16BE(65535, 6)
|
||||
variable.input.frames[0].payloadHex = packet([set(2, variable.template), set(256, variable.data)])
|
||||
assert.throws(() => decodeOwnedIpfix(variable.input), /Variable/)
|
||||
const padded = fixture()
|
||||
padded.input.frames[0].payloadHex = packet([set(2, padded.template), set(256, Buffer.concat([padded.data, Buffer.from([1])]))])
|
||||
assert.throws(() => decodeOwnedIpfix(padded.input), /padding/)
|
||||
})
|
||||
@@ -0,0 +1,135 @@
|
||||
/** Narrow independent decoder for the reviewed IPv4 RouterOS IPFIX fixture.
|
||||
* Deliberately imports no product flow parser or SQL aggregation. */
|
||||
import { createHash } from "node:crypto"
|
||||
import { open } from "node:fs/promises"
|
||||
import { isIPv4 } from "node:net"
|
||||
import { resolve } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { z } from "zod"
|
||||
|
||||
const sha = z.string().regex(/^[a-f0-9]{64}$/)
|
||||
const ip = z.string().max(15).refine(isIPv4)
|
||||
const port = z.number().int().min(1).max(65535)
|
||||
const projection = z.object({
|
||||
captureSha256: sha,
|
||||
scope: z.object({ sources: z.tuple([ip, ip]), destination: ip, port, linkType: z.enum(["RAW", "EN10MB"]).optional() }).strict(),
|
||||
frames: z.array(z.object({
|
||||
ordinal: z.number().int().positive().safe(), receivedMs: z.number().int().nonnegative().safe(),
|
||||
receivedNs: z.string().regex(/^\d{1,20}$/), exporter: z.union([z.literal(0), z.literal(1)]),
|
||||
sourcePort: port, version: z.literal(10), domain: z.number().int().min(0).max(0xffffffff),
|
||||
payloadHex: z.string().min(40).max(8192).regex(/^(?:[0-9a-f]{2})+$/),
|
||||
}).strict()).min(2).max(64),
|
||||
}).strict()
|
||||
const required = new Map([[1, [1, 2, 4, 8]], [2, [1, 2, 4, 8]], [4, [1]], [7, [2]], [8, [4]], [10, [1, 2, 4]], [11, [2]], [12, [4]], [14, [1, 2, 4]]])
|
||||
type Field = { offset: number; size: number }
|
||||
type Template = { fields: Map<number, Field>; width: number }
|
||||
function must(value: unknown, error: string): asserts value {
|
||||
if (!value) throw Error(error)
|
||||
}
|
||||
function padding(bytes: Buffer) {
|
||||
return bytes.length <= 3 && bytes.every(value => value === 0)
|
||||
}
|
||||
|
||||
export function decodeOwnedIpfix(raw: unknown) {
|
||||
const input = projection.parse(raw)
|
||||
must(new Set([...input.scope.sources, input.scope.destination]).size === 3, "Invalid exporter scope")
|
||||
const templates = new Map<string, Template>()
|
||||
let last = 0, records = 0, lastTimestamp = -1n
|
||||
const positive = new Set<number>()
|
||||
const frames = input.frames.map(frame => {
|
||||
const timestamp = BigInt(frame.receivedNs)
|
||||
must(frame.ordinal === last + 1 && timestamp >= lastTimestamp && timestamp / 1000000n === BigInt(frame.receivedMs), "Invalid wire ordinal or timestamp")
|
||||
last = frame.ordinal
|
||||
lastTimestamp = timestamp
|
||||
const data = Buffer.from(frame.payloadHex, "hex")
|
||||
must(data.readUInt16BE(0) === 10 && data.readUInt16BE(2) === data.length && data.readUInt32BE(12) === frame.domain, "IPFIX header disagrees with wire metadata")
|
||||
const session = [frame.exporter, frame.sourcePort, frame.domain].join(":")
|
||||
const flows = []
|
||||
const setIds: number[] = []
|
||||
const templateIds: number[] = []
|
||||
for (let offset = 16; offset < data.length;) {
|
||||
must(data.length - offset >= 4 && setIds.length < 32, "Invalid IPFIX set count or header")
|
||||
const id = data.readUInt16BE(offset), length = data.readUInt16BE(offset + 2)
|
||||
must(length >= 4 && offset + length <= data.length, "Invalid IPFIX set length")
|
||||
const body = data.subarray(offset + 4, offset + length)
|
||||
offset += length
|
||||
setIds.push(id)
|
||||
if (id === 2) {
|
||||
for (let at = 0; at < body.length && !padding(body.subarray(at));) {
|
||||
must(body.length - at >= 4, "Truncated IPFIX template")
|
||||
const templateId = body.readUInt16BE(at), fieldCount = body.readUInt16BE(at + 2)
|
||||
must(templateId >= 256 && fieldCount >= required.size && fieldCount <= 64, "Unsupported template or withdrawal")
|
||||
at += 4
|
||||
const fields = new Map<number, Field>()
|
||||
let width = 0
|
||||
for (let index = 0; index < fieldCount; index++) {
|
||||
must(body.length - at >= 4, "Truncated IPFIX field")
|
||||
const field = body.readUInt16BE(at), size = body.readUInt16BE(at + 2)
|
||||
at += 4
|
||||
must(size > 0 && width + size <= 4096, "Variable or oversized IPFIX field")
|
||||
if (field & 0x8000) {
|
||||
must(body.length - at >= 4, "Truncated enterprise field")
|
||||
at += 4
|
||||
} else if (required.has(field)) {
|
||||
must(!fields.has(field) && required.get(field)!.includes(size), "Ambiguous or unsupported required field")
|
||||
fields.set(field, { offset: width, size })
|
||||
}
|
||||
width += size
|
||||
}
|
||||
must(fields.size === required.size, "Required IPv4 flow fields are missing")
|
||||
templates.set(session + ":" + templateId, { fields, width })
|
||||
must(templates.size <= 32, "Template budget exceeded")
|
||||
templateIds.push(templateId)
|
||||
}
|
||||
continue
|
||||
}
|
||||
must(id >= 256, "Unsupported IPFIX control set")
|
||||
const template = templates.get(session + ":" + id)
|
||||
must(template && body.length >= template.width, "Missing IPFIX template or data")
|
||||
let at = 0
|
||||
for (; at + template.width <= body.length; at += template.width) {
|
||||
must(++records <= 200 && flows.length < 64, "Independent record budget exceeded")
|
||||
const part = (field: number) => {
|
||||
const item = template.fields.get(field)!
|
||||
return body.subarray(at + item.offset, at + item.offset + item.size)
|
||||
}
|
||||
const unsigned = (field: number) => part(field).reduce((value, byte) => value * 256n + BigInt(byte), 0n)
|
||||
const bytes = unsigned(1), packets = unsigned(2)
|
||||
must(bytes > 0n && packets > 0n && bytes <= 9223372036854775807n && packets <= 9223372036854775807n, "Counter outside positive SQL bigint range")
|
||||
flows.push({ src: [...part(8)].join("."), dst: [...part(12)].join("."), proto: Number(unsigned(4)),
|
||||
srcPort: Number(unsigned(7)), dstPort: Number(unsigned(11)), inIface: String(unsigned(10)), outIface: String(unsigned(14)),
|
||||
bytes: String(bytes), packets: String(packets) })
|
||||
}
|
||||
must(padding(body.subarray(at)), "Ambiguous IPFIX data padding")
|
||||
}
|
||||
if (flows.length) positive.add(frame.exporter)
|
||||
return { ordinal: frame.ordinal, flows, setIds, templateIds }
|
||||
})
|
||||
must(positive.size === 2, "Both exporters need independently decoded positive traffic")
|
||||
return { captureSha256: input.captureSha256, frames, source: "independent-fixed-ipv4-ipfix-v1", nativeAcceptance: false }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const [path, expectedSha, ...extra] = process.argv.slice(2)
|
||||
must(path && !extra.length && sha.safeParse(expectedSha).success, "Expected pinned wire projection")
|
||||
const handle = await open(path, "r")
|
||||
try {
|
||||
const info = await handle.stat()
|
||||
must(info.isFile() && info.size > 0 && info.size <= 512 * 1024, "Wire projection outside file bound")
|
||||
const buffer = Buffer.alloc(info.size + 1)
|
||||
let size = 0
|
||||
while (size < buffer.length) {
|
||||
const { bytesRead } = await handle.read(buffer, size, buffer.length - size, size)
|
||||
if (!bytesRead) break
|
||||
size += bytesRead
|
||||
}
|
||||
const body = buffer.subarray(0, size)
|
||||
must(size === info.size && createHash("sha256").update(body).digest("hex") === expectedSha, "Wire projection changed or digest differs")
|
||||
console.log(JSON.stringify({ wireProjectionSha256: expectedSha, ...decodeOwnedIpfix(JSON.parse(body.toString("utf8"))) }))
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch(() => { console.error("Independent IPFIX projection refused"); process.exitCode = 1 })
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { discoverChrIngressEndpoint } from "./traffic-flow-chr-ingress-discovery.js"
|
||||
|
||||
type Options = Parameters<typeof discoverChrIngressEndpoint>[0]
|
||||
function fixture() {
|
||||
const reads: string[] = [], toolReads: string[] = []
|
||||
const collectors = { enabled: true, collectorIp: "192.0.2.100", flowListenPort: 4740,
|
||||
listenerBound: true, listenerAddress: "192.0.2.100:4740" }
|
||||
let revoked = false
|
||||
const mappings = new Map([["192.0.2.10", 1], ["192.0.2.20", 2]])
|
||||
const native = (serverId: number) => {
|
||||
const identity = "owned-" + serverId, version = "7.24.5 (stable)", source = "192.0.2." + serverId * 10
|
||||
const table: Record<string, unknown> = {
|
||||
"/system/identity": [{ name: identity }],
|
||||
"/system/resource": { version },
|
||||
"/ip/address": [{ address: source + "/32", disabled: "false" }],
|
||||
"/ip/traffic-flow": [{ enabled: "true" }],
|
||||
"/ip/traffic-flow/target": [{ ".id": "*A", disabled: "false", "dst-address": "192.0.2.100",
|
||||
"src-address": serverId === 1 ? source : "0.0.0.0", port: "4740", version: "ipfix" }],
|
||||
}
|
||||
const device: Options["devices"][0] = {
|
||||
identity, version, source, serverId,
|
||||
assertPinnedTransport: async () => { if (revoked) throw Error("transport pin lost") },
|
||||
client: { async get<T>(path: string, timeout?: number, _signal?: AbortSignal, max?: number, before?: () => Promise<unknown>) {
|
||||
assert(timeout! > 0 && timeout! <= 5000)
|
||||
assert.equal(max, 65536)
|
||||
assert(before)
|
||||
await before()
|
||||
reads.push(serverId + ":" + path)
|
||||
assert(Object.hasOwn(table, path))
|
||||
return structuredClone(table[path]) as T
|
||||
} },
|
||||
}
|
||||
return { table, device }
|
||||
}
|
||||
const a = native(1), b = native(2)
|
||||
const options: Options = { destination: collectors.collectorIp, baselinePort: 4740, testPort: 4739,
|
||||
deadlineAt: Date.now() + 5000, devices: [a.device, b.device], assertCurrent: async () => {},
|
||||
callTool: async (name, input) => { assert.deepEqual(input, {}); toolReads.push(name); return collectors },
|
||||
resolveExporter: async address => mappings.get(address) ?? null,
|
||||
}
|
||||
return { options, reads, toolReads, a, b, mappings, collectors, revoke: () => { revoked = true } }
|
||||
}
|
||||
|
||||
test("discovers exact configured two-CHR baseline endpoint without claiming observed ingress", async () => {
|
||||
const f = fixture()
|
||||
const result = await discoverChrIngressEndpoint(f.options)
|
||||
assert.deepEqual(result.endpoint, { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4740 })
|
||||
assert.deepEqual(result.observations.map(row => row.sourceConfiguredExplicitly), [true, false])
|
||||
assert.equal(result.ingressVerified, false)
|
||||
assert.equal(f.reads.length, 10)
|
||||
assert.deepEqual(f.toolReads, ["mm_netflow_settings_get", "mm_netflow_settings_get"])
|
||||
})
|
||||
|
||||
test("refuses absent exact target, competing test port, source drift and attribution mismatch", async () => {
|
||||
for (const [mutate, reason] of [
|
||||
[(f: ReturnType<typeof fixture>) => { f.b.table["/ip/traffic-flow/target"] = [] }, /baseline IPFIX target/],
|
||||
[(f: ReturnType<typeof fixture>) => { (f.a.table["/ip/traffic-flow/target"] as object[]).push({ ".id": "*B", "dst-address": "192.0.2.100", port: "4739" }) }, /test endpoint/],
|
||||
[(f: ReturnType<typeof fixture>) => { f.b.table["/ip/address"] = [{ address: "192.0.2.99/32", disabled: false }] }, /source is not assigned/],
|
||||
[(f: ReturnType<typeof fixture>) => { f.mappings.set("192.0.2.20", 1) }, /attribution differs/],
|
||||
] as const) {
|
||||
const f = fixture()
|
||||
mutate(f)
|
||||
await assert.rejects(discoverChrIngressEndpoint(f.options), reason)
|
||||
}
|
||||
})
|
||||
|
||||
test("stops on wrong collector bind, lost transport pin and changed CHR identity", async () => {
|
||||
const wrong = fixture(); wrong.collectors.listenerAddress = "0.0.0.0:4740"
|
||||
await assert.rejects(discoverChrIngressEndpoint(wrong.options), /collector endpoint/)
|
||||
assert.deepEqual(wrong.reads, [])
|
||||
const revoked = fixture(); revoked.revoke()
|
||||
await assert.rejects(discoverChrIngressEndpoint(revoked.options), /transport pin lost/)
|
||||
assert.deepEqual(revoked.reads, [])
|
||||
const changed = fixture(); changed.a.table["/system/identity"] = [{ name: "foreign" }]
|
||||
await assert.rejects(discoverChrIngressEndpoint(changed.options), /identity or version/)
|
||||
assert.deepEqual(changed.reads, ["1:/system/identity", "1:/system/resource"])
|
||||
})
|
||||
@@ -0,0 +1,123 @@
|
||||
/** Exact read-only prerequisite for a passive two-CHR ingress capture. */
|
||||
import { isIPv4 } from "node:net"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import type { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { captureOwnedChrPackets } from "./traffic-flow-chr-capture-process.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
type Device = {
|
||||
client: Pick<MikrotikClient, "get">
|
||||
identity: string; version: string; source: string; serverId: number
|
||||
assertPinnedTransport: () => Promise<void>
|
||||
}
|
||||
type Options = {
|
||||
destination: string; baselinePort: number; testPort: number
|
||||
devices: readonly [Device, Device]
|
||||
callTool: (name: "mm_netflow_settings_get", input: Record<string, never>) => Promise<unknown>
|
||||
resolveExporter: (address: string) => Promise<number | null>
|
||||
assertCurrent: () => Promise<void>
|
||||
deadlineAt: number
|
||||
signal?: AbortSignal
|
||||
}
|
||||
const targetPath = "/ip/traffic-flow/target"
|
||||
const trueValue = (value: unknown) => value === true || value === "true" || value === "yes"
|
||||
const falseValue = (value: unknown) => value === false || value === "false" || value === "no"
|
||||
function must(value: unknown, reason: string): asserts value {
|
||||
if (!value) throw Error(reason)
|
||||
}
|
||||
function rows(value: unknown, limit: number): Row[] {
|
||||
const result = Array.isArray(value) ? value : [value]
|
||||
must(result.length <= limit && result.every(row => row && typeof row === "object" && !Array.isArray(row)), "Invalid bounded native observation")
|
||||
return result as Row[]
|
||||
}
|
||||
function singleton(value: unknown) {
|
||||
const result = rows(value, 1)
|
||||
must(result.length === 1, "Missing native singleton observation")
|
||||
return result[0]
|
||||
}
|
||||
|
||||
export async function discoverChrIngressEndpoint(options: Options) {
|
||||
const { destination, baselinePort, testPort, deadlineAt, signal } = options
|
||||
const devices = options.devices.map(device => ({ ...device }))
|
||||
const port = (value: number) => Number.isInteger(value) && value >= 1 && value <= 65535
|
||||
must(devices.length === 2 && new Set([...devices.map(device => device.source), destination]).size === 3 &&
|
||||
[...devices.map(device => device.source), destination].every(isIPv4) && new Set(devices.map(device => device.serverId)).size === 2 &&
|
||||
new Set(devices.map(device => device.identity)).size === 2 &&
|
||||
devices.every(device => Number.isSafeInteger(device.serverId) && device.serverId > 0 && device.identity.length > 0 &&
|
||||
device.identity.length <= 128 && device.version.length > 0 && device.version.length <= 128) &&
|
||||
port(baselinePort) && port(testPort) && baselinePort !== testPort &&
|
||||
Number.isSafeInteger(deadlineAt) && deadlineAt > Date.now() && deadlineAt - Date.now() <= 90000, "Invalid pinned ingress scope")
|
||||
const live = () => {
|
||||
signal?.throwIfAborted()
|
||||
must(Date.now() < deadlineAt, "Ingress discovery deadline ended")
|
||||
}
|
||||
const check = async () => { live(); await options.assertCurrent(); live() }
|
||||
const collector = async () => {
|
||||
await check()
|
||||
const value = singleton(await options.callTool("mm_netflow_settings_get", {}))
|
||||
await check()
|
||||
must(value.enabled === true && value.collectorIp === destination && value.flowListenPort === baselinePort &&
|
||||
value.listenerBound === true && value.listenerAddress === destination + ":" + baselinePort, "Current baseline collector endpoint is not confirmed")
|
||||
}
|
||||
await collector()
|
||||
const observations = []
|
||||
for (const device of devices) {
|
||||
const fence = async () => { await check(); await device.assertPinnedTransport(); await check() }
|
||||
const get = async (path: string) => {
|
||||
await fence()
|
||||
const value = await device.client.get<unknown>(path, Math.min(5000, deadlineAt - Date.now()), signal, 64 * 1024, fence)
|
||||
await fence()
|
||||
return value
|
||||
}
|
||||
const identity = singleton(await get("/system/identity")), resource = singleton(await get("/system/resource"))
|
||||
must(identity.name === device.identity && resource.version === device.version, "Pinned CHR identity or version differs")
|
||||
const addresses = rows(await get("/ip/address"), 256)
|
||||
must(addresses.filter(row => typeof row.address === "string" && row.address.split("/")[0] === device.source && falseValue(row.disabled)).length === 1,
|
||||
"Pinned exporter source is not assigned exactly once")
|
||||
await fence()
|
||||
must(await options.resolveExporter(device.source) === device.serverId, "Collector exporter attribution differs")
|
||||
await fence()
|
||||
const flow = singleton(await get("/ip/traffic-flow"))
|
||||
must(trueValue(flow.enabled), "Traffic Flow is disabled or unconfirmed")
|
||||
const targets = rows(await get(targetPath), 64)
|
||||
must(targets.every(row => typeof row[".id"] === "string" && /^\*[a-f0-9]{1,16}$/i.test(row[".id"])) &&
|
||||
new Set(targets.map(row => row[".id"])).size === targets.length, "Target identity is malformed or ambiguous")
|
||||
const matches = targets.filter(row => row["dst-address"] === destination && String(row.port) === String(baselinePort))
|
||||
must(matches.length === 1 && falseValue(matches[0].disabled) && matches[0].version === "ipfix",
|
||||
"Exact enabled baseline IPFIX target is missing or ambiguous")
|
||||
must(!targets.some(row => row["dst-address"] === destination && String(row.port) === String(testPort)),
|
||||
"Proposed test endpoint is already configured")
|
||||
const source = matches[0]["src-address"]
|
||||
must(source === device.source || source === "0.0.0.0", "Baseline target source conflicts with the pinned address")
|
||||
observations.push({ serverId: device.serverId, identity: device.identity, targetId: matches[0][".id"],
|
||||
sourceAddress: device.source, sourceConfiguredExplicitly: source === device.source })
|
||||
}
|
||||
await collector()
|
||||
return {
|
||||
endpoint: { sources: [devices[0].source, devices[1].source] as const, destination, port: baselinePort },
|
||||
testPort, observations, passiveCaptureCandidate: true,
|
||||
ingressVerified: false, actualNativeAcceptance: false,
|
||||
}
|
||||
}
|
||||
|
||||
/** Passive capture entry for the pinned Linux operator. The exercise only waits. */
|
||||
export async function observeChrBaselineIngress(options: Options & { interfaceName: string; sampleMs: number }) {
|
||||
must(process.platform === "linux" && Number.isInteger(options.sampleMs) && options.sampleMs >= 1000 && options.sampleMs <= 30000,
|
||||
"Invalid passive capture platform or window")
|
||||
const discovered = await discoverChrIngressEndpoint(options)
|
||||
must(Date.now() + options.sampleMs < options.deadlineAt, "Passive capture window exceeds the reviewed deadline")
|
||||
const capture = await captureOwnedChrPackets({ ...discovered.endpoint, interfaceName: options.interfaceName }, {
|
||||
deadlineAt: options.deadlineAt,
|
||||
assertCurrent: async () => {
|
||||
options.signal?.throwIfAborted()
|
||||
await options.assertCurrent()
|
||||
for (const device of options.devices) await device.assertPinnedTransport()
|
||||
options.signal?.throwIfAborted()
|
||||
},
|
||||
exercise: async signal => {
|
||||
await delay(options.sampleMs, undefined, { signal: options.signal ? AbortSignal.any([signal, options.signal]) : signal })
|
||||
},
|
||||
})
|
||||
return { discovery: discovered, ingress: { bytes: capture.bytes, sha256: capture.sha256, existingPort: discovered.endpoint.port },
|
||||
counts: capture.counts, nativeAcceptance: false }
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { createChrInterfaceCounterReader } from "./traffic-flow-chr-interface-counters.js"
|
||||
|
||||
function fixture() {
|
||||
const interfaces = [{ id: "*A", name: "nfa012bcdeea", ifIndex: "12" },
|
||||
{ id: "*B", name: "nfa012bcdema", ifIndex: "13" }] as const
|
||||
const calls: string[] = []
|
||||
const catalog: Record<string, Record<string, unknown>> = {
|
||||
"/system/identity": { name: "chr-a" },
|
||||
"/system/resource": { version: "7.24.5 (stable)" },
|
||||
}
|
||||
for (const pin of interfaces) catalog["/interface/" + pin.id] = {
|
||||
".id": pin.id, name: pin.name, type: "wg", ifindex: pin.ifIndex, running: "true", disabled: "false",
|
||||
"rx-packet": "10", "tx-packet": "11", "fp-rx-packet": "0", "fp-tx-packet": "0",
|
||||
"fp-rx-byte": "0", "fp-tx-byte": "0", opaque: "must stay private",
|
||||
}
|
||||
const config = { identity: "chr-a", version: "7.24.5 (stable)", interfaces, assertCurrent: async () => {},
|
||||
client: { async get<T>(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise<unknown>) {
|
||||
assert.ok(timeout && timeout <= 1500)
|
||||
assert.equal(bytes, 16384)
|
||||
await fence?.()
|
||||
calls.push(path)
|
||||
assert.ok(Object.hasOwn(catalog, path))
|
||||
return structuredClone(catalog[path]) as T
|
||||
} },
|
||||
}
|
||||
return { config, catalog, calls }
|
||||
}
|
||||
|
||||
test("reads only the exact native IDs with bounded identity checks and emits selected counters", async () => {
|
||||
const { config, calls } = fixture()
|
||||
const read = createChrInterfaceCounterReader(config)
|
||||
const result = await read()
|
||||
assert.deepEqual(calls, ["/system/identity", "/system/resource", "/interface/*A", "/interface/*B", "/system/identity", "/system/resource"])
|
||||
assert.ok(result.startedAt <= result.observedAt)
|
||||
assert.deepEqual(result.rows.map(row => row.ifindex), ["12", "13"])
|
||||
assert.ok(result.rows.every(row => row["fp-rx-packet"] === "0" && !Object.hasOwn(row, "opaque")))
|
||||
})
|
||||
|
||||
test("refuses missing and overflowing counters, changed identity and index guesses", async () => {
|
||||
for (const change of [{ "fp-rx-packet": undefined }, { "fp-rx-byte": "18446744073709551616" },
|
||||
{ ifindex: undefined }, { ifindex: "10" }, { ".id": "*F" }]) {
|
||||
const { config, catalog } = fixture()
|
||||
Object.assign(catalog["/interface/*A"], change)
|
||||
await assert.rejects(createChrInterfaceCounterReader(config)(), /counter unavailable|identity or state differs/)
|
||||
}
|
||||
const stale = fixture()
|
||||
stale.config.assertCurrent = async () => { if (stale.calls.length >= 4) throw Error("expired") }
|
||||
await assert.rejects(createChrInterfaceCounterReader(stale.config)(), /expired/)
|
||||
})
|
||||
@@ -0,0 +1,70 @@
|
||||
import { performance } from "node:perf_hooks"
|
||||
import { encodeRosId, type MikrotikClient } from "../services/mikrotik.js"
|
||||
import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
type InterfacePin = { id: string; name: string; ifIndex: string }
|
||||
type Options = {
|
||||
client: Pick<MikrotikClient, "get">
|
||||
identity: string
|
||||
version: string
|
||||
interfaces: readonly [InterfacePin, InterfacePin]
|
||||
assertCurrent: () => Promise<void>
|
||||
}
|
||||
|
||||
export const ownedWgFastPathFields = ["fp-rx-byte", "fp-tx-byte", "fp-rx-packet", "fp-tx-packet"] as const
|
||||
export const ownedWgPacketFields = ["rx-packet", "tx-packet"] as const
|
||||
const fields = [...ownedWgFastPathFields, ...ownedWgPacketFields]
|
||||
const validId = (id: string) => /^\*[a-f0-9]{1,16}$/i.test(id)
|
||||
const index = (value: unknown) => typeof value === "string" && /^[1-9][0-9]{0,9}$/.test(value) && Number(value) <= 0xffffffff
|
||||
const observedUint = (value: unknown) => typeof value === "string" && /^(0|[1-9][0-9]{0,19})$/.test(value) &&
|
||||
BigInt(value) <= 0xffffffffffffffffn
|
||||
const deny = (reason: string): never => { throw Error("RouterOS owned counter read refused: " + reason) }
|
||||
|
||||
/** Uses only fixed read operations. Missing counter/index fields refuse the
|
||||
* profile rather than substituting zero or guessing the wire ifIndex. */
|
||||
export function createChrInterfaceCounterReader(options: Options) {
|
||||
const { client, assertCurrent, identity, version } = options
|
||||
const interfaces = structuredClone(options.interfaces)
|
||||
if (!identity || !version || interfaces.length !== 2 || interfaces.some(pin =>
|
||||
!validId(pin.id) || !/^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.name) || !index(pin.ifIndex)) ||
|
||||
interfaces[0].id === interfaces[1].id || interfaces[0].ifIndex === interfaces[1].ifIndex ||
|
||||
interfaces[0].name.slice(0, -2) !== interfaces[1].name.slice(0, -2) ||
|
||||
!["ea,ma", "eb,mb"].includes(interfaces.map(pin => pin.name.slice(-2)).sort().join(","))) deny("invalid interface pins")
|
||||
return async () => {
|
||||
const startedAt = Date.now(), deadline = performance.now() + 10000
|
||||
const fence = async () => {
|
||||
if (performance.now() >= deadline) deny("observation expired")
|
||||
await assertCurrent()
|
||||
if (performance.now() >= deadline) deny("observation expired")
|
||||
}
|
||||
const read = async (path: string) => {
|
||||
await fence()
|
||||
const result = await client.get<unknown>(path, Math.min(1500, Math.ceil(deadline - performance.now())), undefined, 16384, fence)
|
||||
await fence()
|
||||
const rows = Array.isArray(result) ? result : [result]
|
||||
if (rows.length !== 1 || !rows[0] || typeof rows[0] !== "object" || Array.isArray(rows[0])) deny("incomplete response")
|
||||
return rows[0] as Row
|
||||
}
|
||||
const verifyIdentity = async () => {
|
||||
const device = await read("/system/identity")
|
||||
const resource = await read("/system/resource")
|
||||
if (device.name !== identity || resource.version !== version) deny("device identity changed")
|
||||
}
|
||||
await verifyIdentity()
|
||||
const rows: Row[] = []
|
||||
for (const pin of interfaces) {
|
||||
const row = await read("/interface/" + encodeRosId(pin.id))
|
||||
if (row[".id"] !== pin.id || row.name !== pin.name || !isChrWireGuardInterfaceType(row.type) || row.ifindex !== pin.ifIndex ||
|
||||
![true, "true", "yes"].includes(row.running as string | boolean) ||
|
||||
![false, "false", "no"].includes(row.disabled as string | boolean)) deny("interface identity or state differs")
|
||||
if (!fields.every(field => observedUint(row[field]))) deny("required counter unavailable")
|
||||
rows.push({ ".id": pin.id, name: pin.name, type: "wg", ifindex: pin.ifIndex,
|
||||
...Object.fromEntries(fields.map(field => [field, row[field]])) })
|
||||
}
|
||||
await verifyIdentity()
|
||||
const observedAt = Date.now()
|
||||
if (observedAt < startedAt || observedAt - startedAt > 10000) deny("observation wall clock changed")
|
||||
return { startedAt, observedAt, rows }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { parseChrLinuxUdpTable } from "./traffic-flow-chr-linux-udp-owner.js"
|
||||
|
||||
const header = "sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode"
|
||||
const row = (address: string, port: string, inode: string) =>
|
||||
` 14: ${address}:${port} 00000000:0000 07 00000000:00000000 00:00000000 00000000 1000 0 ${inode} 2 0000000000000000 0`
|
||||
|
||||
test("parses exact kernel IPv4 socket inode and leaves unrelated UDP ports out", () => {
|
||||
const text = [header, row("03F212C6", "C355", "42"), row("00000000", "C356", "43")].join("\n")
|
||||
assert.deepEqual(parseChrLinuxUdpTable(text, 4, 50005), [{ address: "198.18.242.3", inode: "42" }])
|
||||
assert.deepEqual(parseChrLinuxUdpTable(text, 4, 50007), [])
|
||||
})
|
||||
|
||||
test("fails closed for malformed rows, ownerless sockets and IPv6 conflicts", () => {
|
||||
assert.throws(() => parseChrLinuxUdpTable("header", 4, 50005), /kernel table/)
|
||||
assert.throws(() => parseChrLinuxUdpTable(header + "\n14: malformed", 4, 50005), /kernel socket row/)
|
||||
assert.throws(() => parseChrLinuxUdpTable(header + "\n" + row("00000000", "C355", "0"), 4, 50005), /unattributed/)
|
||||
assert.throws(() => parseChrLinuxUdpTable(header + "\n" + row("0".repeat(32), "C355", "42"), 6, 50005), /ambiguous IPv6/)
|
||||
})
|
||||
@@ -0,0 +1,86 @@
|
||||
import { open, opendir, readlink, stat } from "node:fs/promises"
|
||||
|
||||
const MAX_TABLE_BYTES = 256 * 1024
|
||||
const MAX_ROWS = 2048
|
||||
const deny = (reason: string): never => { throw Error("Collector UDP ownership refused: " + reason) }
|
||||
type UdpOwner = { pid: number; address: string; port: number }
|
||||
|
||||
export function parseChrLinuxUdpTable(text: string, family: 4 | 6, port: number) {
|
||||
const lines = text.trim().split("\n")
|
||||
if (Buffer.byteLength(text) > MAX_TABLE_BYTES || lines.length > MAX_ROWS ||
|
||||
!/^sl\s+local_address\s+(?:rem_address|remote_address)\s+st\s/.test(lines[0])) deny("unrecognized or oversized kernel table")
|
||||
const width = family === 4 ? 8 : 32
|
||||
const entries: { address: string; inode: string }[] = []
|
||||
for (const line of lines.slice(1)) {
|
||||
const values = line.trim().split(/\s+/)
|
||||
const socket = values[1]?.split(":")
|
||||
if (!/^\d+:$/.test(values[0]) || socket?.length !== 2 || socket[0].length !== width ||
|
||||
!/^[0-9A-F]+$/.test(socket[0]) || !/^[0-9A-F]{4}$/.test(socket[1]) ||
|
||||
!/^(0|[1-9][0-9]{0,19})$/.test(values[9] ?? "")) deny("incomplete kernel socket row")
|
||||
if (Number.parseInt(socket[1], 16) !== port) continue
|
||||
if (family !== 4 || values[9] === "0") deny("ambiguous IPv6 or unattributed socket")
|
||||
const address = socket[0].match(/../g)!.reverse().map(value => Number.parseInt(value, 16)).join(".")
|
||||
entries.push({ address, inode: values[9] })
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
async function boundedRead(path: "/proc/self/net/udp" | "/proc/self/net/udp6") {
|
||||
const file = await open(path, "r")
|
||||
try {
|
||||
const bytes = Buffer.alloc(MAX_TABLE_BYTES + 1)
|
||||
let length = 0
|
||||
while (length < bytes.length) {
|
||||
const { bytesRead } = await file.read(bytes, length, bytes.length - length, null)
|
||||
if (bytesRead === 0) break
|
||||
length += bytesRead
|
||||
}
|
||||
if (length > MAX_TABLE_BYTES) deny("kernel table exceeds budget")
|
||||
return new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length))
|
||||
} finally {
|
||||
await file.close()
|
||||
}
|
||||
}
|
||||
|
||||
export async function readChrLinuxNamespaceIdentity() {
|
||||
if (process.platform !== "linux") deny("Linux required")
|
||||
const identity = await stat("/proc/self/ns/net", { bigint: true })
|
||||
return { device: String(identity.dev), inode: String(identity.ino) }
|
||||
}
|
||||
|
||||
/** Self-process observation in the collector namespace, with no process
|
||||
* execution, pathname overrides or inference from a missing socket owner. */
|
||||
export async function readChrLinuxUdpOwner(options: {
|
||||
namespace: { device: string; inode: string }; port: number; address: string
|
||||
}): Promise<UdpOwner[]> {
|
||||
const { namespace, port, address } = options
|
||||
const current = async () => {
|
||||
const identity = await readChrLinuxNamespaceIdentity()
|
||||
if (identity.device !== namespace.device || identity.inode !== namespace.inode) deny("network namespace changed")
|
||||
}
|
||||
await current()
|
||||
const scan = async () => {
|
||||
const ipv4 = parseChrLinuxUdpTable(await boundedRead("/proc/self/net/udp"), 4, port)
|
||||
parseChrLinuxUdpTable(await boundedRead("/proc/self/net/udp6"), 6, port)
|
||||
if (ipv4.length > 1 || ipv4.some(row => row.address !== address)) deny("competing or wildcard UDP bind")
|
||||
return ipv4[0]
|
||||
}
|
||||
const candidate = await scan()
|
||||
if (candidate) {
|
||||
let matches = 0, visited = 0
|
||||
const directory = await opendir("/proc/self/fd")
|
||||
for await (const entry of directory) {
|
||||
if (++visited > 2048 || !/^\d+$/.test(entry.name)) deny("process descriptor inventory exceeds budget")
|
||||
try {
|
||||
if (await readlink("/proc/self/fd/" + entry.name) === `socket:[${candidate.inode}]`) matches++
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error
|
||||
}
|
||||
}
|
||||
if (matches !== 1) deny("UDP socket is not exclusively identified in the collector process")
|
||||
}
|
||||
const recheck = await scan()
|
||||
if (candidate?.inode !== recheck?.inode) deny("UDP socket changed during observation")
|
||||
await current()
|
||||
return candidate ? [{ pid: process.pid, address, port }] : []
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { wireguardInterfaceDependencyPaths } from "../services/routeros-native-paths.js"
|
||||
|
||||
const readPaths = new Set(["/system/identity", "/system/resource", "/interface", "/interface/wireguard",
|
||||
"/interface/wireguard/peers", "/interface/bridge", "/interface/list/member", "/ip/address", "/ip/route",
|
||||
"/ipv6/address", "/ipv6/route", "/routing/route", "/routing/rule", "/routing/table", "/ip/vrf", "/ip/settings",
|
||||
"/ip/firewall/filter", "/ip/firewall/raw", "/ip/firewall/mangle", "/ip/firewall/nat", "/ipv6/firewall/filter",
|
||||
"/ip/firewall/connection/tracking", "/ip/firewall/connection", "/ip/traffic-flow", "/ip/traffic-flow/ipfix",
|
||||
"/ip/traffic-flow/target", ...wireguardInterfaceDependencyPaths])
|
||||
const rowMenus = ["/interface/wireguard", "/interface/wireguard/peers", "/ip/address", "/ip/route", "/ip/traffic-flow/target"]
|
||||
export const targetMenu = "/ip/traffic-flow/target"
|
||||
const singletonMenus = ["/ip/traffic-flow", "/ip/traffic-flow/ipfix"]
|
||||
export const nativeId = /^\*[0-9a-f]{1,16}$/i
|
||||
const deny = () => { throw Error("Owned CHR namespace relay refused") }
|
||||
|
||||
export function chrNamespaceRelayOperation(method: string, rest: string) {
|
||||
if (!rest.startsWith("/rest/")) return deny()
|
||||
const path = rest.slice(5)
|
||||
const last = path.lastIndexOf("/")
|
||||
const menu = path.slice(0, last), token = path.slice(last + 1)
|
||||
if (method === "GET" && (readPaths.has(path) || menu === "/interface" && nativeId.test(token))) return { path, write: false }
|
||||
if (method === "PUT" && rowMenus.includes(path) ||
|
||||
method === "DELETE" && rowMenus.includes(menu) && nativeId.test(token) ||
|
||||
method === "PATCH" && (singletonMenus.includes(menu) || menu === targetMenu) && nativeId.test(token) ||
|
||||
method === "POST" && singletonMenus.includes(menu) && token === "set") return { path, write: true }
|
||||
return deny()
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { execFile } from "node:child_process"
|
||||
import { createHash } from "node:crypto"
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises"
|
||||
import { join } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { promisify } from "node:util"
|
||||
import { test } from "node:test"
|
||||
import { replayRecordedChrEvidence } from "./traffic-flow-chr-offline-replay.js"
|
||||
import { verifyChrOwnedWgEvidence } from "./traffic-flow-chr-owned-wg-evidence.js"
|
||||
|
||||
const digest = (value: Buffer | string) => createHash("sha256").update(value).digest("hex")
|
||||
function fixture(ethernet = false, earlyTemplates = false) {
|
||||
const link = ethernet ? 14 : 0
|
||||
const seconds = 1700000010
|
||||
const bucket = Math.floor(seconds / 60) * 60000
|
||||
const from = new Date(bucket).toISOString(), until = new Date(bucket + 60000).toISOString()
|
||||
const scope = { sources: ["192.0.2.10", "192.0.2.20"], destination: "192.0.2.100", port: 4739, ...(ethernet ? { linkType: "EN10MB" as const } : {}) }
|
||||
const fields = [[1, 4], [2, 4], [4, 1], [7, 2], [8, 4], [10, 4], [11, 2], [12, 4], [14, 4]]
|
||||
const template = Buffer.alloc(8 + fields.length * 4)
|
||||
template.writeUInt16BE(2); template.writeUInt16BE(template.length, 2)
|
||||
template.writeUInt16BE(256, 4); template.writeUInt16BE(fields.length, 6)
|
||||
fields.forEach(([id, width], index) => { template.writeUInt16BE(id, 8 + index * 4); template.writeUInt16BE(width, 10 + index * 4) })
|
||||
const data = Buffer.alloc(33)
|
||||
data.writeUInt16BE(256); data.writeUInt16BE(data.length, 2)
|
||||
data.writeUInt32BE(1200, 4); data.writeUInt32BE(3, 8); data[12] = 17
|
||||
data.writeUInt16BE(9000, 13); Buffer.from([192, 0, 2, 1]).copy(data, 15); data.writeUInt32BE(11, 19)
|
||||
data.writeUInt16BE(9001, 23); Buffer.from([198, 51, 100, 1]).copy(data, 25); data.writeUInt32BE(12, 29)
|
||||
const ipfix = Buffer.alloc(16)
|
||||
ipfix.writeUInt16BE(10); ipfix.writeUInt16BE(ipfix.length + template.length + data.length, 2); ipfix.writeUInt32BE(7, 12)
|
||||
const header = Buffer.alloc(24)
|
||||
header.writeUInt32LE(0xa1b2c3d4); header.writeUInt16LE(2, 4); header.writeUInt16LE(4, 6)
|
||||
header.writeUInt32LE(4124 + link, 16); header.writeUInt32LE(ethernet ? 1 : 101, 20)
|
||||
const frames = [10, 20, 10, 20].map((octet, index) => ({
|
||||
octet, at: seconds, micros: index * 1000, sequence: Math.floor(index / 2),
|
||||
sets: earlyTemplates && index < 2 ? [data] : [template, data],
|
||||
}))
|
||||
if (earlyTemplates) frames.unshift(...[10, 20].map(octet =>
|
||||
({ octet, at: seconds - 1, micros: 0, sequence: 0, sets: [template] })))
|
||||
const packets = frames.map(({ octet, at, micros, sequence, sets }) => {
|
||||
const payload = Buffer.concat([ipfix, ...sets])
|
||||
payload.writeUInt16BE(payload.length, 2)
|
||||
const packet = Buffer.alloc(44 + link + payload.length)
|
||||
packet.writeUInt32LE(at, 0); packet.writeUInt32LE(micros, 4)
|
||||
packet.writeUInt32LE(packet.length - 16, 8); packet.writeUInt32LE(packet.length - 16, 12)
|
||||
if (ethernet) packet.writeUInt16BE(0x0800, 28)
|
||||
const ip = packet.subarray(16 + link)
|
||||
ip[0] = 0x45; ip.writeUInt16BE(ip.length, 2); ip[9] = 17
|
||||
Buffer.from([192, 0, 2, octet, 192, 0, 2, 100]).copy(ip, 12)
|
||||
ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(scope.port, 22); ip.writeUInt16BE(ip.length - 20, 24)
|
||||
payload.copy(ip, 28)
|
||||
ip.writeUInt32BE(seconds, 32)
|
||||
ip.writeUInt32BE(sequence, 36)
|
||||
return packet
|
||||
})
|
||||
const pcap = Buffer.concat([header, ...packets])
|
||||
const raw = [1, 2].map(serverId => ({ serverId, bucketAt: from, src: "192.0.2.1", dst: "198.51.100.1",
|
||||
proto: 17, srcPort: 9000, dstPort: 9001, inIface: "11", outIface: "12", bytes: "2400", packets: "6" }))
|
||||
const minute = raw.map(({ serverId, bucketAt, bytes, packets }) => ({ serverId, bucketAt, bytes, packets }))
|
||||
const evidence = {
|
||||
captureSha256: digest(pcap), complete: true, dropped: 0, scope, serverIds: [1, 2], from, until, minutes: 2,
|
||||
interfaceLabels: [1, 2].map(serverId => ({ serverId, index: "11", label: "ether-" + serverId })),
|
||||
before: { raw, minute }, after: structuredClone({ raw, minute }),
|
||||
talkers: raw.map(row => ({ serverId: row.serverId, result: {
|
||||
serverIds: [row.serverId], from: new Date(bucket - 60000).toISOString(), to: until,
|
||||
topConversationsLimit: 20, total: 1, totalConversations: 1, observedExporters: 1, retainedRowsBytes: 2400, nextOffset: null,
|
||||
coverageVerified: false, rowSumIsNetworkTotal: false, deduplicatedAcrossInterfaces: false, forwardingVerified: false,
|
||||
items: [{ ...row, serverId: String(row.serverId), inIface: "ether-" + row.serverId, inIfaceIndex: "11", bytes: 2400, packets: 6, bps: 160 }],
|
||||
} })),
|
||||
}
|
||||
return { pcap, evidence }
|
||||
}
|
||||
|
||||
test("owned WG evidence requires exact IPFIX tuples and unchanged FastPath counters", () => {
|
||||
const { pcap, evidence } = fixture()
|
||||
const makeExporter = (roles: [string, string]) => {
|
||||
const interfaces = roles.map((role, index) => ({ id: "*" + (index + 1), name: "nfa012bcde" + role,
|
||||
ifIndex: String(index + 11) })) as [{ id: string; name: string; ifIndex: string }, { id: string; name: string; ifIndex: string }]
|
||||
const snapshot = (observedAt: number, packetCount: string) => ({ startedAt: observedAt, observedAt, rows: interfaces.map(item => ({
|
||||
".id": item.id, name: item.name, type: "wg", ifindex: item.ifIndex, "rx-packet": packetCount, "tx-packet": packetCount,
|
||||
"fp-rx-byte": "0", "fp-tx-byte": "0", "fp-rx-packet": "0", "fp-tx-packet": "0",
|
||||
})) })
|
||||
return { interfaces, before: snapshot(1700000010000, "0"), after: snapshot(1700000011000, "6") }
|
||||
}
|
||||
const input = { bytes: pcap, sha256: evidence.captureSha256, complete: true, dropped: 0,
|
||||
scope: { ...evidence.scope, sources: [evidence.scope.sources[0], evidence.scope.sources[1]] as [string, string] },
|
||||
exporters: [makeExporter(["ea", "ma"]), makeExporter(["mb", "eb"])] as const,
|
||||
expected: ([0, 1] as const).map(exporter => ({ exporter, src: "192.0.2.1", dst: "198.51.100.1",
|
||||
srcPort: 9000, dstPort: 9001, inIface: "11", outIface: "12", packets: 6 })),
|
||||
}
|
||||
assert.equal(verifyChrOwnedWgEvidence(input).nativeAcceptance, false)
|
||||
const initialAndRefreshed = input.expected.flatMap(row => (["initial", "refreshed"] as const)
|
||||
.map(phase => ({ ...row, phase, packets: 3 })))
|
||||
assert.equal(verifyChrOwnedWgEvidence({ ...input, expected: initialAndRefreshed, requireRefresh: true }).exactIpfixMatched, true)
|
||||
const staged = fixture(false, true)
|
||||
const stagedInput = { ...input, bytes: staged.pcap, sha256: staged.evidence.captureSha256,
|
||||
expected: initialAndRefreshed, requireRefresh: true }
|
||||
assert.throws(() => verifyChrOwnedWgEvidence(stagedInput), /counter window/)
|
||||
assert.equal(verifyChrOwnedWgEvidence({ ...stagedInput, counterWindow: "traffic" }).exactIpfixMatched, true)
|
||||
input.exporters[0].before.observedAt++
|
||||
assert.throws(() => verifyChrOwnedWgEvidence({ ...stagedInput, counterWindow: "traffic" }), /counter window/)
|
||||
input.exporters[0].before.observedAt--
|
||||
assert.throws(() => verifyChrOwnedWgEvidence({ ...input, requireRefresh: true }), /initial and refreshed/)
|
||||
assert.throws(() => verifyChrOwnedWgEvidence({ ...input, expected: input.expected.map(row => ({ ...row, packets: 5 })) }), /totals/)
|
||||
input.exporters[0].after.rows[0]["fp-rx-packet"] = "1"
|
||||
assert.throws(() => verifyChrOwnedWgEvidence(input), /FastPath/)
|
||||
})
|
||||
|
||||
test("offline CLI joins original multi-set IPFIX pcap, product parser and recorded SQL/MCP evidence", async () => {
|
||||
const { pcap, evidence } = fixture()
|
||||
const dir = await mkdtemp(join(fileURLToPath(new URL("../../../tmp/", import.meta.url)), "mm-chr-offline-"))
|
||||
try {
|
||||
const packetPath = join(dir, "owned.pcap"), evidencePath = join(dir, "owned.json"), body = JSON.stringify(evidence)
|
||||
await Promise.all([writeFile(packetPath, pcap), writeFile(evidencePath, body)])
|
||||
const { stdout } = await promisify(execFile)(process.execPath, ["--import", "tsx",
|
||||
fileURLToPath(new URL("./traffic-flow-chr-offline-replay.ts", import.meta.url)), packetPath, digest(pcap), evidencePath, digest(body)],
|
||||
{ timeout: 15000, windowsHide: true })
|
||||
const result = JSON.parse(stdout)
|
||||
assert.equal(result.recordedEvidenceMatched, true)
|
||||
assert.equal(result.replay.frames, 4)
|
||||
assert.equal(result.reconciliation.rawRows, 2)
|
||||
assert.equal(result.actualNativeAcceptance, false)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
test("refuses a mixed capture pin or drift in post-MCP SQL evidence", async () => {
|
||||
const { pcap, evidence } = fixture()
|
||||
await assert.rejects(replayRecordedChrEvidence(pcap, "0".repeat(64), evidence), /Capture SHA/)
|
||||
evidence.after.minute[0].packets = "5"
|
||||
await assert.rejects(replayRecordedChrEvidence(pcap, digest(pcap), evidence), /persisted SQL/)
|
||||
})
|
||||
|
||||
test("offline replay preserves an explicit Ethernet capture scope across both decoders", async () => {
|
||||
const { pcap, evidence } = fixture(true)
|
||||
const result = await replayRecordedChrEvidence(pcap, digest(pcap), evidence)
|
||||
assert.equal(result.recordedEvidenceMatched, true)
|
||||
assert.equal(result.replay.frames, 4)
|
||||
await assert.rejects(replayRecordedChrEvidence(pcap, digest(pcap), { ...evidence,
|
||||
scope: { ...evidence.scope, linkType: "RAW" } }), /pinned link type RAW/)
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
/** Offline comparison of one pcap with recorded SQL and MCP evidence. */
|
||||
import { createHash } from "node:crypto"
|
||||
import { open } from "node:fs/promises"
|
||||
import { isIPv4 } from "node:net"
|
||||
import { resolve } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { z } from "zod"
|
||||
import { checkChrCapture } from "./traffic-flow-chr-capture-check.js"
|
||||
import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js"
|
||||
import { reconcileOwnedChrFlow } from "./traffic-flow-chr-reconcile.js"
|
||||
import { projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
|
||||
const sha = z.string().regex(/^[a-f0-9]{64}$/)
|
||||
const ip = z.string().max(15).refine(isIPv4)
|
||||
const serverId = z.number().int().positive().safe()
|
||||
const snapshot = z.object({ raw: z.array(z.unknown()).max(200), minute: z.array(z.unknown()).max(20) }).strict()
|
||||
const evidenceSchema = z.object({
|
||||
captureSha256: sha, complete: z.literal(true), dropped: z.literal(0),
|
||||
scope: z.object({ sources: z.tuple([ip, ip]), destination: ip, port: z.number().int().min(1).max(65535), linkType: z.enum(["RAW", "EN10MB"]).optional() }).strict(),
|
||||
serverIds: z.tuple([serverId, serverId]),
|
||||
from: z.string().max(24), until: z.string().max(24), minutes: z.number().int().min(1).max(15),
|
||||
interfaceLabels: z.array(z.object({ serverId, index: z.string().regex(/^(?:0|[1-9]\d{0,9})$/), label: z.string().min(1).max(200) }).strict()).max(200),
|
||||
before: snapshot, after: snapshot,
|
||||
talkers: z.tuple([z.object({ serverId, result: z.unknown() }).strict(), z.object({ serverId, result: z.unknown() }).strict()]),
|
||||
}).strict()
|
||||
function must(value: unknown, reason: string): asserts value {
|
||||
if (!value) throw Error(reason)
|
||||
}
|
||||
function replayCounter(value: string) {
|
||||
const integer = BigInt(value)
|
||||
must(integer <= BigInt(Number.MAX_SAFE_INTEGER), "Product replay requires exact JavaScript integer counters")
|
||||
return Number(integer)
|
||||
}
|
||||
|
||||
export function prepareChrReplay(wire: ReturnType<typeof projectChrWire>) {
|
||||
const decoded = decodeOwnedIpfix(wire)
|
||||
const exporters = wire.scope.sources.map((address, index) => {
|
||||
const frames = wire.frames.filter(frame => frame.exporter === index)
|
||||
const first = frames[0]
|
||||
must(first && frames.every(frame => frame.sourcePort === first.sourcePort && frame.version === first.version && frame.domain === first.domain), "Exporter session changed")
|
||||
const ids = new Set(decoded.frames.flatMap((frame, ordinal) => wire.frames[ordinal].exporter === index ? frame.templateIds : []))
|
||||
must(ids.size === 1, "Replay requires one pinned template ID per exporter")
|
||||
return { address, port: first.sourcePort, version: first.version, domain: first.domain, templateId: [...ids][0] }
|
||||
})
|
||||
const phase = [0, 0]
|
||||
const frames = wire.frames.map((frame, index) => {
|
||||
const { flows, templateIds } = decoded.frames[index]
|
||||
let kind = "data"
|
||||
if (templateIds.length) kind = phase[frame.exporter]++ === 0 ? "template" : "refresh"
|
||||
const expected = flows.map(({ src, dst, bytes, packets, inIface, outIface }) =>
|
||||
({ src, dst, bytes: replayCounter(bytes), packets: replayCounter(packets), inIface, outIface }))
|
||||
return { exporter: frame.exporter, ordinal: frame.ordinal, receivedMs: frame.receivedMs, payloadHex: frame.payloadHex, kind, expected }
|
||||
})
|
||||
return { decoded, projection: { destination: wire.scope.destination, exporters, frames } }
|
||||
}
|
||||
|
||||
export async function replayRecordedChrEvidence(bytes: Buffer, expectedSha: string, rawEvidence: unknown) {
|
||||
const evidence = evidenceSchema.parse(rawEvidence)
|
||||
const wire = projectChrWire(bytes, evidence.scope)
|
||||
must(wire.captureSha256 === expectedSha && evidence.captureSha256 === expectedSha, "Capture SHA-256 does not match recorded evidence")
|
||||
const { decoded, projection } = prepareChrReplay(wire)
|
||||
const replay = checkChrCapture(projection)
|
||||
const labels = new Map<string, string>()
|
||||
for (const row of evidence.interfaceLabels) {
|
||||
const key = row.serverId + ":" + row.index
|
||||
must(evidence.serverIds.includes(row.serverId) && !labels.has(key), "Interface evidence identity differs")
|
||||
labels.set(key, row.label)
|
||||
}
|
||||
must(evidence.talkers.every((row, index) => row.serverId === evidence.serverIds[index]), "Public evidence order or scope differs")
|
||||
let snapshots = 0, publicReads = 0
|
||||
const reconciliation = await reconcileOwnedChrFlow({
|
||||
wire, decoded, serverIds: evidence.serverIds, from: evidence.from, until: evidence.until, minutes: evidence.minutes,
|
||||
interfaceLabel: (id, index) => {
|
||||
const label = labels.get(id + ":" + index)
|
||||
must(label, "Missing independently observed interface label")
|
||||
return label
|
||||
},
|
||||
readOnlySnapshot: async work => {
|
||||
const recorded = [evidence.before, evidence.after][snapshots++]
|
||||
must(recorded, "Unexpected SQL evidence read")
|
||||
return work(async sql => {
|
||||
if (sql.includes("FROM flow_buckets")) return recorded.raw
|
||||
must(sql.includes("FROM flow_minute_stats"), "Unexpected SQL evidence query")
|
||||
return recorded.minute
|
||||
})
|
||||
},
|
||||
callTool: async (tool, request) => {
|
||||
const recorded = evidence.talkers[publicReads++]
|
||||
must(tool === "mm_netflow_talkers_get" && recorded?.serverId === request.serverId, "Unexpected recorded MCP scope")
|
||||
return recorded.result
|
||||
},
|
||||
assertCurrent: async () => { /* Offline comparison of immutable recorded evidence. */ },
|
||||
})
|
||||
return { captureSha256: wire.captureSha256, replay, reconciliation, recordedEvidenceMatched: true, actualNativeAcceptance: false }
|
||||
}
|
||||
|
||||
async function readPinned(path: string, digest: string) {
|
||||
sha.parse(digest)
|
||||
const file = await open(path, "r")
|
||||
try {
|
||||
const info = await file.stat()
|
||||
must(info.isFile() && info.size > 0 && info.size <= 512 * 1024, "Offline evidence file outside size bound")
|
||||
const bytes = Buffer.alloc(info.size + 1)
|
||||
let read = 0
|
||||
while (read < bytes.length) {
|
||||
const { bytesRead } = await file.read(bytes, read, bytes.length - read, read)
|
||||
if (!bytesRead) break
|
||||
read += bytesRead
|
||||
}
|
||||
const body = bytes.subarray(0, read)
|
||||
must(read === info.size && createHash("sha256").update(body).digest("hex") === digest, "Offline evidence hash differs")
|
||||
return body
|
||||
} finally {
|
||||
await file.close()
|
||||
}
|
||||
}
|
||||
async function main() {
|
||||
const args = process.argv.slice(2)
|
||||
must(args.length === 4, "Expected pcap and recorded evidence pins")
|
||||
const [capture, captureSha, evidence, evidenceSha] = args
|
||||
const [bytes, raw] = await Promise.all([readPinned(capture, captureSha), readPinned(evidence, evidenceSha)])
|
||||
console.log(JSON.stringify({ evidenceSha256: evidenceSha, ...await replayRecordedChrEvidence(bytes, captureSha, JSON.parse(raw.toString("utf8"))) }))
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch(() => {
|
||||
console.error("Recorded CHR evidence refused")
|
||||
process.exitCode = 1
|
||||
})
|
||||
}
|
||||
@@ -89,6 +89,17 @@ test("owned journal is synchronized before target dispatch and captures target c
|
||||
assert.deepEqual(events, ["start", "create:A", "create:B", "exercise", "delete:B", "delete:A", "stop", "restore", "reconcile"])
|
||||
await assert.rejects(executeOwnedChrCapture(options), { code: "EEXIST" })
|
||||
assert.equal(events.length, 9)
|
||||
bound = true
|
||||
await assert.rejects(executeOwnedChrCapture({
|
||||
...options, journalPath: join(directory, "aborted.jsonl"),
|
||||
capture: async run => {
|
||||
const owner = new AbortController()
|
||||
owner.abort()
|
||||
await run(owner.signal)
|
||||
return fixtures(Date.now())
|
||||
},
|
||||
}), /ownership ended/)
|
||||
assert.equal(events.length, 9)
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { createChrTargetHarness } from "./traffic-flow-chr-target-harness.js"
|
||||
import { projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
import { checkChrCapture } from "./traffic-flow-chr-capture-check.js"
|
||||
|
||||
type Target = Parameters<typeof createChrTargetHarness>[0]
|
||||
type Target = Extract<Parameters<typeof createChrTargetHarness>[0], { stagedDisabled?: false }>
|
||||
type Scope = Parameters<typeof projectChrWire>[1]
|
||||
type Wire = ReturnType<typeof projectChrWire>
|
||||
type Projection = {
|
||||
@@ -15,10 +15,19 @@ type Projection = {
|
||||
frames: readonly { exporter: number; ordinal: number; receivedMs: number; payloadHex: string }[]
|
||||
}
|
||||
type Capture = { bytes: Buffer; sha256: string; dropped: number; complete: boolean; projection: Projection }
|
||||
type Ingress = { bytes: Buffer; sha256: string; existingPort: number }
|
||||
function must(value: unknown, reason: string): asserts value {
|
||||
if (!value) throw Error(reason)
|
||||
}
|
||||
|
||||
export function checkChrBaselineIngress(input: Ingress, scope: Scope, now = Date.now()) {
|
||||
const wire = projectChrWire(input.bytes, { sources: scope.sources, destination: scope.destination, port: input.existingPort })
|
||||
must(wire.captureSha256 === input.sha256, "Baseline ingress SHA-256 differs")
|
||||
must(wire.frames.every(frame => frame.receivedMs >= now - 10 * 60_000 && frame.receivedMs <= now),
|
||||
"Baseline ingress is outside the current evidence window")
|
||||
return wire
|
||||
}
|
||||
|
||||
/** Bind every reviewed row to its original wire record and require complete
|
||||
* capture coverage for conservation. Sample-only replay uses the separate CLI. */
|
||||
export function bindChrCapture(capture: Capture, scope: Scope) {
|
||||
@@ -42,11 +51,11 @@ type Options = {
|
||||
journalPath: string
|
||||
deadlineAt: number
|
||||
scope: Scope
|
||||
ingress: { bytes: Buffer; sha256: string; existingPort: number }
|
||||
ingress: Ingress
|
||||
serverIds: readonly [number, number]
|
||||
targets: readonly [Omit<Target, "source" | "destination" | "port" | "authorize" | "checkpoint">, Omit<Target, "source" | "destination" | "port" | "authorize" | "checkpoint">]
|
||||
collector: { authorize: () => Promise<void>; assertBound: () => Promise<void>; restore: () => Promise<void> }
|
||||
capture: (exercise: () => Promise<void>) => Promise<Capture>
|
||||
capture: (exercise: (signal?: AbortSignal) => Promise<void>) => Promise<Capture>
|
||||
exercise: () => Promise<void>
|
||||
reconcile: (wire: Wire, serverIds: readonly [number, number]) => Promise<void>
|
||||
}
|
||||
@@ -54,17 +63,13 @@ type Options = {
|
||||
export async function executeOwnedChrCapture(options: Options) {
|
||||
const scope = structuredClone(options.scope), serverIds = Object.freeze([...options.serverIds]) as readonly [number, number]
|
||||
const deadlineAt = options.deadlineAt
|
||||
let active = true
|
||||
const live = () => must(active && Number.isSafeInteger(deadlineAt) && Date.now() < deadlineAt, "Owned capture deadline expired")
|
||||
let active = true, captureSignal: AbortSignal | undefined
|
||||
const live = () => must(active && !captureSignal?.aborted && Number.isSafeInteger(deadlineAt) && Date.now() < deadlineAt, "Owned capture deadline or ownership ended")
|
||||
live()
|
||||
must(deadlineAt - Date.now() <= 10 * 60_000 && serverIds.length === 2 && new Set(serverIds).size === 2 &&
|
||||
serverIds.every(id => Number.isSafeInteger(id) && id > 0) && options.targets.length === 2 &&
|
||||
options.targets[0].baseline.deviceId !== options.targets[1].baseline.deviceId, "Invalid owned capture identity")
|
||||
const ingress = projectChrWire(options.ingress.bytes, { ...scope, port: options.ingress.existingPort })
|
||||
must(ingress.captureSha256 === options.ingress.sha256, "Baseline ingress SHA-256 differs")
|
||||
const admittedAt = Date.now()
|
||||
must(ingress.frames.every(frame => frame.receivedMs >= admittedAt - 10 * 60_000 && frame.receivedMs <= admittedAt),
|
||||
"Baseline ingress is outside the current evidence window")
|
||||
const ingress = checkChrBaselineIngress(options.ingress, scope)
|
||||
const file = await open(options.journalPath, "wx", 0o600)
|
||||
const append = async (record: object) => {
|
||||
live()
|
||||
@@ -93,9 +98,10 @@ export async function executeOwnedChrCapture(options: Options) {
|
||||
await authorize()
|
||||
let started = false, exercised = false
|
||||
const captureStartedAt = Date.now()
|
||||
const capture = await options.capture(async () => {
|
||||
const capture = await options.capture(async signal => {
|
||||
must(!started, "Capture exercise was already attempted")
|
||||
started = true
|
||||
captureSignal = signal
|
||||
await authorize()
|
||||
for (const target of targets) await target.create()
|
||||
await authorize()
|
||||
@@ -106,6 +112,7 @@ export async function executeOwnedChrCapture(options: Options) {
|
||||
})
|
||||
const captureEndedAt = Date.now()
|
||||
must(exercised, "Capture omitted or did not complete its exercise")
|
||||
captureSignal = undefined
|
||||
await authorize()
|
||||
await append({ phase: "collector-restore-intent" })
|
||||
await options.collector.authorize()
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import { decodeOwnedIpfix } from "./traffic-flow-chr-independent-ipfix.js"
|
||||
import { projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
import { ownedWgFastPathFields as fp, ownedWgPacketFields as packets } from "./traffic-flow-chr-interface-counters.js"
|
||||
import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js"
|
||||
|
||||
type Scope = Parameters<typeof projectChrWire>[1]
|
||||
type Counters = { startedAt: number; observedAt: number; rows: readonly Record<string, unknown>[] }
|
||||
type Interface = { id: string; name: string; ifIndex: string }
|
||||
type Expected = { exporter: 0 | 1; src: string; dst: string; srcPort: number; dstPort: number;
|
||||
inIface: string; outIface: string; packets: number; phase?: "initial" | "refreshed" }
|
||||
function must(value: unknown, reason: string): asserts value { if (!value) throw Error(reason) }
|
||||
function counter(row: Record<string, unknown>, name: string) {
|
||||
const value = row[name]
|
||||
must(typeof value === "string" && /^(0|[1-9][0-9]{0,19})$/.test(value), "Owned interface counter unavailable")
|
||||
return BigInt(value)
|
||||
}
|
||||
|
||||
/** Requires real before/after counters from the runner's pinned native reader
|
||||
* and original pcap bytes. Artifact agreement alone confers no live authority. */
|
||||
export function verifyChrOwnedWgEvidence(options: {
|
||||
bytes: Buffer; sha256: string; complete: boolean; dropped: number; scope: Scope
|
||||
exporters: readonly [{ interfaces: readonly [Interface, Interface]; before: Counters; after: Counters },
|
||||
{ interfaces: readonly [Interface, Interface]; before: Counters; after: Counters }]
|
||||
expected: readonly Expected[]
|
||||
requireRefresh?: boolean
|
||||
counterWindow?: "traffic"
|
||||
}) {
|
||||
const { scope, expected, exporters } = options
|
||||
const wire = projectChrWire(options.bytes, scope)
|
||||
must(options.complete === true && options.dropped === 0 && options.sha256 === wire.captureSha256, "Owned capture provenance differs")
|
||||
const decoded = decodeOwnedIpfix(wire)
|
||||
must(options.counterWindow !== "traffic" || options.requireRefresh, "Staged traffic counters require refreshed data")
|
||||
const matches = expected.map(() => 0n)
|
||||
must(expected.length >= 2 && expected.length <= 16 && [0, 1].every(exporter => expected.some(flow => flow.exporter === exporter)) &&
|
||||
expected.every(flow => Number.isSafeInteger(flow.packets) && flow.packets > 0), "Expected owned flow scope is incomplete")
|
||||
const sessions = new Map<number, { key: string; templateId: number; phase: "initial" | "refreshed" }>()
|
||||
if (options.requireRefresh) must([0, 1].every(exporter =>
|
||||
["initial", "refreshed"].every(phase => expected.some(pin => pin.exporter === exporter && pin.phase === phase))),
|
||||
"Both exporters need initial and refreshed flow pins")
|
||||
for (const [index, { flows, templateIds }] of decoded.frames.entries()) {
|
||||
const { exporter, sourcePort, domain } = wire.frames[index]
|
||||
if (options.requireRefresh) {
|
||||
const key = sourcePort + "/" + domain
|
||||
const session = sessions.get(exporter)
|
||||
must(!session || session.key === key, "Owned exporter session changed")
|
||||
if (templateIds.length) {
|
||||
must(templateIds.length === 1 && (!session || session.templateId === templateIds[0]), "Owned template identity changed")
|
||||
sessions.set(exporter, { key, templateId: templateIds[0], phase: session ? "refreshed" : "initial" })
|
||||
}
|
||||
}
|
||||
for (const flow of flows) {
|
||||
const matching = expected.map((pin, offset) => ({ pin, offset })).filter(({ pin }) =>
|
||||
pin.exporter === exporter && pin.src === flow.src && pin.dst === flow.dst && pin.srcPort === flow.srcPort &&
|
||||
pin.dstPort === flow.dstPort && pin.inIface === flow.inIface && pin.outIface === flow.outIface && flow.proto === 17 &&
|
||||
(!options.requireRefresh || sessions.get(exporter)?.phase === pin.phase))
|
||||
must(matching.length === 1, "Unscoped or ambiguous owned IPFIX flow")
|
||||
matches[matching[0].offset] += BigInt(flow.packets)
|
||||
}
|
||||
}
|
||||
must(expected.every((pin, offset) => matches[offset] === BigInt(pin.packets)), "Owned IPFIX packet totals differ")
|
||||
for (const [index, { before, after, interfaces }] of exporters.entries()) {
|
||||
// A staged target can emit a template as soon as it is enabled. Only
|
||||
// record-bearing frames are relevant to the forwarding counter interval.
|
||||
const timestamps = wire.frames.filter((frame, offset) => frame.exporter === index &&
|
||||
(options.counterWindow !== "traffic" || decoded.frames[offset].flows.length > 0)).map(frame => frame.receivedMs)
|
||||
must([before.startedAt, before.observedAt, after.startedAt, after.observedAt].every(Number.isSafeInteger) &&
|
||||
before.startedAt <= before.observedAt && after.startedAt <= after.observedAt &&
|
||||
timestamps.length > 0 && before.observedAt <= Math.min(...timestamps) && after.startedAt >= Math.max(...timestamps) &&
|
||||
after.observedAt - before.startedAt <= 180000 &&
|
||||
interfaces[0].id !== interfaces[1].id && interfaces[0].name !== interfaces[1].name &&
|
||||
interfaces[0].ifIndex !== interfaces[1].ifIndex &&
|
||||
interfaces.every(pin => /^\*[a-f0-9]{1,16}$/i.test(pin.id) && /^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.name) &&
|
||||
/^[1-9][0-9]{0,9}$/.test(pin.ifIndex)) &&
|
||||
expected.filter(flow => flow.exporter === index).every(flow => flow.inIface !== flow.outIface &&
|
||||
interfaces.some(pin => pin.ifIndex === flow.inIface) && interfaces.some(pin => pin.ifIndex === flow.outIface)),
|
||||
"Owned counter window or interface scope differs")
|
||||
for (const pin of interfaces) {
|
||||
const read = (sample: Counters) => {
|
||||
const rows = sample.rows.filter(row => row[".id"] === pin.id || row.name === pin.name)
|
||||
must(rows.length === 1 && rows[0][".id"] === pin.id && rows[0].name === pin.name && isChrWireGuardInterfaceType(rows[0].type) &&
|
||||
rows[0].ifindex === pin.ifIndex,
|
||||
"Owned interface counter identity differs")
|
||||
return rows[0]
|
||||
}
|
||||
const old = read(before), current = read(after)
|
||||
must(fp.every(name => counter(old, name) === counter(current, name)), "Owned interface FastPath counter changed")
|
||||
must(packets.every(name => counter(current, name) > counter(old, name)), "Owned interface did not observe bidirectional packets")
|
||||
}
|
||||
}
|
||||
return { captureSha256: wire.captureSha256, exactIpfixMatched: true as const,
|
||||
ownedFastPathDeltasZero: true as const, nativeAcceptance: false as const }
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
import { fstatSync } from "node:fs"
|
||||
import { Socket } from "node:net"
|
||||
import type { Duplex } from "node:stream"
|
||||
import { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js"
|
||||
|
||||
type Parent = Awaited<ReturnType<typeof createChrParentOrchestration>>
|
||||
|
||||
/** The root launcher inherits one private socketpair end from its credential
|
||||
* verifying AF_UNIX acceptor. The collector's pathname never reaches Node. */
|
||||
function inheritedSocket(descriptor: number) {
|
||||
if (process.platform !== "linux" || process.geteuid?.() !== 0 ||
|
||||
!Number.isSafeInteger(descriptor) || descriptor < 3 || descriptor > 1024 ||
|
||||
!fstatSync(descriptor).isSocket()) {
|
||||
throw Error("Inherited parent CHR admission socket refused")
|
||||
}
|
||||
return new Socket({ fd: descriptor, readable: true, writable: true, allowHalfOpen: true })
|
||||
}
|
||||
|
||||
export async function serveChrParentInheritedAdmission(parent: Parent, descriptor: number) {
|
||||
let socket: Socket | undefined
|
||||
try {
|
||||
socket = inheritedSocket(descriptor)
|
||||
return await parent.serve({ input: socket, output: socket })
|
||||
} finally {
|
||||
socket?.destroy()
|
||||
parent.retire()
|
||||
}
|
||||
}
|
||||
|
||||
/** Ask the host owner to recheck actual bridge milestones. This channel is
|
||||
* separate from the collector connection and has no mutation operations. */
|
||||
export function openChrParentHostPhaseChecks(descriptor: number, deadlineAt: number, staged = false) {
|
||||
return openChrParentConnectedPhaseChecks(inheritedSocket(descriptor), deadlineAt, staged)
|
||||
}
|
||||
|
||||
export function openChrParentConnectedPhaseChecks(socket: Duplex, deadlineAt: number, staged = false) {
|
||||
socket.on("error", () => {})
|
||||
const responses = socket[Symbol.asyncIterator]()
|
||||
let sequence = 0, pending = false
|
||||
const check = async (phase: "preflight" | "stage" | "ready" | "drain" | "inverse" | "timer") => {
|
||||
if (pending || socket.destroyed || ++sequence > (staged ? 99 : 91) || Date.now() >= deadlineAt) {
|
||||
socket.destroy()
|
||||
throw Error("Host phase query refused")
|
||||
}
|
||||
pending = true
|
||||
const timer = setTimeout(() => socket.destroy(), Math.min(8000, deadlineAt - Date.now()))
|
||||
try {
|
||||
const expected = sequence + ":ok"
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.write(sequence + ":" + phase + "\n", error => error ? reject(error) : resolve())
|
||||
})
|
||||
let received = Buffer.alloc(0)
|
||||
while (received.at(-1) !== 10) {
|
||||
const { value, done } = await responses.next()
|
||||
if (done || !Buffer.isBuffer(value) || received.length + value.length > 64) throw Error("Host phase query refused")
|
||||
received = Buffer.concat([received, value])
|
||||
}
|
||||
const text = received.toString("ascii")
|
||||
if (phase !== "timer") {
|
||||
if (text !== expected + "\n") throw Error("Host phase query refused")
|
||||
return
|
||||
}
|
||||
const deadline = text.startsWith(expected + ":") ? text.slice(expected.length + 1) : ""
|
||||
const value = Number(deadline.trim())
|
||||
if (!/^[1-9][0-9]{12}\n$/.test(deadline) || !Number.isSafeInteger(value) || value <= Date.now() ||
|
||||
value > Math.min(deadlineAt, Date.now() + 240000)) throw Error("Host capture deadline refused")
|
||||
return value
|
||||
} catch (error) { socket.destroy(); throw error }
|
||||
finally { clearTimeout(timer); pending = false }
|
||||
}
|
||||
const fence = async (phase: Exclude<Parameters<typeof check>[0], "timer">) => { await check(phase) }
|
||||
return Object.freeze({
|
||||
assertTimingPreflight: () => fence("preflight"),
|
||||
...(staged ? { assertTransitAndCollectorStagingReady: () => fence("stage") } : {}),
|
||||
assertTransitAndCollectorReady: () => fence("ready"),
|
||||
assertCaptureAndDrainComplete: () => fence("drain"),
|
||||
assertLinuxInverseReady: () => fence("inverse"),
|
||||
async captureDeadline() {
|
||||
if (!staged) throw Error("Staged host capture deadline unavailable")
|
||||
const result = await check("timer")
|
||||
if (result === undefined) throw Error("Staged host capture deadline unavailable")
|
||||
return result
|
||||
},
|
||||
close: () => socket.destroy(),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
type HostCheck = keyof Omit<ReturnType<typeof openChrParentHostPhaseChecks>, "close" | "captureDeadline">
|
||||
|
||||
/** The host launches one root Node with two private descriptors. Call serve
|
||||
* after opening the CHR relay with the returned hooks and sealing its pins. */
|
||||
export async function openChrParentAdmissionTransport(
|
||||
options: Omit<Parameters<typeof createChrParentOrchestration>[0], HostCheck>,
|
||||
descriptors: { admission: number; phase: number },
|
||||
) {
|
||||
if (descriptors.admission === descriptors.phase) throw Error("Parent admission descriptors overlap")
|
||||
const socket = inheritedSocket(descriptors.admission)
|
||||
socket.on("error", () => {})
|
||||
let phases: ReturnType<typeof openChrParentHostPhaseChecks> | undefined
|
||||
let parent: Parent | undefined
|
||||
try {
|
||||
phases = openChrParentHostPhaseChecks(descriptors.phase, options.deadlineAt, !!options.stagedTarget)
|
||||
parent = await createChrParentOrchestration({ ...options, ...phases })
|
||||
const owner = parent
|
||||
const close = () => { owner.retire(); socket.destroy(); phases?.close() }
|
||||
return Object.freeze({
|
||||
authorizeWrite: owner.authorizeWrite,
|
||||
recordWriteAcknowledgement: owner.recordWriteAcknowledgement,
|
||||
async serve() {
|
||||
try { return await owner.serve({ input: socket, output: socket }) }
|
||||
finally { close() }
|
||||
},
|
||||
close,
|
||||
})
|
||||
} catch (error) {
|
||||
parent?.retire()
|
||||
socket.destroy()
|
||||
phases?.close()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { Duplex } from "node:stream"
|
||||
import type { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js"
|
||||
import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js"
|
||||
|
||||
type Owner = Awaited<ReturnType<typeof createChrParentOrchestration>>
|
||||
type Hooks = Pick<Owner, "authorizeWrite" | "recordWriteAcknowledgement">
|
||||
const frameLimit = 16384
|
||||
const nativeId = /^\*[a-f0-9]{1,16}$/i
|
||||
const fail = () => { throw Error("Parent native boundary refused. Automatic replay forbidden.") }
|
||||
const object = (raw: unknown): raw is Record<string, unknown> => !!raw && typeof raw === "object" && !Array.isArray(raw)
|
||||
const shape = (raw: Record<string, unknown>, keys: string[]) =>
|
||||
Object.keys(raw).length === keys.length && keys.every(key => Object.hasOwn(raw, key))
|
||||
|
||||
async function* frames(socket: Duplex) {
|
||||
let buffered = Buffer.alloc(0)
|
||||
for await (const data of socket) {
|
||||
const bytes = Buffer.isBuffer(data) ? data : Buffer.from(data)
|
||||
if (bytes.length > frameLimit * 2) return fail()
|
||||
for (let offset = 0; offset < bytes.length;) {
|
||||
const end = bytes.indexOf(10, offset)
|
||||
const length = (end < 0 ? bytes.length : end) - offset
|
||||
if (buffered.length + length > frameLimit) return fail()
|
||||
buffered = Buffer.concat([buffered, bytes.subarray(offset, offset + length)])
|
||||
if (end < 0) break
|
||||
const value: unknown = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(buffered))
|
||||
if (!object(value)) return fail()
|
||||
yield value
|
||||
buffered = Buffer.alloc(0)
|
||||
offset = end + 1
|
||||
}
|
||||
}
|
||||
if (buffered.length) return fail()
|
||||
}
|
||||
|
||||
/** Host obtains a consumed grant before sending native bytes. Only its
|
||||
* authenticated peer can submit the subsequent complete response projection. */
|
||||
export async function serveChrParentNativeBoundary(socket: Duplex, hooks: Hooks, assertCurrent: () => Promise<void>) {
|
||||
let sequence = 0
|
||||
let pending: ChrWriteIntent | undefined
|
||||
socket.on("error", () => {})
|
||||
for await (const value of frames(socket)) {
|
||||
await assertCurrent()
|
||||
const { side, method, path } = value
|
||||
if (value.sequence !== ++sequence || sequence > 100 || side !== "A" && side !== "B" ||
|
||||
!["PUT", "PATCH", "POST", "DELETE"].includes(String(method)) || typeof path !== "string" || path.length > 128) return fail()
|
||||
if (value.op === "consume") {
|
||||
if (pending || !shape(value, ["sequence", "op", "side", "method", "path", "body"]) || !object(value.body) ||
|
||||
!Object.values(value.body).every(item => typeof item === "string")) return fail()
|
||||
pending = await hooks.authorizeWrite({ side, method: String(method), path, body: value.body as Record<string, string> })
|
||||
if (pending.side !== side || pending.method !== method || pending.path !== path) return fail()
|
||||
} else if (value.op === "ack") {
|
||||
const { status, id } = value
|
||||
if (!pending || pending.side !== side || pending.method !== method || pending.path !== path ||
|
||||
!shape(value, ["sequence", "op", "side", "method", "path", "status", ...(Object.hasOwn(value, "id") ? ["id"] : [])]) ||
|
||||
typeof status !== "number" || !Number.isInteger(status) || status < 100 || status > 599) return fail()
|
||||
const needsId = (method === "PUT" && (status === 200 || status === 201)) ||
|
||||
(pending.step === "target:" + side + ":enable" && status === 200)
|
||||
if (needsId ? typeof id !== "string" || !nativeId.test(id) : Object.hasOwn(value, "id")) return fail()
|
||||
hooks.recordWriteAcknowledgement({ intent: pending, status, body: needsId ? { ".id": id } : undefined })
|
||||
pending = undefined
|
||||
} else return fail()
|
||||
await assertCurrent()
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
socket.write(JSON.stringify({ sequence, ok: true }) + "\n", error => error ? reject(error) : resolve())
|
||||
})
|
||||
}
|
||||
return fail()
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { runChrParentSynthetic } from "./traffic-flow-chr-parent-synthetic.js"
|
||||
|
||||
for (const mode of ["direct", "staged", "wrong-ack"] as const) test("paired catalogue and native ACK parent profile: " + mode, async () => {
|
||||
const result = await runChrParentSynthetic(mode)
|
||||
assert.equal(result.writes, mode === "direct" ? 48 : mode === "staged" ? 50 : 25)
|
||||
})
|
||||
@@ -0,0 +1,131 @@
|
||||
import type { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { createChrParentAckMailbox, createChrParentTrustedStages } from "./traffic-flow-chr-parent-stages.js"
|
||||
import { createChrTransitTopologyInventory } from "./traffic-flow-chr-topology-inventory.js"
|
||||
import { createChrParentWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js"
|
||||
import { serveChrParentAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js"
|
||||
import { chrFixtureDeadlineMaxMs, type ChrStagedFixtureLease } from "./traffic-flow-chr-target-timing-profile.js"
|
||||
|
||||
type StagesOptions = Parameters<typeof createChrParentTrustedStages>[0]
|
||||
type Side = "A" | "B"
|
||||
type Row = Record<string, unknown>
|
||||
type Options = Pick<StagesOptions, "scope" | "linux" | "deadlineAt" | "authorize" | "timerProfile" | "stagedTarget"> & {
|
||||
leaseProfile?: ChrStagedFixtureLease
|
||||
routers: Record<Side, { deviceId: string; endpoint: string; client: Pick<MikrotikClient, "get"> }>
|
||||
assertTimingPreflight(): Promise<void>
|
||||
assertRouterIdentity(side: Side, deviceId: string): Promise<void>
|
||||
assertTransitAndCollectorReady(): Promise<void>
|
||||
assertTransitAndCollectorStagingReady?(): Promise<void>
|
||||
assertCaptureAndDrainComplete(): Promise<void>
|
||||
assertLinuxInverseReady(): Promise<void>
|
||||
}
|
||||
|
||||
const fail = () => { throw Error("Parent CHR composition refused. Automatic replay forbidden.") }
|
||||
const isRow = (value: unknown): value is Row => !!value && typeof value === "object" && !Array.isArray(value)
|
||||
|
||||
export async function createChrParentOrchestration(options: Options) {
|
||||
const { deadlineAt, leaseProfile, authorize, assertRouterIdentity } = options
|
||||
const stagedTarget = options.stagedTarget && { ...options.stagedTarget }
|
||||
const stageReady = options.assertTransitAndCollectorStagingReady
|
||||
if ((stagedTarget && !stageReady) || (leaseProfile && !stagedTarget)) return fail()
|
||||
const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } }
|
||||
const sides = ["A", "B"] as const
|
||||
const check = () => {
|
||||
const left = deadlineAt - Date.now()
|
||||
if (!Number.isSafeInteger(deadlineAt) || left <= 0 || left > chrFixtureDeadlineMaxMs(leaseProfile)) return fail()
|
||||
return Math.min(8000, left)
|
||||
}
|
||||
const assertCurrent = async (side: Side, deviceId: string) => {
|
||||
check()
|
||||
if (routers[side].deviceId !== deviceId) return fail()
|
||||
await authorize()
|
||||
await assertRouterIdentity(side, deviceId)
|
||||
check()
|
||||
}
|
||||
const assertPair = async () => {
|
||||
for (const side of sides) await assertCurrent(side, routers[side].deviceId)
|
||||
}
|
||||
const assertStageAuthority = async (intent: Readonly<ChrWriteIntent>) => {
|
||||
await assertPair()
|
||||
if (intent.step.startsWith("topology:delete:")) {
|
||||
await options.assertLinuxInverseReady()
|
||||
} else if (intent.step.endsWith(".restore") || intent.step.startsWith("target:") && intent.method === "DELETE") {
|
||||
await options.assertCaptureAndDrainComplete()
|
||||
} else if (stageReady && stagedTarget && (intent.step.endsWith(".enable") || intent.step.startsWith("target:") && intent.method === "PUT")) {
|
||||
await stageReady()
|
||||
} else if (!intent.step.startsWith("topology:create:")) {
|
||||
await options.assertTransitAndCollectorReady()
|
||||
}
|
||||
check()
|
||||
}
|
||||
const exactGet = async (side: Side, path: string, singleton = false) => {
|
||||
const { client } = routers[side]
|
||||
const fence = async () => { check(); await authorize(); check() }
|
||||
await fence()
|
||||
const timeout = check()
|
||||
const raw = await client.get<unknown>(path, timeout, AbortSignal.timeout(timeout), 65536, fence)
|
||||
await fence()
|
||||
const rows = Array.isArray(raw) ? raw : singleton && isRow(raw) ? [raw] : undefined
|
||||
if (!rows || Buffer.byteLength(JSON.stringify(rows)) > 65536 || !rows.every(isRow)) return fail()
|
||||
return rows
|
||||
}
|
||||
const readFlow = async (side: Side) => {
|
||||
const deviceId = routers[side].deviceId
|
||||
await assertCurrent(side, deviceId)
|
||||
const [flow, ipfix, targets] = await Promise.all([
|
||||
exactGet(side, "/ip/traffic-flow", true), exactGet(side, "/ip/traffic-flow/ipfix", true),
|
||||
exactGet(side, "/ip/traffic-flow/target"),
|
||||
])
|
||||
await assertCurrent(side, deviceId)
|
||||
if (flow.length !== 1 || ipfix.length !== 1 || targets.length > 1) return fail()
|
||||
return { deviceId, flow: flow[0], ipfix: ipfix[0], targets }
|
||||
}
|
||||
check()
|
||||
await assertPair()
|
||||
await options.assertTimingPreflight()
|
||||
check()
|
||||
const scope = structuredClone(options.scope)
|
||||
const inventory = createChrTransitTopologyInventory({
|
||||
runId: scope[0], deadlineAt, authorize: assertPair, authorizeRequest: authorize, parallelism: 2, routers,
|
||||
})
|
||||
const mailbox = createChrParentAckMailbox()
|
||||
const trusted = await createChrParentTrustedStages({
|
||||
scope, linux: options.linux, deadlineAt, timerProfile: options.timerProfile, stagedTarget, authorize: assertPair,
|
||||
routers: {
|
||||
A: { ...routers.A, read: () => inventory.read("A"), readFlow: () => readFlow("A") },
|
||||
B: { ...routers.B, read: () => inventory.read("B"), readFlow: () => readFlow("B") },
|
||||
},
|
||||
assertOutsideScope: inventory.assertOutsideScope, assertStageAuthority, takeAcknowledgement: mailbox.take,
|
||||
})
|
||||
const gate = createChrParentWriteAdmission({
|
||||
deadlineAt, profile: trusted.profile, deviceIds: { A: routers.A.deviceId, B: routers.B.deviceId }, assertCurrent, assertIntentCurrent: assertStageAuthority,
|
||||
})
|
||||
let started = false, retired = false
|
||||
let activeInput: Parameters<typeof serveChrParentAdmissionPipe>[0]["input"] | undefined
|
||||
const retire = () => {
|
||||
retired = true
|
||||
gate.retire()
|
||||
mailbox.retire()
|
||||
trusted.retire()
|
||||
activeInput?.destroy()
|
||||
}
|
||||
return Object.freeze({
|
||||
authorizeWrite: gate.authorizeWrite,
|
||||
recordWriteAcknowledgement: mailbox.record,
|
||||
retire,
|
||||
async serve(pipes: Parameters<typeof serveChrParentAdmissionPipe>[0]) {
|
||||
if (started || retired) return fail()
|
||||
started = true
|
||||
activeInput = pipes.input
|
||||
try {
|
||||
const result = await serveChrParentAdmissionPipe(pipes, gate, trusted.stages, deadlineAt, trusted.profile, leaseProfile)
|
||||
if (result.completed !== trusted.stages.length || gate.consumed !== result.completed) return fail()
|
||||
return result
|
||||
} finally { retire() }
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { chrTransitNames } from "./traffic-flow-chr-transit-fixture.js"
|
||||
|
||||
|
||||
const runId = "a012bcde"
|
||||
const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106,
|
||||
collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 }
|
||||
const directions = ["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", "underlay-A-to-linuxA",
|
||||
"underlay-A-to-B", "underlay-B-to-A", "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const
|
||||
const isolation = { checkedPorts: Object.values(ports), checkedNames: Object.values(chrTransitNames(runId)),
|
||||
checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"],
|
||||
occupiedPorts: [], overlappingCidrs: [], occupiedNames: [],
|
||||
policies: directions.map(direction => ({ direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0 } as const)) }
|
||||
import { createChrParentAckMailbox, createChrParentTrustedStages } from "./traffic-flow-chr-parent-stages.js"
|
||||
import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js"
|
||||
|
||||
const menus = { "/interface/wireguard": "wireguard", "/interface/wireguard/peers": "peer", "/ip/address": "address", "/ip/route": "route" } as const
|
||||
async function fixture(fault?: "foreign" | "ack" | "singleton" | "target" | "authority") {
|
||||
const makeRows = () => ({ wireguard: [] as Row[], peer: [] as Row[], address: [] as Row[], route: [] as Row[] })
|
||||
const ipfix = Object.fromEntries(["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"].map(key => [key, "no"]))
|
||||
const rows = { A: makeRows(), B: makeRows() }
|
||||
const makeFlow = (side: "A" | "B") => ({
|
||||
deviceId: side, flow: { ...(side === "B" ? { ".id": "*F" } : {}), enabled: "no", interfaces: "all",
|
||||
"inactive-flow-timeout": "15s" }, ipfix: { ...ipfix }, targets: [] as Row[],
|
||||
})
|
||||
const flow = { A: makeFlow("A"), B: makeFlow("B") }
|
||||
function router(side: "A" | "B", endpoint: string) {
|
||||
return { deviceId: side, endpoint, read: async () => ({ deviceId: side, complete: true as const, rows: structuredClone(rows[side]) }),
|
||||
readFlow: async () => structuredClone(flow[side]) }
|
||||
}
|
||||
let count = 0
|
||||
const writes: ChrWriteIntent[] = []
|
||||
const result = await createChrParentTrustedStages({
|
||||
scope: [runId, ports, isolation], deadlineAt: 100000, now: () => 10000,
|
||||
timerProfile: { kind: "bounded-ipfix-timeout-20s", captureDeadlineAt: 99000, now: () => 10000 },
|
||||
routers: { A: router("A", "192.0.2.1"), B: router("B", "192.0.2.2") },
|
||||
linux: { endpoint: "192.0.2.3", publicKeys: { linuxA: "X".repeat(43) + "=", linuxB: "Y".repeat(43) + "=" } },
|
||||
authorize: async () => {}, assertOutsideScope: async () => {},
|
||||
async assertStageAuthority(intent) {
|
||||
if (fault === "authority" && intent.step === "flow:A:ipfix.enable") throw Error("Owner phase refused")
|
||||
},
|
||||
async takeAcknowledgement(intent) {
|
||||
const recorded = writes.at(-1)!
|
||||
const id = "*" + (++count).toString(16)
|
||||
const slash = recorded.path.lastIndexOf("/")
|
||||
const menuPath = recorded.method === "PUT" ? recorded.path : recorded.path.slice(0, slash)
|
||||
const menu = menus[menuPath as keyof typeof menus]
|
||||
if (menu) {
|
||||
if (recorded.method === "PUT") {
|
||||
rows[recorded.side][menu].push({ ...recorded.body, ".id": id, ...(menu === "wireguard" ? { "public-key": String.fromCharCode(64 + count).repeat(43) + "=" } : {}) })
|
||||
if (fault === "foreign") rows.B.address.push({ ".id": "*FF" })
|
||||
} else rows[recorded.side][menu] = rows[recorded.side][menu].filter(row => row[".id"] !== decodeURIComponent(recorded.path.slice(slash + 1)))
|
||||
} else if (menuPath === "/ip/traffic-flow/target") {
|
||||
if (recorded.method === "PUT") {
|
||||
flow[recorded.side].targets.push({ ".id": id, disabled: false, ...recorded.body,
|
||||
...(fault === "target" ? { "v9-template-timeout": "21s" } : {}) })
|
||||
} else flow[recorded.side].targets = []
|
||||
} else {
|
||||
Object.assign(menuPath.endsWith("/ipfix") ? flow[recorded.side].ipfix : flow[recorded.side].flow, recorded.body)
|
||||
if (fault === "singleton") flow[recorded.side].flow.interfaces = "foreign"
|
||||
}
|
||||
return { intent: { ...intent, ...(fault === "ack" ? { side: "B" } : {}) },
|
||||
status: recorded.method === "PUT" ? 201 : 204, body: recorded.method === "PUT" ? { ".id": id } : undefined }
|
||||
},
|
||||
})
|
||||
return { ...result, rows, flow, writes, async step(index: number) {
|
||||
const stage = result.stages[index]
|
||||
const intent = await stage.intent()
|
||||
writes.push(intent)
|
||||
await stage.verify(intent)
|
||||
} }
|
||||
}
|
||||
|
||||
test("parent resolves exact 48 stages from real proof owners and restores both baselines", async () => {
|
||||
const f = await fixture()
|
||||
assert.equal(f.stages.length, 48)
|
||||
for (let i = 0; i < f.stages.length; i++) await f.step(i)
|
||||
assert(f.writes.some(write => write.step === "flow:B:flow.enable" && write.method === "PATCH" && write.path === "/ip/traffic-flow/*F"))
|
||||
assert(f.writes.some(write => write.step === "flow:A:flow.enable" && write.method === "POST" && write.path === "/ip/traffic-flow/set"))
|
||||
assert(f.writes.some(write => write.step === "target:A:create" && write.body["v9-template-timeout"] === "20s"))
|
||||
assert.deepEqual(Object.values(f.rows).flatMap(side => Object.values(side).flat()), [])
|
||||
assert.equal(f.flow.B.flow.enabled, "no")
|
||||
})
|
||||
|
||||
test("parent refuses ACK identity, peer drift, singleton drift and wrong target timer without releasing next stage", async () => {
|
||||
for (const [fault, failedAt] of [["ack", 0], ["foreign", 0], ["singleton", 18], ["target", 20]] as const) {
|
||||
const f = await fixture(fault)
|
||||
for (let i = 0; i < failedAt; i++) await f.step(i)
|
||||
await assert.rejects(f.step(failedAt))
|
||||
await assert.rejects(f.stages[failedAt + 1].intent())
|
||||
await assert.rejects(f.stages[failedAt].intent())
|
||||
assert.equal(f.writes.length, failedAt + 1)
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
test("outer relay ACK mailbox allows one exact take and latches missing or duplicate data", async () => {
|
||||
const f = await fixture()
|
||||
const intent = await f.stages[0].intent()
|
||||
const ack = { intent, status: 201, body: { ".id": "*A" } }
|
||||
f.retire()
|
||||
const mailbox = createChrParentAckMailbox()
|
||||
mailbox.record(ack)
|
||||
assert.deepEqual(await mailbox.take(intent), ack)
|
||||
await assert.rejects(mailbox.take(intent))
|
||||
assert.throws(() => mailbox.record(ack))
|
||||
const duplicate = createChrParentAckMailbox()
|
||||
duplicate.record(ack)
|
||||
assert.throws(() => duplicate.record(ack))
|
||||
await assert.rejects(duplicate.take(intent))
|
||||
})
|
||||
|
||||
|
||||
test("parent readiness refusal blocks the export phase before any singleton or target grant", async () => {
|
||||
const f = await fixture("authority")
|
||||
for (let i = 0; i < 18; i++) await f.step(i)
|
||||
await assert.rejects(f.stages[18].intent())
|
||||
await assert.rejects(f.stages[19].intent())
|
||||
assert.equal(f.writes.length, 18)
|
||||
})
|
||||
@@ -0,0 +1,280 @@
|
||||
import { isDeepStrictEqual } from "node:util"
|
||||
import { createDisabledChrFlowHarness } from "./traffic-flow-chr-disabled-baseline.js"
|
||||
import { assertChrTargetTimerWindow, createChrTargetHarness, type ChrTargetTimerProfile, type ChrStagedTargetTimerProfile } from "./traffic-flow-chr-target-harness.js"
|
||||
import { createChrTransitTopologyOwner } from "./traffic-flow-chr-topology-owner.js"
|
||||
import { planChrTransitFixture } from "./traffic-flow-chr-transit-fixture.js"
|
||||
import { chrWriteScheduleLimit, type ChrWriteAdmission, type ChrWriteIntent } from "./traffic-flow-chr-write-admission.js"
|
||||
import type { serveChrParentAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js"
|
||||
|
||||
type Topology = Parameters<typeof createChrTransitTopologyOwner>[0]
|
||||
type Flow = Parameters<typeof createDisabledChrFlowHarness>[0]
|
||||
type Side = "A" | "B"
|
||||
type Stage = Parameters<typeof serveChrParentAdmissionPipe>[2][number]
|
||||
export type ChrParentNativeAck = { intent: ChrWriteIntent; status: number; body: unknown }
|
||||
type Options = Pick<Topology, "scope" | "linux" | "deadlineAt" | "authorize" | "assertOutsideScope" | "now"> & {
|
||||
routers: Record<Side, Omit<Topology["routers"][Side], "native"> & { readFlow: Flow["read"] }>
|
||||
timerProfile?: ChrTargetTimerProfile
|
||||
stagedTarget?: { captureTimer(): Promise<ChrStagedTargetTimerProfile> }
|
||||
/** Assert the parent-owned phase permits this stage (for example, isolated transit ready before export). */
|
||||
assertStageAuthority(intent: Readonly<ChrWriteIntent>): Promise<void>
|
||||
takeAcknowledgement(intent: Readonly<ChrWriteIntent>): Promise<ChrParentNativeAck>
|
||||
}
|
||||
|
||||
const roles = ["edgeA", "middleA", "middleB", "edgeB"] as const
|
||||
const sides = ["A", "B"] as const
|
||||
const exportFields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"]
|
||||
const disabled = (value: unknown) => value === false || value === "false" || value === "no"
|
||||
const refuse = (): never => { throw Error("Parent native stage refused. Automatic replay forbidden.") }
|
||||
const require = (value: unknown) => { if (!value) refuse() }
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T) => void, reject!: (error: unknown) => void
|
||||
const promise = new Promise<T>((yes, no) => { resolve = yes; reject = no })
|
||||
void promise.catch(() => {})
|
||||
return { promise, resolve, reject }
|
||||
}
|
||||
|
||||
/** The parent executes the same bounded proof owners using independent native reads.
|
||||
* Its synthetic transport cannot send a write and can only consume the outer relay ACK. */
|
||||
export async function createChrParentTrustedStages(options: Options) {
|
||||
const scope = structuredClone(options.scope)
|
||||
const plan = planChrTransitFixture(...scope)
|
||||
const routers = { A: { ...options.routers.A }, B: { ...options.routers.B } }
|
||||
const { deadlineAt, now = Date.now, authorize, takeAcknowledgement, assertOutsideScope, assertStageAuthority } = options
|
||||
const timerProfile = options.timerProfile && { ...options.timerProfile }
|
||||
const stagedTarget = options.stagedTarget && { ...options.stagedTarget }
|
||||
require(!stagedTarget || !timerProfile)
|
||||
const profile = stagedTarget ? "staged-disabled-target" : "direct-target"
|
||||
let failed = false, cursor = 0
|
||||
const live = () => require(!failed && Number.isSafeInteger(deadlineAt) && now() < deadlineAt)
|
||||
const readFlow = async (side: Side) => {
|
||||
live()
|
||||
await authorize()
|
||||
const value = structuredClone(await routers[side].readFlow())
|
||||
await authorize()
|
||||
live()
|
||||
require(value.deviceId === routers[side].deviceId && value.flow && value.ipfix &&
|
||||
Array.isArray(value.targets) && value.targets.length <= 1 && Buffer.byteLength(JSON.stringify(value)) <= 65536)
|
||||
return value
|
||||
}
|
||||
const baseline = { A: await readFlow("A"), B: await readFlow("B") }
|
||||
if (timerProfile) {
|
||||
require(timerProfile.captureDeadlineAt <= deadlineAt)
|
||||
for (const side of sides) assertChrTargetTimerWindow(timerProfile, baseline[side].flow["inactive-flow-timeout"])
|
||||
}
|
||||
const expected = structuredClone(baseline)
|
||||
const hasIpfixChange = (side: Side) => exportFields.some(field => disabled(baseline[side].ipfix[field]))
|
||||
const keys = [...roles, ...roles.map(role => "address:" + role), ...roles.map(role => "peer:" + role),
|
||||
...plan.routes.filter(route => route.node === "A" || route.node === "B").map((route, i) => [route.node, "route:" + i].join(":"))]
|
||||
const topologyStep = (action: string, key: string) => {
|
||||
const side = key.startsWith("A:route:") || key.startsWith("B:route:") ? key[0] : key.at(-1)
|
||||
return "topology:" + action + ":" + side + ":" + key.replace(/^[AB]:(?=route:)/, "")
|
||||
}
|
||||
const labels = keys.map(key => topologyStep("create", key))
|
||||
for (const side of sides) {
|
||||
if (hasIpfixChange(side)) labels.push("flow:" + side + ":ipfix.enable")
|
||||
labels.push("flow:" + side + ":flow.enable", "target:" + side + ":create")
|
||||
}
|
||||
if (stagedTarget) for (const side of sides) labels.push("target:" + side + ":enable")
|
||||
for (const side of [...sides].reverse()) {
|
||||
labels.push("target:" + side + ":delete", "flow:" + side + ":flow.restore")
|
||||
if (hasIpfixChange(side)) labels.push("flow:" + side + ":ipfix.restore")
|
||||
}
|
||||
labels.push(...[...keys].reverse().map(key => topologyStep("delete", key)))
|
||||
require(keys.length === 18 && labels.length <= chrWriteScheduleLimit(profile) && new Set(labels).size === labels.length)
|
||||
type Slot = {
|
||||
intent: ChrWriteIntent
|
||||
acknowledgement: ReturnType<typeof deferred<unknown>>
|
||||
verified: ReturnType<typeof deferred<void>>
|
||||
requested: boolean
|
||||
supplied: boolean
|
||||
}
|
||||
const ready = labels.map(() => deferred<Slot>())
|
||||
const requested = labels.map(() => deferred<void>())
|
||||
let slot: Slot | undefined
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
let launched = false
|
||||
const abort = (error: unknown) => {
|
||||
failed = true
|
||||
clearTimeout(timer)
|
||||
slot?.acknowledgement.reject(error)
|
||||
slot?.verified.reject(error)
|
||||
for (const entry of ready) entry.reject(error)
|
||||
for (const entry of requested) entry.reject(error)
|
||||
}
|
||||
const expectFlow = async (side: Side) => require(isDeepStrictEqual(await readFlow(side), expected[side]))
|
||||
const otherSide = (side: Side) => side === "A" ? "B" : "A"
|
||||
const conserveFlow = async (intent: ChrWriteIntent) => {
|
||||
await expectFlow(otherSide(intent.side))
|
||||
if (intent.step.startsWith("topology:")) await expectFlow(intent.side)
|
||||
}
|
||||
|
||||
const broker: ChrWriteAdmission = {
|
||||
async run<T>(intent: ChrWriteIntent, send: (arm: () => Promise<void>) => Promise<T>): Promise<T> {
|
||||
live()
|
||||
require(!slot && labels[cursor] === intent.step)
|
||||
await requested[cursor].promise
|
||||
live()
|
||||
slot = { intent: structuredClone(intent), acknowledgement: deferred(), verified: deferred(), requested: false, supplied: false }
|
||||
return send(async () => {
|
||||
live()
|
||||
require(slot?.intent.step === intent.step)
|
||||
await assertStageAuthority(structuredClone(intent))
|
||||
await conserveFlow(intent)
|
||||
if (!intent.step.startsWith("topology:")) await topology.assertCurrent()
|
||||
live()
|
||||
ready[cursor].resolve(slot!)
|
||||
})
|
||||
},
|
||||
}
|
||||
const nativeFor = () => {
|
||||
const simulate = async <T>(method: ChrWriteIntent["method"], path: string, body: unknown, before?: () => Promise<void>) => {
|
||||
const entry = slot
|
||||
require(entry && before && isDeepStrictEqual({ method, path, body }, {
|
||||
method: entry?.intent.method, path: entry?.intent.path, body: entry?.intent.body,
|
||||
}))
|
||||
await before!()
|
||||
return await entry!.acknowledgement.promise as T
|
||||
}
|
||||
return {
|
||||
put: <T>(path: string, body: unknown, _timeout?: number, before?: () => Promise<void>) => simulate<T>("PUT", path, body, before),
|
||||
patch: <T>(path: string, body: unknown, _timeout?: number, before?: () => Promise<void>) => simulate<T>("PATCH", path, body, before),
|
||||
post: <T>(path: string, body: unknown, _timeout?: number, _headers?: unknown, before?: () => Promise<void>) => simulate<T>("POST", path, body, before),
|
||||
delete: async (path: string, _timeout?: number, before?: () => Promise<void>) => { await simulate("DELETE", path, {}, before) },
|
||||
}
|
||||
}
|
||||
const checkpoint = async (entry: { phase: string }) => {
|
||||
if (entry.phase !== "verified") return
|
||||
const verified = slot
|
||||
require(verified?.supplied)
|
||||
const intent = verified!.intent
|
||||
await conserveFlow(intent)
|
||||
if (!intent.step.startsWith("topology:")) {
|
||||
await topology.assertCurrent()
|
||||
// The harness already performed exact local readback. See checkedRead below.
|
||||
require(nextFlow && nextFlow.side === intent.side)
|
||||
expected[intent.side] = nextFlow!.value
|
||||
nextFlow = undefined
|
||||
}
|
||||
slot = undefined
|
||||
cursor++
|
||||
verified!.verified.resolve()
|
||||
}
|
||||
let nextFlow: { side: Side; value: Flow["baseline"] } | undefined
|
||||
const checkedRead = async (side: Side) => {
|
||||
const value = await readFlow(side)
|
||||
if (slot?.supplied) nextFlow = { side, value }
|
||||
return value
|
||||
}
|
||||
const native = nativeFor()
|
||||
const topology = createChrTransitTopologyOwner({
|
||||
scope, linux: options.linux, deadlineAt, now, authorize, assertOutsideScope,
|
||||
routers: { A: { ...routers.A, native }, B: { ...routers.B, native } },
|
||||
writeAdmission: broker, checkpoint,
|
||||
assertFlowRestored: async () => { await expectFlow("A"); await expectFlow("B"); require(isDeepStrictEqual(expected, baseline)) },
|
||||
})
|
||||
const flows = Object.fromEntries(sides.map(side => [side, createDisabledChrFlowHarness({
|
||||
baseline: baseline[side], transitInterface: "nf" + plan.runId + (side === "A" ? "ea" : "eb"),
|
||||
authorize, read: () => checkedRead(side), native, checkpoint, relayWrite: { side, admission: broker },
|
||||
})])) as Record<Side, ReturnType<typeof createDisabledChrFlowHarness>>
|
||||
|
||||
const workflow = async () => {
|
||||
await topology.acquire()
|
||||
const targets: Partial<Record<Side, ReturnType<typeof createChrTargetHarness>>> = {}
|
||||
for (const side of sides) {
|
||||
const active = await flows[side].enable()
|
||||
const targetOptions = {
|
||||
baseline: active, source: side === "A" ? plan.address.exportA : plan.address.exportB,
|
||||
destination: plan.address.collector, port: plan.ports.collector,
|
||||
authorize, read: () => checkedRead(side), native, checkpoint, relayWrite: { side, admission: broker },
|
||||
}
|
||||
const target = stagedTarget
|
||||
? createChrTargetHarness({ ...targetOptions, stagedDisabled: true })
|
||||
: createChrTargetHarness({ ...targetOptions, timerProfile })
|
||||
await target.create()
|
||||
targets[side] = target
|
||||
}
|
||||
if (stagedTarget) for (const side of sides) {
|
||||
await requested[cursor].promise
|
||||
live()
|
||||
const timer = await stagedTarget.captureTimer()
|
||||
require(timer.kind === "staged-ipfix-timeout-20s" && timer.captureDeadlineAt <= deadlineAt)
|
||||
await targets[side]!.enable(timer)
|
||||
}
|
||||
for (const side of [...sides].reverse()) {
|
||||
await targets[side]!.remove()
|
||||
await flows[side].restore()
|
||||
}
|
||||
await topology.restore()
|
||||
await expectFlow("A")
|
||||
await expectFlow("B")
|
||||
require(cursor === labels.length && isDeepStrictEqual(expected, baseline))
|
||||
}
|
||||
let completed: Promise<void> | undefined
|
||||
const retire = () => abort(Error("Parent native stage retired"))
|
||||
const stages: Stage[] = labels.map((step, index) => Object.freeze({
|
||||
step,
|
||||
async intent() {
|
||||
try {
|
||||
live()
|
||||
require(cursor === index)
|
||||
if (!launched) {
|
||||
launched = true
|
||||
timer = setTimeout(retire, Math.max(1, deadlineAt - now()))
|
||||
completed = workflow()
|
||||
void completed.then(() => clearTimeout(timer), abort)
|
||||
}
|
||||
requested[index].resolve()
|
||||
const entry = await ready[index].promise
|
||||
live()
|
||||
require(slot === entry && !entry.requested)
|
||||
entry.requested = true
|
||||
return structuredClone(entry.intent)
|
||||
} catch (error) { abort(error); throw error }
|
||||
},
|
||||
async verify(intent: Readonly<ChrWriteIntent>) {
|
||||
try {
|
||||
live()
|
||||
const entry = slot
|
||||
require(cursor === index && entry?.requested && !entry.supplied && isDeepStrictEqual(entry.intent, intent))
|
||||
const ack = await takeAcknowledgement(intent)
|
||||
live()
|
||||
require(ack && isDeepStrictEqual(ack.intent, intent) &&
|
||||
(intent.method === "PUT" ? [200, 201].includes(ack.status) :
|
||||
intent.step === "target:" + intent.side + ":enable" ? ack.status === 200 : [200, 204].includes(ack.status)))
|
||||
entry!.supplied = true
|
||||
entry!.acknowledgement.resolve(structuredClone(ack.body))
|
||||
await entry!.verified.promise
|
||||
if (index === labels.length - 1) await completed
|
||||
live()
|
||||
} catch (error) { abort(error); throw error }
|
||||
},
|
||||
}))
|
||||
return Object.freeze({ profile, stages: Object.freeze(stages), retire })
|
||||
}
|
||||
|
||||
|
||||
/** Connect directly to the owned parent relay. Each response is disposable and
|
||||
* usable once. Only exact consumed gate intent and projected ACK travel here. */
|
||||
export function createChrParentAckMailbox() {
|
||||
let pending: ChrParentNativeAck | undefined, retired = false
|
||||
return Object.freeze({
|
||||
record(ack: ChrParentNativeAck) {
|
||||
if (retired || pending) { retired = true; pending = undefined; return refuse() }
|
||||
pending = structuredClone(ack)
|
||||
},
|
||||
async take(intent: Readonly<ChrWriteIntent>) {
|
||||
const ack = pending
|
||||
pending = undefined
|
||||
if (retired || !ack || !isDeepStrictEqual(ack.intent, intent)) { retired = true; return refuse() }
|
||||
return ack
|
||||
},
|
||||
retire() { retired = true; pending = undefined },
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { Duplex, PassThrough } from "node:stream"
|
||||
import { serveChrParentNativeBoundary } from "./traffic-flow-chr-parent-native-boundary.js"
|
||||
import { chrTransitNames } from "./traffic-flow-chr-transit-fixture.js"
|
||||
|
||||
|
||||
const runId = "a012bcde"
|
||||
const ports = { linuxA: 50101, edgeA: 50102, middleA: 50103, middleB: 50104, edgeB: 50105, linuxB: 50106,
|
||||
collector: 50107, calibrationSource: 50112, calibrationDestination: 50113, phase1Source: 50108, phase1Destination: 50109, phase2Source: 50110, phase2Destination: 50111 }
|
||||
const directions = ["A-to-B", "B-to-A", "export-A", "export-B", "underlay-linuxA-to-A", "underlay-A-to-linuxA",
|
||||
"underlay-A-to-B", "underlay-B-to-A", "underlay-B-to-linuxB", "underlay-linuxB-to-B"] as const
|
||||
const isolation = { checkedPorts: Object.values(ports), checkedNames: Object.values(chrTransitNames(runId)),
|
||||
checkedCidrs: ["198.18.240.1/32", "198.18.240.2/32", "198.18.241.0/30", "198.18.241.4/30", "198.18.241.8/30", "198.18.242.0/29"],
|
||||
occupiedPorts: [], overlappingCidrs: [], occupiedNames: [],
|
||||
policies: directions.map(direction => ({ direction, verdict: "allow", nat: "none", fastTrack: "none", ambiguousRules: 0 } as const)) }
|
||||
import { createChrParentOrchestration } from "./traffic-flow-chr-parent-orchestration.js"
|
||||
import { createChrChildAdmissionPipe } from "./traffic-flow-chr-write-admission-ipc.js"
|
||||
import { parseChrVrfPrefix } from "./chr-vrf-companion-address-space.js"
|
||||
import type { ChrWriteIntent } from "./traffic-flow-chr-write-admission.js"
|
||||
|
||||
const admittedMenus = ["/interface/wireguard", "/ip/address", "/interface/wireguard/peers", "/ip/route", "/ip/traffic-flow/target"]
|
||||
const roles = ["edgeA", "middleA", "middleB", "edgeB"]
|
||||
const roleSuffix: Record<string, string> = { edgeA: "ea", middleA: "ma", middleB: "mb", edgeB: "eb" }
|
||||
const names = Object.fromEntries(roles.map(role => [role, "nf" + runId + roleSuffix[role]]))
|
||||
const marker = "netflow-" + runId
|
||||
const exportFields = ["bytes", "packets", "src-address", "dst-address", "protocol", "src-port", "dst-port", "in-interface", "out-interface"]
|
||||
const singleton = "/ip/traffic-flow", ipfix = singleton + "/ipfix", target = singleton + "/target"
|
||||
type Row = Record<string, unknown>
|
||||
type Side = "A" | "B"
|
||||
|
||||
function state(side: Side) {
|
||||
return {
|
||||
[singleton]: [{ ...(side === "B" ? { ".id": "*F" } : {}), enabled: "no", interfaces: "all", "inactive-flow-timeout": "15s" }],
|
||||
[ipfix]: [Object.fromEntries(exportFields.map(key => [key, "no"]))],
|
||||
"/interface": [{ ".id": "*F0", name: "ether1", type: "ether" }],
|
||||
} as Record<string, Row[]>
|
||||
}
|
||||
|
||||
export async function runChrParentSynthetic(mode: "direct" | "staged" | "wrong-ack", density: "empty" | "dense" = "empty") {
|
||||
const staged = mode !== "direct"
|
||||
const phaseCalls: string[] = []
|
||||
const rows = { A: state("A"), B: state("B") }, baseline = structuredClone(rows)
|
||||
const stressPaths = new Set(["/ip/firewall/filter", "/ip/firewall/nat", "/ip/firewall/raw", "/ip/firewall/mangle",
|
||||
"/ipv6/firewall/filter", "/ipv6/firewall/nat", "/ipv6/firewall/raw", "/ipv6/firewall/mangle"])
|
||||
const denseBody = density === "dense" ? JSON.stringify(Array.from({ length: 700 }, (_, i) => ({
|
||||
".id": "*" + (i + 4096).toString(16), chain: "forward", action: "accept", disabled: "true",
|
||||
comment: "synthetic-stress-" + i + "-" + "x".repeat(1150),
|
||||
}))) : ""
|
||||
const catalogueFixtureBytesPerSide = Buffer.byteLength(denseBody) * stressPaths.size
|
||||
const now = Date.now()
|
||||
let writes = 0, staging = false, ready = false, drained = false, inverse = false
|
||||
const reads: Record<Side, number> = { A: 0, B: 0 }
|
||||
const makeRouter = (side: Side, endpoint: string) => ({
|
||||
endpoint, deviceId: side,
|
||||
client: { async get<T>(path: string, _timeout?: number, _signal?: AbortSignal, _max?: number, before?: () => Promise<unknown>) {
|
||||
await before?.()
|
||||
reads[side]++
|
||||
if (denseBody && stressPaths.has(path)) return JSON.parse(denseBody) as T
|
||||
return JSON.parse(JSON.stringify(rows[side][path] ?? [])) as T
|
||||
} },
|
||||
})
|
||||
const orchestration = await createChrParentOrchestration({
|
||||
scope: [runId, ports, isolation], deadlineAt: now + (density === "dense" ? 580000 : 100000),
|
||||
linux: { endpoint: "192.0.2.3", publicKeys: { linuxA: "X".repeat(43) + "=", linuxB: "Y".repeat(43) + "=" } },
|
||||
routers: { A: makeRouter("A", "192.0.2.1"), B: makeRouter("B", "192.0.2.2") },
|
||||
...(staged ? { stagedTarget: { async captureTimer() {
|
||||
assert(ready)
|
||||
return { kind: "staged-ipfix-timeout-20s" as const, captureDeadlineAt: Math.min(Date.now() + 180000, now + (density === "dense" ? 580000 : 100000)) }
|
||||
} } } : { timerProfile: { kind: "bounded-ipfix-timeout-20s" as const, captureDeadlineAt: now + 89000 } }),
|
||||
assertTransitAndCollectorStagingReady: async () => { phaseCalls.push("staging"); assert(staging && !ready) },
|
||||
authorize: async () => {}, assertTimingPreflight: async () => { phaseCalls.push("preflight") }, assertRouterIdentity: async (side, id) => assert.equal(side, id),
|
||||
assertTransitAndCollectorReady: async () => { phaseCalls.push("ready"); assert(ready) },
|
||||
assertCaptureAndDrainComplete: async () => { phaseCalls.push("drained"); assert(drained) },
|
||||
assertLinuxInverseReady: async () => { phaseCalls.push("inverse"); assert(inverse) },
|
||||
})
|
||||
const toParent = new PassThrough(), toChild = new PassThrough()
|
||||
const done = orchestration.serve({ input: toParent, output: toChild })
|
||||
void done.catch(() => {})
|
||||
const child = createChrChildAdmissionPipe({ input: toChild, output: toParent }, staged ? "staged-disabled-target" : "direct-target")
|
||||
const nativeInput = new PassThrough(), nativeOutput = new PassThrough()
|
||||
const nativeStream = Duplex.from({ readable: nativeInput, writable: nativeOutput })
|
||||
const physicalBoundary = serveChrParentNativeBoundary(nativeStream, orchestration, async () => {})
|
||||
void physicalBoundary.catch(() => { nativeStream.destroy(); nativeOutput.destroy() })
|
||||
let nativeSequence = 0
|
||||
const physicalExchange = async (data: object) => {
|
||||
const sequence = ++nativeSequence
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const receive = (reply: Buffer) => {
|
||||
nativeOutput.off("close", closed)
|
||||
try { assert.deepEqual(JSON.parse(reply.toString()), { sequence, ok: true }); resolve() }
|
||||
catch (error) { reject(error) }
|
||||
}
|
||||
const closed = () => { nativeOutput.off("data", receive); reject(Error("Synthetic native boundary closed")) }
|
||||
nativeOutput.once("data", receive)
|
||||
nativeOutput.once("close", closed)
|
||||
nativeInput.write(JSON.stringify({ sequence, ...data }) + "\n")
|
||||
})
|
||||
}
|
||||
const owned = new Map<string, { intent: ChrWriteIntent; id: string }>()
|
||||
const execute = async (intent: ChrWriteIntent) => {
|
||||
await child.run(intent, async arm => {
|
||||
await arm()
|
||||
const projection = { side: intent.side, method: intent.method, path: intent.path }
|
||||
await physicalExchange({ op: "consume", ...projection, body: intent.body })
|
||||
const catalogue = rows[intent.side]
|
||||
const id = "*" + (++writes).toString(16)
|
||||
const slash = intent.path.lastIndexOf("/")
|
||||
const menu = intent.method === "PUT" ? intent.path : intent.path.slice(0, slash)
|
||||
const list = catalogue[menu] ??= []
|
||||
let body: unknown
|
||||
if (intent.method === "PUT") {
|
||||
assert(admittedMenus.includes(menu))
|
||||
const row: Row = { ...intent.body, ".id": id }
|
||||
if (menu === "/interface/wireguard") {
|
||||
row["public-key"] = String.fromCharCode(64 + writes).repeat(43) + "="
|
||||
catalogue["/interface"].push({ ".id": id, name: row.name, type: "wireguard", comment: marker, disabled: false })
|
||||
}
|
||||
if (menu === "/ip/address") {
|
||||
row.dynamic = false
|
||||
row.vrf = "main"
|
||||
const prefix = parseChrVrfPrefix(row.address, 4)!
|
||||
const ip = [24n, 16n, 8n, 0n].map(shift => Number((prefix.first >> shift) & 255n)).join(".")
|
||||
const route = { ".id": "*a" + id.slice(1), "dst-address": ip + "/" + prefix.prefixLength, gateway: row.interface,
|
||||
"routing-table": "main", connect: true, distance: "0", scope: "10", "target-scope": "5" }
|
||||
;(catalogue["/ip/route"] ??= []).push({ ...route, dynamic: true })
|
||||
;(catalogue["/routing/route"] ??= []).push({ ...route, afi: "ip4", "belongs-to": "connected",
|
||||
"local-address": String(row.address).split("/")[0] + "%" + row.interface })
|
||||
}
|
||||
if (menu === target) row.disabled = staged ? "yes" : false
|
||||
list.push(row)
|
||||
owned.set(intent.step, { intent, id })
|
||||
body = { ".id": id }
|
||||
} else if (intent.method === "DELETE") {
|
||||
const previous = decodeURIComponent(intent.path.slice(slash + 1))
|
||||
assert.equal(list.filter(row => row[".id"] === previous).length, 1)
|
||||
catalogue[menu] = list.filter(row => row[".id"] !== previous)
|
||||
if (menu === "/interface/wireguard") catalogue["/interface"] = catalogue["/interface"].filter(row => row[".id"] !== previous)
|
||||
if (menu === "/ip/address") for (const path of ["/ip/route", "/routing/route"]) {
|
||||
catalogue[path] = catalogue[path].filter(row => row[".id"] !== "*a" + previous.slice(1))
|
||||
}
|
||||
} else if (menu === target) {
|
||||
const previous = decodeURIComponent(intent.path.slice(slash + 1))
|
||||
const matching = list.filter(row => row[".id"] === previous)
|
||||
assert.equal(matching.length, 1)
|
||||
assert.equal(matching[0].disabled, "yes")
|
||||
Object.assign(matching[0], intent.body)
|
||||
body = { ".id": mode === "wrong-ack" ? "*FFFF" : previous }
|
||||
} else Object.assign(list[0], intent.body)
|
||||
await physicalExchange({ op: "ack", ...projection, status: intent.method === "PUT" ? 201 : body ? 200 : 204,
|
||||
...(body && typeof body === "object" ? { id: (body as Row)[".id"] } : {}) })
|
||||
})
|
||||
}
|
||||
const send = (step: string, side: Side, method: ChrWriteIntent["method"], path: string, body: ChrWriteIntent["body"]) =>
|
||||
execute({ step, side, deviceId: side, method, path, body })
|
||||
const create = (key: string, side: Side, path: string, body: Record<string, string>) =>
|
||||
send("topology:create:" + side + ":" + key, side, "PUT", path, { ...body, disabled: "false", comment: marker })
|
||||
const addr: Record<string, string> = { edgeA: "198.18.241.2/30", middleA: "198.18.241.5/30", middleB: "198.18.241.6/30", edgeB: "198.18.241.9/30" }
|
||||
const sideOf = (role: string): Side => role.endsWith("A") ? "A" : "B"
|
||||
const portOf = (role: string) => String(ports[role as keyof typeof ports])
|
||||
try {
|
||||
for (const role of roles) await create(role, sideOf(role), "/interface/wireguard", { name: names[role], "listen-port": portOf(role), mtu: "1420" })
|
||||
for (const role of roles) await create("address:" + role, sideOf(role), "/ip/address", { interface: names[role], address: addr[role] })
|
||||
const peerSpecs = [
|
||||
["edgeA", "linuxA", "192.0.2.3", "X", "198.18.240.1/32,198.18.242.3/32"],
|
||||
["middleA", "middleB", "192.0.2.2", "C", "198.18.240.2/32"],
|
||||
["middleB", "middleA", "192.0.2.1", "B", "198.18.240.1/32"],
|
||||
["edgeB", "linuxB", "192.0.2.3", "Y", "198.18.240.2/32,198.18.242.3/32"],
|
||||
]
|
||||
for (const [role, opposite, endpoint, key, allowed] of peerSpecs) await create("peer:" + role, sideOf(role), "/interface/wireguard/peers", {
|
||||
interface: names[role], "public-key": key.repeat(43) + "=", "allowed-address": allowed,
|
||||
"endpoint-address": endpoint, "endpoint-port": portOf(opposite), "persistent-keepalive": "10",
|
||||
})
|
||||
const routes = [
|
||||
["edgeA", "198.18.240.1", "198.18.241.1"], ["middleA", "198.18.240.2", "198.18.241.6"], ["edgeA", "198.18.242.3", "198.18.241.1"],
|
||||
["edgeB", "198.18.240.2", "198.18.241.10"], ["middleB", "198.18.240.1", "198.18.241.5"], ["edgeB", "198.18.242.3", "198.18.241.10"],
|
||||
]
|
||||
for (const [i, [role, destination, gateway]] of routes.entries()) await create("route:" + i, sideOf(role), "/ip/route", {
|
||||
"dst-address": destination + "/32", gateway: gateway + "%" + names[role], "routing-table": "main",
|
||||
})
|
||||
staging = staged
|
||||
ready = !staged
|
||||
const changeIpfix = (side: Side, enable: boolean) => send("flow:" + side + ":ipfix." + (enable ? "enable" : "restore"),
|
||||
side, "POST", ipfix + "/set", Object.fromEntries(exportFields.map(key => [key, enable ? "yes" : "no"])))
|
||||
const changeFlow = (side: Side, enable: boolean) => send("flow:" + side + ":flow." + (enable ? "enable" : "restore"),
|
||||
side, side === "B" ? "PATCH" : "POST", singleton + (side === "B" ? "/*F" : "/set"),
|
||||
{ enabled: enable ? "yes" : "no", interfaces: enable ? names["edge" + side] : "all" })
|
||||
for (const side of ["A", "B"] as const) {
|
||||
await changeIpfix(side, true)
|
||||
await changeFlow(side, true)
|
||||
await send("target:" + side + ":create", side, "PUT", target, { "dst-address": "198.18.242.3",
|
||||
"src-address": side === "A" ? "198.18.241.2" : "198.18.241.9", port: portOf("collector"), version: "ipfix", "v9-template-timeout": "20s", ...(staged ? { disabled: "yes" } : {}) })
|
||||
}
|
||||
if (staged) {
|
||||
ready = true
|
||||
for (const side of ["A", "B"] as const) {
|
||||
const enabling = send("target:" + side + ":enable", side, "PATCH", target + "/" + owned.get("target:" + side + ":create")!.id, { disabled: "no" })
|
||||
if (mode === "wrong-ack") {
|
||||
await assert.rejects(enabling)
|
||||
await assert.rejects(done)
|
||||
assert.equal(writes, 25)
|
||||
assert.equal(rows.B[target][0].disabled, "yes")
|
||||
return { completed: 0, writes, reads, catalogueFixtureBytesPerSide, nativeFrames: nativeSequence }
|
||||
}
|
||||
await enabling
|
||||
}
|
||||
}
|
||||
drained = true
|
||||
for (const side of ["B", "A"] as const) {
|
||||
await send("target:" + side + ":delete", side, "DELETE", target + "/" + owned.get("target:" + side + ":create")!.id, {})
|
||||
await changeFlow(side, false)
|
||||
await changeIpfix(side, false)
|
||||
}
|
||||
inverse = true
|
||||
for (const { intent, id } of [...owned.values()].reverse().filter(value => value.intent.step.startsWith("topology:"))) {
|
||||
await send(intent.step.replace(":create:", ":delete:"), intent.side, "DELETE", intent.path + "/" + id, {})
|
||||
}
|
||||
toParent.end()
|
||||
assert.deepEqual(await done, { completed: staged ? 50 : 48 })
|
||||
assert.equal(writes, staged ? 50 : 48)
|
||||
assert(reads.A > 100 && reads.B > 100)
|
||||
assert.deepEqual([...new Set(phaseCalls)], ["preflight", ...(staged ? ["staging"] : []), "ready", "drained", "inverse"])
|
||||
for (const side of ["A", "B"] as const) {
|
||||
for (const [path, records] of Object.entries(rows[side])) assert.deepEqual(records, baseline[side][path] ?? [])
|
||||
}
|
||||
assert.equal(nativeSequence, writes * 2)
|
||||
return { completed: writes, writes, reads, catalogueFixtureBytesPerSide, nativeFrames: nativeSequence }
|
||||
} finally {
|
||||
nativeStream.destroy()
|
||||
nativeInput.destroy()
|
||||
nativeOutput.destroy()
|
||||
toParent.destroy()
|
||||
toChild.destroy()
|
||||
orchestration.retire()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { createHash } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import { inspectChrReadiness } from "./traffic-flow-chr-readiness.js"
|
||||
|
||||
const now = 1700000005000
|
||||
function fixture() {
|
||||
const devices = [1, 2].map(serverId => ({
|
||||
managementAddress: "198.51.100." + serverId, deviceId: "fixture-" + serverId,
|
||||
version: "7.24.5 (stable)", exporterSource: "192.0.2." + serverId, serverId,
|
||||
}))
|
||||
const pins = { vpsHostKeySha256: "a".repeat(64), collectorInstance: "owned-backend", collectorAddress: "192.0.2.100",
|
||||
interfaceName: "test0", testPort: 4739, baselinePort: 4740, devices }
|
||||
const evidence = { observedAt: now - 1, vpsHostKeySha256: pins.vpsHostKeySha256, collectorInstance: pins.collectorInstance,
|
||||
interfaceName: pins.interfaceName, interfaceAddresses: [pins.collectorAddress], collectorPid: 42,
|
||||
udpOwners: [{ pid: 42, address: pins.collectorAddress, port: pins.testPort }],
|
||||
worker: { alive: true, bound: true, address: "192.0.2.100:4739" },
|
||||
settings: { enabled: true, collectorIp: pins.collectorAddress, flowListenPort: pins.testPort,
|
||||
listenerBound: true, listenerAddress: "192.0.2.100:4739" }, devices: structuredClone(devices) }
|
||||
return { pins, evidence }
|
||||
}
|
||||
function capture() {
|
||||
const bytes = Buffer.alloc(24 + 2 * 60)
|
||||
bytes.writeUInt32LE(0xa1b2c3d4)
|
||||
bytes.writeUInt16LE(2, 4); bytes.writeUInt16LE(4, 6)
|
||||
bytes.writeUInt32LE(4124, 16); bytes.writeUInt32LE(101, 20)
|
||||
for (const [index, at] of [24, 84].entries()) {
|
||||
bytes.writeUInt32LE(1700000000, at); bytes.writeUInt32LE(44, at + 8); bytes.writeUInt32LE(44, at + 12)
|
||||
const ip = bytes.subarray(at + 16, at + 60)
|
||||
ip[0] = 0x45; ip.writeUInt16BE(44, 2); ip[9] = 17
|
||||
Buffer.from([192, 0, 2, index + 1, 192, 0, 2, 100]).copy(ip, 12)
|
||||
ip.writeUInt16BE(2055, 20); ip.writeUInt16BE(4740, 22); ip.writeUInt16BE(24, 24)
|
||||
ip.writeUInt16BE(10, 28); ip.writeUInt16BE(16, 30)
|
||||
}
|
||||
return { bytes, sha256: createHash("sha256").update(bytes).digest("hex") }
|
||||
}
|
||||
|
||||
test("pinned identity and owned bind do not fill a missing ingress observation", () => {
|
||||
const { pins, evidence } = fixture()
|
||||
const result = inspectChrReadiness(pins, evidence, undefined, now)
|
||||
assert.equal(result.identityMatched, true)
|
||||
assert.equal(result.collectorBindMatched, true)
|
||||
assert.equal(result.ingressStatus, "missing")
|
||||
assert.equal(result.readyForReview, false)
|
||||
assert.deepEqual(result.targetScopes.map(row => [row.deviceId, row.fields["src-address"], row.fields.port]), [
|
||||
["fixture-1", "192.0.2.1", "4739"], ["fixture-2", "192.0.2.2", "4739"],
|
||||
])
|
||||
})
|
||||
|
||||
test("requires fresh pinned observations, exact listener address and exclusive UDP owner", () => {
|
||||
for (const mutate of [
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.devices[0].version = "different" },
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.observedAt = now - 120001 },
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.udpOwners[0].pid = 999 },
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.udpOwners[0].address = "0.0.0.0" },
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.udpOwners.push({ ...f.evidence.udpOwners[0] }) },
|
||||
(f: ReturnType<typeof fixture>) => { f.evidence.worker.alive = false },
|
||||
]) {
|
||||
const f = fixture(); mutate(f)
|
||||
assert.equal(inspectChrReadiness(f.pins, f.evidence, capture(), now).readyForReview, false)
|
||||
}
|
||||
})
|
||||
|
||||
test("two-source scoped baseline pcap is checked separately from the owned test port", () => {
|
||||
const { pins, evidence } = fixture(), input = capture()
|
||||
const result = inspectChrReadiness(pins, evidence, input, now)
|
||||
assert.equal(result.ingressStatus, "matched")
|
||||
assert.equal(result.readyForReview, true)
|
||||
assert.equal(result.actualNativeAcceptance, false)
|
||||
assert.equal(inspectChrReadiness(pins, evidence, { ...input, sha256: "0".repeat(64) }, now).ingressStatus, "refused")
|
||||
assert.equal(inspectChrReadiness(pins, { ...evidence, observedAt: now + 600001 }, input, now + 600001).ingressStatus, "refused")
|
||||
})
|
||||
@@ -0,0 +1,104 @@
|
||||
/** Local preflight report from reviewed VPS/CHR observations and optional pcap. */
|
||||
import { createHash } from "node:crypto"
|
||||
import { open } from "node:fs/promises"
|
||||
import { isIPv4 } from "node:net"
|
||||
import { resolve } from "node:path"
|
||||
import { fileURLToPath } from "node:url"
|
||||
import { isDeepStrictEqual } from "node:util"
|
||||
import { z } from "zod"
|
||||
import { projectChrWire } from "./traffic-flow-chr-wire.js"
|
||||
|
||||
const ip = z.string().max(15).refine(isIPv4)
|
||||
const port = z.number().int().min(1).max(65535)
|
||||
const id = z.string().min(1).max(128)
|
||||
const sha = z.string().regex(/^[a-f0-9]{64}$/)
|
||||
const device = z.object({ managementAddress: ip, deviceId: id, version: id, exporterSource: ip, serverId: z.number().int().positive().safe() }).strict()
|
||||
const pinsSchema = z.object({
|
||||
vpsHostKeySha256: sha, collectorInstance: id, collectorAddress: ip, interfaceName: z.string().regex(/^[A-Za-z0-9_.-]{1,15}$/),
|
||||
testPort: port, baselinePort: port, devices: z.tuple([device, device]),
|
||||
}).strict()
|
||||
const evidenceSchema = z.object({
|
||||
observedAt: z.number().int().nonnegative().safe(),
|
||||
vpsHostKeySha256: sha, collectorInstance: id, interfaceName: z.string().max(15),
|
||||
interfaceAddresses: z.array(ip).max(32),
|
||||
collectorPid: z.number().int().positive().max(0x7fffffff),
|
||||
udpOwners: z.array(z.object({ pid: z.number().int().positive().max(0x7fffffff), address: ip, port }).strict()).max(16),
|
||||
worker: z.object({ alive: z.boolean(), bound: z.boolean(), address: z.string().max(32).nullable() }).strict(),
|
||||
settings: z.object({ enabled: z.boolean(), collectorIp: ip, flowListenPort: port, listenerBound: z.boolean(), listenerAddress: z.string().max(32).nullable() }).strict(),
|
||||
devices: z.tuple([device, device]),
|
||||
}).strict()
|
||||
|
||||
type Ingress = { bytes: Buffer; sha256: string }
|
||||
export function inspectChrReadiness(rawPins: unknown, rawEvidence: unknown, ingress?: Ingress, now = Date.now()) {
|
||||
const pins = pinsSchema.parse(rawPins), evidence = evidenceSchema.parse(rawEvidence)
|
||||
const distinct = new Set([...pins.devices.map(row => row.exporterSource), pins.collectorAddress]).size === 3 &&
|
||||
new Set(pins.devices.map(row => row.deviceId)).size === 2 && new Set(pins.devices.map(row => row.managementAddress)).size === 2 &&
|
||||
new Set(pins.devices.map(row => row.serverId)).size === 2 && pins.testPort !== pins.baselinePort
|
||||
const fresh = evidence.observedAt <= now && now - evidence.observedAt <= 120000
|
||||
const identityMatched = distinct && fresh && pins.vpsHostKeySha256 === evidence.vpsHostKeySha256 &&
|
||||
pins.collectorInstance === evidence.collectorInstance && isDeepStrictEqual(pins.devices, evidence.devices)
|
||||
const endpoint = pins.collectorAddress + ":" + pins.testPort
|
||||
const owners = evidence.udpOwners.filter(row => row.port === pins.testPort)
|
||||
const collectorBindMatched = identityMatched && pins.interfaceName === evidence.interfaceName &&
|
||||
evidence.interfaceAddresses.includes(pins.collectorAddress) && owners.length === 1 &&
|
||||
owners[0].pid === evidence.collectorPid && owners[0].address === pins.collectorAddress &&
|
||||
evidence.worker.alive && evidence.worker.bound && evidence.worker.address === endpoint &&
|
||||
evidence.settings.enabled && evidence.settings.collectorIp === pins.collectorAddress &&
|
||||
evidence.settings.flowListenPort === pins.testPort && evidence.settings.listenerBound && evidence.settings.listenerAddress === endpoint
|
||||
let ingressStatus: "missing" | "refused" | "matched" = ingress ? "refused" : "missing"
|
||||
if (identityMatched && ingress) {
|
||||
try {
|
||||
const wire = projectChrWire(ingress.bytes, {
|
||||
sources: [pins.devices[0].exporterSource, pins.devices[1].exporterSource], destination: pins.collectorAddress, port: pins.baselinePort,
|
||||
})
|
||||
if (wire.captureSha256 === ingress.sha256 &&
|
||||
wire.frames.every(frame => frame.receivedMs <= now && frame.receivedMs >= now - 600000)) ingressStatus = "matched"
|
||||
} catch {
|
||||
// Keep the failed evidence state explicit without including packet data.
|
||||
}
|
||||
}
|
||||
return {
|
||||
identityMatched, collectorBindMatched, ingressStatus,
|
||||
readyForReview: collectorBindMatched && ingressStatus === "matched",
|
||||
targetScopes: pins.devices.map(row => ({ deviceId: row.deviceId, serverId: row.serverId, method: "PUT", path: "/ip/traffic-flow/target",
|
||||
fields: { "dst-address": pins.collectorAddress, "src-address": row.exporterSource, port: String(pins.testPort), version: "ipfix" } })),
|
||||
actualNativeAcceptance: false,
|
||||
}
|
||||
}
|
||||
|
||||
async function pinnedFile(path: string, digest: string, max: number) {
|
||||
sha.parse(digest)
|
||||
const file = await open(path, "r")
|
||||
try {
|
||||
const info = await file.stat()
|
||||
if (!info.isFile() || info.size < 1 || info.size > max) throw Error("Input outside file bound")
|
||||
const bytes = Buffer.alloc(info.size + 1)
|
||||
let size = 0
|
||||
while (size < bytes.length) {
|
||||
const { bytesRead } = await file.read(bytes, size, bytes.length - size, size)
|
||||
if (!bytesRead) break
|
||||
size += bytesRead
|
||||
}
|
||||
const body = bytes.subarray(0, size)
|
||||
if (size !== info.size || createHash("sha256").update(body).digest("hex") !== digest) throw Error("Input digest differs")
|
||||
return body
|
||||
} finally {
|
||||
await file.close()
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = process.argv.slice(2)
|
||||
if (args.length !== 4 && args.length !== 6) throw Error("Expected pinned inputs")
|
||||
const [pins, pinsSha, evidence, evidenceSha, capture, captureSha] = args
|
||||
const report = inspectChrReadiness(
|
||||
JSON.parse((await pinnedFile(pins, pinsSha, 16384)).toString("utf8")),
|
||||
JSON.parse((await pinnedFile(evidence, evidenceSha, 16384)).toString("utf8")),
|
||||
capture ? { bytes: await pinnedFile(capture, captureSha, 512 * 1024), sha256: captureSha } : undefined,
|
||||
)
|
||||
console.log(JSON.stringify(report))
|
||||
if (!report.readyForReview) process.exitCode = 2
|
||||
}
|
||||
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
main().catch(() => { console.error("CHR readiness inputs refused"); process.exitCode = 1 })
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { test } from "node:test"
|
||||
import { createChrRouteConntrackReader } from "./traffic-flow-chr-route-conntrack.js"
|
||||
|
||||
function fixture() {
|
||||
const calls: string[] = []
|
||||
const catalog: Record<string, Record<string, unknown>[]> = {
|
||||
"/system/identity": [{ name: "chr-a" }],
|
||||
"/system/resource": [{ version: "7.24.5 (stable)" }],
|
||||
"/ip/settings": [{ "ip-forward": "true", "allow-fast-path": "false" }],
|
||||
"/routing/rule": [],
|
||||
"/ip/vrf": [{ ".id": "*1", name: "main", interfaces: "all" }],
|
||||
"/routing/table": [{ ".id": "*1", name: "main", fib: "true" }],
|
||||
"/interface": [{ ".id": "*2", name: "nfedge", type: "wg", running: "true", disabled: "false" }],
|
||||
"/ip/address": [{ ".id": "*3", address: "198.18.241.2/30", interface: "nfedge", disabled: "false" }],
|
||||
"/routing/route": [{ ".id": "*4", "routing-table": "main", "dst-address": "198.18.240.2/32",
|
||||
gateway: "198.18.241.1", "immediate-gw": "198.18.241.1%nfedge", active: "true", disabled: "false" }],
|
||||
"/ip/firewall/connection/tracking": [{ enabled: "yes" }],
|
||||
"/ip/firewall/connection": [],
|
||||
}
|
||||
const config = {
|
||||
identity: "chr-a", version: "7.24.5 (stable)",
|
||||
routes: [{ routeId: "*4", destination: "198.18.240.2", nextHop: "198.18.241.1",
|
||||
interfaceName: "nfedge", interfaceId: "*2", addressId: "*3", interfaceCidr: "198.18.241.2/30" }],
|
||||
udp: [{ source: { address: "198.18.240.1", port: 50001 }, destination: { address: "198.18.240.2", port: 50002 } }],
|
||||
assertCurrent: async () => {},
|
||||
client: { async get<T>(path: string, timeout?: number, _signal?: AbortSignal, bytes?: number, fence?: () => Promise<unknown>) {
|
||||
assert.ok(timeout && timeout <= 1800)
|
||||
assert.equal(bytes, 512 * 1024)
|
||||
await fence?.()
|
||||
calls.push(path)
|
||||
assert.ok(Object.hasOwn(catalog, path))
|
||||
return structuredClone(catalog[path]) as T
|
||||
} },
|
||||
}
|
||||
return { calls, catalog, config, read: createChrRouteConntrackReader(config) }
|
||||
}
|
||||
const foreignUdp = { ".id": "*1", protocol: "udp", "src-address": "192.0.2.1:50001", "dst-address": "192.0.2.2:50002",
|
||||
"reply-src-address": "192.0.2.2:50002", "reply-dst-address": "192.0.2.1:50001" }
|
||||
|
||||
test("owned two-interface profile permits observed FastPath flag and auto only with empty conntrack", async () => {
|
||||
const { catalog, config } = fixture()
|
||||
const [route] = config.routes
|
||||
const edge = "nfa012bcdeea", middle = "nfa012bcdema"
|
||||
const owned = createChrRouteConntrackReader({ ...config, profile: "owned-wg-empty-firewall", routes: [
|
||||
{ ...route, interfaceName: edge },
|
||||
{ ...route, interfaceName: middle, interfaceId: "*5", addressId: "*6", routeId: "*7",
|
||||
interfaceCidr: "198.18.241.5/30", destination: "198.18.240.1", nextHop: "198.18.241.6" },
|
||||
] })
|
||||
catalog["/ip/settings"][0]["allow-fast-path"] = "true"
|
||||
catalog["/ip/firewall/connection/tracking"][0].enabled = "auto"
|
||||
catalog["/interface"][0].name = edge
|
||||
catalog["/interface"].push({ ...catalog["/interface"][0], ".id": "*5", name: middle })
|
||||
catalog["/ip/address"][0].interface = edge
|
||||
catalog["/ip/address"].push({ ...catalog["/ip/address"][0], ".id": "*6", interface: middle, address: "198.18.241.5/30" })
|
||||
catalog["/routing/route"][0]["immediate-gw"] = "198.18.241.1%" + edge
|
||||
catalog["/routing/route"].push({ ...catalog["/routing/route"][0], ".id": "*7", "dst-address": "198.18.240.1/32",
|
||||
gateway: "198.18.241.6", "immediate-gw": "198.18.241.6%" + middle })
|
||||
assert.deepEqual((await owned()).ownedInterfaces, [edge, middle])
|
||||
catalog["/ip/firewall/connection"].push(foreignUdp)
|
||||
await assert.rejects(owned(), /not observed empty/)
|
||||
})
|
||||
|
||||
test("reads complete bounded native catalogs with identity fences and stable exact path fingerprint", async () => {
|
||||
const { calls, catalog, read } = fixture()
|
||||
const first = await read()
|
||||
assert.equal(first.routesVerified, true)
|
||||
assert.equal(first.freshConntrackTuples, true)
|
||||
assert.match(first.fingerprint, /^[a-f0-9]{64}$/)
|
||||
assert.deepEqual(calls, [...Object.keys(catalog).slice(0, 5), "/interface", "/ip/address", "/routing/table",
|
||||
"/routing/route", "/ip/firewall/connection/tracking", "/ip/firewall/connection", "/system/identity", "/system/resource"])
|
||||
catalog["/ip/firewall/connection"].push(foreignUdp)
|
||||
assert.deepEqual(await read(), first)
|
||||
catalog["/routing/route"][0].distance = "2"
|
||||
assert.notEqual((await read()).fingerprint, first.fingerprint)
|
||||
})
|
||||
|
||||
test("accepts the owned scoped gateway and skips explicitly typed non-IP link routes", async () => {
|
||||
const { catalog, read } = fixture()
|
||||
catalog["/routing/route"][0].gateway = "198.18.241.1%nfedge"
|
||||
catalog["/routing/route"].push({ ".id": "*5", "routing-table": "main", afi: "link", "dst-address": "nfedge" })
|
||||
assert.equal((await read()).routesVerified, true)
|
||||
catalog["/routing/route"][1].afi = "ip"
|
||||
await assert.rejects(read(), /invalid prefix/)
|
||||
})
|
||||
|
||||
test("refuses original or reply tuple reuse and malformed nonmatching UDP rows", async () => {
|
||||
for (const fields of [
|
||||
{ "src-address": "198.18.240.1:50001", "dst-address": "198.18.240.2:50002" },
|
||||
{ "reply-src-address": "198.18.240.2:50002", "reply-dst-address": "198.18.240.1:50001" },
|
||||
{ "reply-src-address": undefined },
|
||||
]) {
|
||||
const { catalog, read } = fixture()
|
||||
catalog["/ip/firewall/connection"].push({ ...foreignUdp, ...fields })
|
||||
await assert.rejects(read(), /tracked|incomplete UDP/)
|
||||
}
|
||||
})
|
||||
|
||||
test("refuses competing routes, routing policy, unproven conntrack and changed device identity", async () => {
|
||||
const competitor = fixture()
|
||||
competitor.catalog["/routing/route"].push({ ...competitor.catalog["/routing/route"][0], ".id": "*5" })
|
||||
await assert.rejects(competitor.read(), /exact active host route/)
|
||||
const policy = fixture()
|
||||
policy.catalog["/routing/rule"].push({ ".id": "*1", disabled: "false" })
|
||||
await assert.rejects(policy.read(), /active routing policy/)
|
||||
const tracking = fixture()
|
||||
tracking.catalog["/ip/firewall/connection/tracking"][0].enabled = "auto"
|
||||
await assert.rejects(tracking.read(), /not explicitly enabled/)
|
||||
const identity = fixture()
|
||||
identity.catalog["/system/identity"][0].name = "foreign"
|
||||
await assert.rejects(identity.read(), /device identity changed/)
|
||||
})
|
||||
@@ -0,0 +1,160 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import { isIP, isIPv4 } from "node:net"
|
||||
import { performance } from "node:perf_hooks"
|
||||
import type { MikrotikClient } from "../services/mikrotik.js"
|
||||
import { isChrWireGuardInterfaceType } from "./traffic-flow-chr-transit-fixture.js"
|
||||
|
||||
type Row = Record<string, unknown>
|
||||
type Socket = { address: string; port: number }
|
||||
type UdpTuple = { source: Socket; destination: Socket }
|
||||
type RoutePin = { routeId: string; destination: string; nextHop: string; interfaceName: string;
|
||||
interfaceId: string; addressId: string; interfaceCidr: string }
|
||||
type Options = {
|
||||
client: Pick<MikrotikClient, "get">
|
||||
identity: string
|
||||
version: string
|
||||
routes: readonly RoutePin[]
|
||||
udp: readonly UdpTuple[]
|
||||
assertCurrent: () => Promise<void>
|
||||
profile?: "owned-wg-empty-firewall"
|
||||
}
|
||||
|
||||
const deny = (message: string): never => { throw Error("RouterOS route/conntrack proof refused: " + message) }
|
||||
const id = (value: unknown) => typeof value === "string" && /^\*[a-f0-9]{1,16}$/i.test(value)
|
||||
const yes = (value: unknown) => value === true || value === "true" || value === "yes"
|
||||
const no = (value: unknown) => value === false || value === "false" || value === "no"
|
||||
const port = (value: number) => Number.isInteger(value) && value >= 1 && value <= 65535
|
||||
const socketKey = (value: Socket) => value.address + ":" + value.port
|
||||
const endpointsKey = (a: string, b: string) => [a, b].sort().join("/")
|
||||
const canonicalRow = (row: Row) => Object.fromEntries(Object.keys(row).sort().map(key => [key, row[key]]))
|
||||
const ipNumber = (address: string) => address.split(".").reduce((sum, octet) => sum * 256 + Number(octet), 0)
|
||||
|
||||
function prefix(value: unknown) {
|
||||
if (typeof value !== "string") return deny("invalid prefix")
|
||||
const [address, length, ...rest] = value.split("/")
|
||||
const family = isIP(address)
|
||||
if (!family || rest.length || !/^(0|[1-9][0-9]{0,2})$/.test(length) || Number(length) > (family === 4 ? 32 : 128)) {
|
||||
return deny("invalid prefix")
|
||||
}
|
||||
return { address, length: Number(length), family }
|
||||
}
|
||||
function endpoint(value: unknown) {
|
||||
if (typeof value !== "string") return deny("incomplete UDP conntrack row")
|
||||
const match = /^([0-9.]+):([1-9][0-9]{0,4})$/.exec(value)
|
||||
if (!match || !isIPv4(match[1]) || !port(Number(match[2]))) return deny("unsupported UDP conntrack endpoint")
|
||||
return socketKey({ address: match[1], port: Number(match[2]) })
|
||||
}
|
||||
function sameSubnet(cidr: string, nextHop: string) {
|
||||
const net = prefix(cidr)
|
||||
const size = 2 ** (32 - net.length)
|
||||
return net.family === 4 && net.length < 32 && net.address !== nextHop &&
|
||||
Math.floor(ipNumber(net.address) / size) === Math.floor(ipNumber(nextHop) / size)
|
||||
}
|
||||
|
||||
/** Narrow main-table reader for an already provisioned fixture. An authoritative
|
||||
* caller brackets policy evaluation with independent fresh calls to this reader. */
|
||||
export function createChrRouteConntrackReader(options: Options) {
|
||||
const { client, assertCurrent, profile } = options
|
||||
const { identity, version, routes, udp } = structuredClone({
|
||||
identity: options.identity, version: options.version, routes: options.routes, udp: options.udp,
|
||||
})
|
||||
if (!identity || !version || !routes.length || routes.length > 16 || !udp.length || udp.length > 32 ||
|
||||
new Set(routes.map(pin => pin.destination)).size !== routes.length || routes.some(pin =>
|
||||
!id(pin.routeId) || !id(pin.interfaceId) || !id(pin.addressId) ||
|
||||
!isIPv4(pin.destination) || !isIPv4(pin.nextHop) || !/^[\w.-]{1,64}$/.test(pin.interfaceName) ||
|
||||
!sameSubnet(pin.interfaceCidr, pin.nextHop)) ||
|
||||
udp.some(tuple => [tuple.source, tuple.destination].some(value => !isIPv4(value.address) || !port(value.port)))) deny("invalid fixture scope")
|
||||
const pinnedTuples = new Set(udp.map(tuple => endpointsKey(socketKey(tuple.source), socketKey(tuple.destination))))
|
||||
const ownedWg = profile === "owned-wg-empty-firewall"
|
||||
const ownedInterfaces = [...new Set(routes.map(pin => pin.interfaceName))].sort()
|
||||
if (ownedWg && (routes.some(pin => !/^nf[a-f0-9]{8}(ea|ma|mb|eb)$/.test(pin.interfaceName)) ||
|
||||
new Set(routes.map(pin => pin.interfaceName.slice(0, -2))).size !== 1 ||
|
||||
!["ea,ma", "eb,mb"].includes(ownedInterfaces.map(name => name.slice(-2)).join(",")) ||
|
||||
new Set(routes.map(pin => pin.interfaceId)).size !== 2)) deny("owned WireGuard interface scope differs")
|
||||
|
||||
return async function readPathEvidence() {
|
||||
const deadline = performance.now() + 12000
|
||||
async function fence() {
|
||||
if (performance.now() >= deadline) deny("expired evidence")
|
||||
await assertCurrent()
|
||||
if (performance.now() >= deadline) deny("expired evidence")
|
||||
}
|
||||
async function read(path: string, limit: number, single = false) {
|
||||
await fence()
|
||||
const response = await client.get<unknown>(path, Math.min(1800, Math.ceil(deadline - performance.now())),
|
||||
undefined, 512 * 1024, fence)
|
||||
await fence()
|
||||
const rows = Array.isArray(response) ? response : single ? [response] : null
|
||||
if (!rows || rows.length > limit || single && rows.length !== 1 ||
|
||||
rows.some(row => !row || typeof row !== "object" || Array.isArray(row))) return deny("incomplete catalog")
|
||||
return rows as Row[]
|
||||
}
|
||||
async function identityFence() {
|
||||
const [device] = await read("/system/identity", 1, true)
|
||||
const [resource] = await read("/system/resource", 1, true)
|
||||
if (device.name !== identity || resource.version !== version) deny("device identity changed")
|
||||
}
|
||||
function requireCatalogIds(rows: Row[]) {
|
||||
if (rows.some(row => !id(row[".id"])) || new Set(rows.map(row => row[".id"])).size !== rows.length) deny("invalid catalog IDs")
|
||||
return rows
|
||||
}
|
||||
await identityFence()
|
||||
const [ip] = await read("/ip/settings", 1, true)
|
||||
if (!yes(ip["ip-forward"]) || (!no(ip["allow-fast-path"]) && !(ownedWg && yes(ip["allow-fast-path"])))) {
|
||||
deny("forwarding or fast-path state unknown")
|
||||
}
|
||||
const policy = requireCatalogIds(await read("/routing/rule", 128))
|
||||
if (policy.some(row => !yes(row.disabled))) deny("active routing policy")
|
||||
const vrfs = requireCatalogIds(await read("/ip/vrf", 64))
|
||||
if (vrfs.length !== 1 || vrfs[0].name !== "main" || vrfs[0].interfaces !== "all" || yes(vrfs[0].disabled)) deny("non-default VRF policy")
|
||||
const interfaces = requireCatalogIds(await read("/interface", 256))
|
||||
const addresses = requireCatalogIds(await read("/ip/address", 256))
|
||||
const main = requireCatalogIds(await read("/routing/table", 64)).filter(row => row.name === "main")
|
||||
if (main.length !== 1 || !yes(main[0].fib) || yes(main[0].disabled)) deny("main FIB unconfirmed")
|
||||
const candidates = requireCatalogIds(await read("/routing/route", 2048))
|
||||
for (const row of candidates) {
|
||||
if (typeof row["routing-table"] !== "string" || !row["routing-table"]) deny("incomplete route catalog")
|
||||
if (row.afi === "link") {
|
||||
if (typeof row["dst-address"] !== "string" || !/^[\w.-]{1,64}$/.test(row["dst-address"])) deny("invalid link route")
|
||||
} else prefix(row["dst-address"])
|
||||
}
|
||||
const bindings = routes.map(pin => {
|
||||
const interfaceMatches = interfaces.filter(row => row.name === pin.interfaceName)
|
||||
const iface = interfaceMatches[0]
|
||||
const assigned = addresses.filter(row => row.address === pin.interfaceCidr)
|
||||
const local = assigned[0]
|
||||
if (interfaceMatches.length !== 1 || iface[".id"] !== pin.interfaceId || !isChrWireGuardInterfaceType(iface.type) ||
|
||||
!yes(iface.running) || !no(iface.disabled) || assigned.length !== 1 || local[".id"] !== pin.addressId ||
|
||||
local.interface !== pin.interfaceName || !no(local.disabled) || yes(local.invalid) ||
|
||||
(local["actual-interface"] !== undefined && local["actual-interface"] !== pin.interfaceName)) deny("interface address binding differs")
|
||||
const exact = candidates.filter(row => row["routing-table"] === "main" && row["dst-address"] === pin.destination + "/32")
|
||||
const row = exact[0]
|
||||
if (exact.length !== 1 || row[".id"] !== pin.routeId || !yes(row.active) || !no(row.disabled) ||
|
||||
![pin.nextHop, pin.nextHop + "%" + pin.interfaceName].includes(String(row.gateway)) ||
|
||||
row["immediate-gw"] !== pin.nextHop + "%" + pin.interfaceName ||
|
||||
yes(row.blackhole) || yes(row.unreachable) || yes(row.prohibit) || yes(row.ecmp)) deny("exact active host route unavailable")
|
||||
return [pin, canonicalRow(row), canonicalRow(local), iface.type, iface.running, iface.disabled]
|
||||
})
|
||||
const [tracking] = await read("/ip/firewall/connection/tracking", 1, true)
|
||||
if (!yes(tracking.enabled) && !(ownedWg && tracking.enabled === "auto")) deny("conntrack is not explicitly enabled")
|
||||
const connections = requireCatalogIds(await read("/ip/firewall/connection", 2048))
|
||||
if (ownedWg && tracking.enabled === "auto" && connections.length !== 0) deny("automatic conntrack is not observed empty")
|
||||
for (const row of connections) {
|
||||
const protocol = row.protocol
|
||||
if (protocol === "udp" || protocol === "17") {
|
||||
const original = endpointsKey(endpoint(row["src-address"]), endpoint(row["dst-address"]))
|
||||
const reply = endpointsKey(endpoint(row["reply-src-address"]), endpoint(row["reply-dst-address"]))
|
||||
if (pinnedTuples.has(original) || pinnedTuples.has(reply)) deny("UDP fixture tuple already tracked")
|
||||
} else if (typeof protocol !== "string" || !/^(tcp|icmp|gre|ipsec-esp|ipsec-ah|sctp|udplite|ospf|ipv6-encap|0|[1-9][0-9]{0,2})$/.test(protocol)) {
|
||||
deny("unsupported conntrack protocol")
|
||||
}
|
||||
}
|
||||
await identityFence()
|
||||
const fingerprint = createHash("sha256").update(JSON.stringify([identity, version, profile, ip["allow-fast-path"], tracking.enabled, bindings,
|
||||
canonicalRow(main[0]), canonicalRow(vrfs[0]), [...pinnedTuples].sort()])).digest("hex")
|
||||
return { routesVerified: true as const, freshConntrackTuples: true as const, fingerprint,
|
||||
...(profile && { profile, ownedInterfaces,
|
||||
ownedRoutes: routes.map(pin => ({ destination: pin.destination, interfaceName: pin.interfaceName })),
|
||||
udpPairs: [...pinnedTuples].sort() }) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { z } from "zod"
|
||||
import { CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS, CHR_STAGED_TARGET_CAPTURE_MAX_MS } from "./traffic-flow-chr-target-timing-profile.js"
|
||||
|
||||
export const chrStagedCaptureProfileSchema = z.object({
|
||||
kind: z.literal("staged-disabled-target"),
|
||||
captureMs: z.number().int().min(90000).max(CHR_STAGED_TARGET_CAPTURE_MAX_MS),
|
||||
requiredCollectorWindowMs: z.number().int().positive().lt(CHR_LEGACY_FIXTURE_DEADLINE_MAX_MS),
|
||||
}).strict().refine(profile => profile.requiredCollectorWindowMs > profile.captureMs + 210000,
|
||||
"Staged collector window must include both measured phases and the cleanup reserve")
|
||||
Loaded 100 of 117 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user