/** Actual public fixture runner/admission/legacy locks, real SQL ACK faults. * No native HTTP, production registration or default global-writer activation. */ import assert from "node:assert/strict" import { randomUUID } from "node:crypto" import { performance } from "node:perf_hooks" import { setTimeout as delay } from "node:timers/promises" import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js" export async function acceptOwnedMcpOperationRoot() { assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true") const { pool } = await import("../../db/index.js"), { Client } = await import("pg"), { z } = await import("zod") const { add } = await import("./actions.js"), { registerMcpService } = await import("./service-registry.js") const { createMcpConnection, authenticateMcpToken } = await import("./store.js") const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp") const { invokeMcpTool, approvePlan, applyPlan, recoverMcpOperations } = await import("./runtime.js") const root = await import("./operation-execution-origin.js") const { readMcpTemporaryGrantOrigin } = await import("./temporary-grant-origin.js") const { captureVerifiedMcpServiceExecution } = await import("../../services/config-mutation-policy.js") const operationName = "fixture.operation-root." + randomUUID(), serviceName = "fixture.operation-root-service." + randomUUID(), tool = "mm_owned_operation_root_" + randomUUID().replaceAll("-", "") const secret = "OWNED_PRIVATE_ROOT_PAYLOAD_" + randomUUID(), application = await pool.connect(), rival = await pool.connect() const query = pool.query, clientQuery = Client.prototype.query, ids: number[] = [], checks: string[] = [] const expected = new Map(), caps = new Map(), lockKinds = new Map>() let sourceFaultApplied = false let mode = "normal", serviceCalls = 0, preLockCaptures = 0, legacyEntries = 0, callbackContenders = 0 let selectedOperation: string | undefined, heldRival: string | undefined, cleanupRows = -1, claimRows: number[] = [], rivalNonce: string | undefined let reached!: () => void, proceed!: () => void, cleanupFinished!: () => void let reachedGate = Promise.resolve(), proceedGate = Promise.resolve(), cleanupGate = Promise.resolve(), arrivals = 0 const reset = (next: string) => { mode = next; sourceFaultApplied = false; selectedOperation = undefined; cleanupRows = -1; claimRows = []; rivalNonce = undefined; arrivals = 0 reachedGate = new Promise(yes => { reached = yes }); proceedGate = new Promise(yes => { proceed = yes }); cleanupGate = new Promise(yes => { cleanupFinished = yes }) } const bounded = async (promise: Promise): Promise => { let timer: ReturnType | undefined try { return await Promise.race([promise, new Promise((_yes, no) => { timer = setTimeout(() => no(Error("Owned operation root fixture deadline")), 20000) })]) } finally { clearTimeout(timer) } } const terminal = async (id: string) => { for (const until = Date.now() + 20000; Date.now() < until;) { const row = (await application.query("SELECT state,result,error_code FROM public.mcp_operations WHERE id=$1", [id])).rows[0] if (row && !["pending", "running"].includes(row.state)) return row await delay(10) } throw Error("Owned operation did not reach terminal state") } try { for (let i = 0; i < 2; i++) ids.push(Number((await application.query("INSERT INTO public.servers(name,host,enabled) VALUES('owned-operation-root','127.0.0.1',false) RETURNING id")).rows[0].id)) await application.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1") registerMcpService(serviceName, { current: async () => ({ fixture: "before" }), verify: async (_input, result) => !!result && typeof result === "object" && "fixture" in result, execute: async () => { serviceCalls++; const old = await captureVerifiedMcpServiceExecution(); assert(old); await assert.rejects(root.captureMcpOperationExecutionOrigin()); return { fixture: "done" } } }) add(operationName, tool, "Owned operation root acceptance", "servers", "mm:servers:write", "write", "servers", z.object({ password: z.string() }).strict(), { version: 1, target: () => ({ scope: "servers", serverIds: ids }), prepare: async input => ({ target: { scope: "servers", serverIds: ids }, objects: ids.map(serverId => ({ kind: "server", id: String(serverId), serverId })), snapshots: [], before: { fixture: "before" }, warnings: [], steps: [0, 1].map(index => ({ kind: "service", service: serviceName, input: { index, password: input.password } })) }) }) const cleanupPids = new Set() Client.prototype.query = function(this: InstanceType, ...args: unknown[]) { const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined const key = String(values?.[0] ?? ""), opLock = text.includes("pg_try_advisory_lock") && key.startsWith("mcp-operation:") const legacy = opLock ? "operation" : text.includes("pg_try_advisory_lock") && text.includes("mm-config:application") ? "application" : text.includes("pg_try_advisory_lock") && /^mm-config:\d+$/.test(key) ? "server" : undefined const physical = this as unknown as { processID: number } const run = async () => { if (opLock) { const id = key.slice("mcp-operation:".length); selectedOperation = id; legacyEntries++ const cap = await root.captureMcpOperationExecutionOrigin(), metadata = await root.readCurrentMcpOperationExecutionOrigin(cap), planned = expected.get(metadata.planId); assert(planned) assert.equal(metadata.operationId, id); assert.equal(metadata.approved, planned.approved); assert.equal(metadata.ownerId, planned.ownerId) assert.equal(metadata.evidence.steps.length, 2); assert.equal(metadata.privateInput.password, secret); assert.deepEqual(metadata.serverIds, ids); assert(!("stepIndex" in metadata)) const held = await rival.query("SELECT count(*)::int AS n FROM pg_catalog.pg_locks WHERE pid=$1 AND locktype='advisory' AND granted", [physical.processID]); assert.equal(held.rows[0].n, 0) caps.set(id, cap); preLockCaptures++ if (mode === "hold-public") { reached(); await proceedGate } if (mode === "rival-operation-lock") { await rival.query("SELECT pg_advisory_lock(hashtextextended($1,0))", [key]); heldRival = key } } else if (legacy && selectedOperation) { await assert.rejects(root.captureMcpOperationExecutionOrigin()); await root.readCurrentMcpOperationExecutionOrigin(caps.get(selectedOperation)) } if (legacy && selectedOperation) { const set = lockKinds.get(selectedOperation) ?? new Set(); set.add(legacy); lockKinds.set(selectedOperation, set) } const result = await Reflect.apply(clientQuery, this, args) if (opLock && mode === "callback-error") throw Error("Owned callback failure after actual operation lock ACK") if (text.startsWith("UPDATE public.mcp_operations SET state='unknown'") && values?.length === 4) { cleanupRows = result.rowCount; cleanupPids.add(physical.processID) } if (text === "COMMIT" && cleanupPids.delete(physical.processID)) cleanupFinished() return result } // Only direct Promise driver calls are instrumented asynchronously. All // pg-pool callback calls and their results stay completely unchanged. if (legacy || text.startsWith("UPDATE public.mcp_operations SET state='unknown'") || text === "COMMIT" && cleanupPids.has(physical.processID)) return run() return Reflect.apply(clientQuery, this, args) } as typeof Client.prototype.query pool.query = function(this: typeof pool, ...args: unknown[]) { const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined if (text.startsWith("/* mcp-operation-root:source */") && mode === "preclaim-permission" && !sourceFaultApplied) return (async () => { const result = await Reflect.apply(query, this, args); sourceFaultApplied = true await application.query("UPDATE public.mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [values![1]]) return result })() if (!text.startsWith("/* mcp-operation-root:claim */")) return Reflect.apply(query, this, args) return (async () => { if (mode === "dual-claim") arrivals++ if (mode === "lost-zero") { rivalNonce = randomUUID(); const won = await application.query("UPDATE public.mcp_operations SET state='running',result=jsonb_build_object('__mmappMcpOperationClaimV1',$2::text) WHERE id=$1 AND state='pending'", [values![0], rivalNonce]); assert.equal(won.rowCount, 1) } const result = await Reflect.apply(query, this, args); claimRows.push(result.rowCount) if (mode === "lost-own") { assert.equal(result.rowCount, 1); throw Error("Owned actual one-row claim ACK lost") } if (mode === "lost-zero") { assert.equal(result.rowCount, 0); throw Error("Owned actual zero-row claim ACK lost") } return result })() } as typeof pool.query async function prepare(approved: boolean) { const actor = { id: randomUUID(), email: "owned-root@example.test", name: "Owned operation root", apps: ["mm"], permissions: ["mm:servers:write", "mm:mcp:approve"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 300000 } const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Owned operation root", mode: approved ? "approval" : "autonomous", permissions: ["mm:servers:write"], expiresAt: new Date(actor.expiresAt).toISOString(), scope: { serverIds: ids, operationIds: [operationName, "operations.read"], allowGlobal: false }, ...(approved ? {} : { dangerousPolicy: "explicit-allow", autonomousDangerousOperationIds: [operationName] }), limits: { callsPerMinute: 500, mutationsPerHour: 1000 } }), actor) const principal = await authenticateMcpToken(access.token), plan = await invokeMcpTool(principal, tool, { password: secret }) as { planId: string } if (approved) await approvePlan(plan.planId, actor, true) expected.set(plan.planId, { approved, ownerId: actor.id }) return { principal, planId: plan.planId } } for (const approved of [true, false]) { reset("hold-public"); const prepared = await prepare(approved), before = serviceCalls const operation = await applyPlan(prepared.principal, prepared.planId, randomUUID()); await bounded(reachedGate) const privateRow = (await application.query("SELECT result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0].result const nonce = privateRow[root.MCP_OPERATION_PRIVATE_CLAIM_KEY]; assert.equal(typeof nonce, "string") const model = await invokeMcpTool(prepared.principal, "mm_operation_get", { operationId: operation.id }), web = await applyPlan(prepared.principal, prepared.planId, randomUUID()) for (const output of [model, web]) { assert(!JSON.stringify(output).includes(nonce)); assert(!JSON.stringify(output).includes(root.MCP_OPERATION_PRIVATE_CLAIM_KEY)) } proceed(); const finished = await terminal(operation.id); assert.equal(finished.state, "applied"); assert.equal(serviceCalls - before, 2) assert.deepEqual([...lockKinds.get(operation.id)!].sort(), ["application", "operation", "server"]) await assert.rejects(root.readCurrentMcpOperationExecutionOrigin(caps.get(operation.id))) } checks.push("actual human/autonomous public fixture runner publishes full root before operation/app/server locks, retains per-step origins and hides nonce from real model/direct-web apply results") { const { withMcpDatabaseRequest } = await import("./database-request-lease.js") const { SYSTEM_DATABASE_REQUEST_LOCK, withSystemDatabaseOperationLock } = await import("../../db/operation-lock.js") reset("normal") const prepared = await prepare(true), before = serviceCalls, key = randomUUID() let releaseMaintenance!: () => void, enteredMaintenance!: () => void const release = new Promise(resolve => { releaseMaintenance = resolve }) const entered = new Promise(resolve => { enteredMaintenance = resolve }) let maintenance: Promise | undefined try { const admitted = await withMcpDatabaseRequest(async () => { maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => { enteredMaintenance() await release }) void maintenance.catch(() => {}) for (const end = Date.now() + 2000; Date.now() < end;) { const queued = await application.query( "SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid=0 AND objid=$1 AND mode='ExclusiveLock' AND NOT granted", [SYSTEM_DATABASE_REQUEST_LOCK]) if (queued.rowCount) return applyPlan(prepared.principal, prepared.planId, key) await delay(10) } throw Error("Owned maintenance lock did not queue") }) await bounded(entered) await delay(350) await application.query("UPDATE mcp_operations SET created_at=now()-interval '3 minutes' WHERE id=$1", [admitted.id]) const orphanId = randomUUID(), orphanPlanId = randomUUID() await application.query(`INSERT INTO mcp_plans (id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at) SELECT $1,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,'executing',now()-interval '1 minute' FROM mcp_plans WHERE id=$2`, [orphanPlanId, prepared.planId]) await application.query(`INSERT INTO mcp_operations (id,plan_id,connection_id,idempotency_key,state,created_at) VALUES($1,$2,$3,$1::uuid::text,'pending',now()-interval '3 minutes')`, [orphanId, orphanPlanId, prepared.principal.connectionId]) // Exercise the recovery predicate while the real runner is held at admission. await recoverMcpOperations() assert.equal((await application.query("SELECT state FROM mcp_operations WHERE id=$1", [admitted.id])).rows[0].state, "pending") assert.deepEqual((await application.query("SELECT state,error_code FROM mcp_operations WHERE id=$1", [orphanId])).rows[0], { state: "unknown", error_code: "PROCESS_INTERRUPTED" }) assert.equal(serviceCalls, before) releaseMaintenance() await maintenance assert.equal((await terminal(admitted.id)).state, "applied") assert.equal((await applyPlan(prepared.principal, prepared.planId, key)).id, admitted.id) assert.equal(serviceCalls - before, 2) checks.push("recovery preserves a queued operation under its original plan deadline, closes an expired orphan, and the queued operation executes both steps once") } finally { releaseMaintenance() await maintenance?.catch(() => {}) } } for (const fault of ["lost-own", "lost-zero", "rival-operation-lock", "callback-error", "preclaim-permission"] as const) { reset(fault); const prepared = await prepare(false), before = serviceCalls, operation = await applyPlan(prepared.principal, prepared.planId, randomUUID()) if (fault === "lost-zero") { await bounded(cleanupGate); assert.equal(cleanupRows, 0) const row = (await application.query("SELECT state,result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0] assert.equal(row.state, "running"); assert.equal(row.result[root.MCP_OPERATION_PRIVATE_CLAIM_KEY], rivalNonce) await application.query("UPDATE public.mcp_operations SET state='unknown',result='{}'::jsonb,finished_at=now() WHERE id=$1 AND result->>'__mmappMcpOperationClaimV1'=$2", [operation.id, rivalNonce]) } else { const row = await terminal(operation.id); assert.equal(row.state, "unknown"); assert.equal(row.error_code, "PROCESS_INTERRUPTED") } if (heldRival) { await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]); heldRival = undefined } assert.equal(serviceCalls, before) } checks.push("actual one-row/zero-row ACK loss, real rival operation lock refusal and post-real-lock callback error and pre-claim permission loss: no legacy service/native replay; only own nonce becomes unknown, rival winner preserved") reset("dual-claim"); const prepared = await prepare(false), anchor = root.beginMcpOperationAdmission(prepared.principal), wall = Date.now(), mono = performance.now() await application.query("BEGIN") let issued: Awaited> try { const plan = (await application.query("SELECT * FROM public.mcp_plans WHERE id=$1 FOR UPDATE", [prepared.planId])).rows[0] const grant = await readMcpTemporaryGrantOrigin(application, prepared.principal.connectionId, prepared.principal.credentialId), expires = Math.min(plan.expires_at.getTime(), Date.parse(prepared.principal.credentialExpiresAt)) issued = await root.insertMcpOperationAdmission(application, anchor, prepared.planId, randomUUID(), { ...grant, monotonicEnd: grant.expiresAt === null ? Infinity : mono + grant.expiresAt - wall, admissionExpiresAt: expires, admissionMonotonicEnd: mono + expires - wall }) await application.query("UPDATE public.mcp_plans SET state='executing' WHERE id=$1", [prepared.planId]); assert.equal((await application.query("COMMIT")).command, "COMMIT") } catch (error) { await application.query("ROLLBACK"); throw error } const beforeEntries = legacyEntries const outcomes = await Promise.allSettled([0, 1].map(() => root.withMcpOperationExecutionOrigin(issued.admission, async () => { callbackContenders++ }))) const finished = await terminal(issued.operation.id) assert.equal(arrivals, 1); assert.deepEqual(claimRows, [1]); assert.equal(outcomes.filter(value => value.status === "rejected").length, 1) assert.equal(legacyEntries - beforeEntries + callbackContenders, 1); assert.equal(finished.state, "unknown") checks.push("actual creating-transaction INSERT owns private one-shot admission: two pending publishers enter one callback/one real CAS; separate rival case proves actual zero-row ACK isolation") const later = await prepare(false), lateStart = root.beginMcpOperationAdmission(later.principal), latePlan = (await application.query("SELECT expires_at FROM public.mcp_plans WHERE id=$1", [later.planId])).rows[0] const lateGrant = await readMcpTemporaryGrantOrigin(application, later.principal.connectionId, later.principal.credentialId), end = Math.min(latePlan.expires_at.getTime(), Date.parse(later.principal.credentialExpiresAt)) await assert.rejects(root.insertMcpOperationAdmission(application, lateStart, later.planId, randomUUID(), { ...lateGrant, monotonicEnd: Infinity, admissionExpiresAt: end, admissionMonotonicEnd: performance.now() + end - Date.now() })) assert.equal((await application.query("SELECT id FROM public.mcp_operations WHERE plan_id=$1", [later.planId])).rows.length, 0) checks.push("later/autocommit caller cannot mint admission or orphan an INSERT; no existing-pending UPDATE mint API") return { checks, actualPublicFixtureRunner: true, preLockCaptures, legacyEntries, serviceCalls, claimRows, actualSql: true, nativeCalls: 0, globalWriterActivated: false, terminalRestoreRecoveryCovered: false, extraProductionPools: 0, extraProductionProcesses: 0, privateNonceExposed: false } } finally { proceed?.(); pool.query = query; Client.prototype.query = clientQuery if (heldRival) await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]).catch(() => {}) application.release(true); rival.release(true) } }