212 lines
20 KiB
TypeScript
212 lines
20 KiB
TypeScript
/** Actual public fixture runner/admission/legacy locks, real SQL ACK faults.
|
|
* No native HTTP, production registration or default global-writer activation. */
|
|
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { performance } from "node:perf_hooks"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js"
|
|
|
|
export async function acceptOwnedMcpOperationRoot() {
|
|
assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true")
|
|
const { pool } = await import("../../db/index.js"), { Client } = await import("pg"), { z } = await import("zod")
|
|
const { add } = await import("./actions.js"), { registerMcpService } = await import("./service-registry.js")
|
|
const { createMcpConnection, authenticateMcpToken } = await import("./store.js")
|
|
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
|
const { invokeMcpTool, approvePlan, applyPlan, recoverMcpOperations } = await import("./runtime.js")
|
|
const root = await import("./operation-execution-origin.js")
|
|
const { readMcpTemporaryGrantOrigin } = await import("./temporary-grant-origin.js")
|
|
const { captureVerifiedMcpServiceExecution } = await import("../../services/config-mutation-policy.js")
|
|
const operationName = "fixture.operation-root." + randomUUID(), serviceName = "fixture.operation-root-service." + randomUUID(), tool = "mm_owned_operation_root_" + randomUUID().replaceAll("-", "")
|
|
const secret = "OWNED_PRIVATE_ROOT_PAYLOAD_" + randomUUID(), application = await pool.connect(), rival = await pool.connect()
|
|
const query = pool.query, clientQuery = Client.prototype.query, ids: number[] = [], checks: string[] = []
|
|
const expected = new Map<string, { approved: boolean; ownerId: string }>(), caps = new Map<string, unknown>(), lockKinds = new Map<string, Set<string>>()
|
|
let sourceFaultApplied = false
|
|
let mode = "normal", serviceCalls = 0, preLockCaptures = 0, legacyEntries = 0, callbackContenders = 0
|
|
let selectedOperation: string | undefined, heldRival: string | undefined, cleanupRows = -1, claimRows: number[] = [], rivalNonce: string | undefined
|
|
let reached!: () => void, proceed!: () => void, cleanupFinished!: () => void
|
|
let reachedGate = Promise.resolve(), proceedGate = Promise.resolve(), cleanupGate = Promise.resolve(), arrivals = 0
|
|
const reset = (next: string) => { mode = next; sourceFaultApplied = false; selectedOperation = undefined; cleanupRows = -1; claimRows = []; rivalNonce = undefined; arrivals = 0
|
|
reachedGate = new Promise<void>(yes => { reached = yes }); proceedGate = new Promise<void>(yes => { proceed = yes }); cleanupGate = new Promise<void>(yes => { cleanupFinished = yes }) }
|
|
const bounded = async <T>(promise: Promise<T>): Promise<T> => {
|
|
let timer: ReturnType<typeof setTimeout> | undefined
|
|
try { return await Promise.race([promise, new Promise<never>((_yes, no) => { timer = setTimeout(() => no(Error("Owned operation root fixture deadline")), 20000) })]) }
|
|
finally { clearTimeout(timer) }
|
|
}
|
|
const terminal = async (id: string) => {
|
|
for (const until = Date.now() + 20000; Date.now() < until;) {
|
|
const row = (await application.query("SELECT state,result,error_code FROM public.mcp_operations WHERE id=$1", [id])).rows[0]
|
|
if (row && !["pending", "running"].includes(row.state)) return row
|
|
await delay(10)
|
|
}
|
|
throw Error("Owned operation did not reach terminal state")
|
|
}
|
|
try {
|
|
for (let i = 0; i < 2; i++) ids.push(Number((await application.query("INSERT INTO public.servers(name,host,enabled) VALUES('owned-operation-root','127.0.0.1',false) RETURNING id")).rows[0].id))
|
|
await application.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
|
registerMcpService(serviceName, { current: async () => ({ fixture: "before" }), verify: async (_input, result) => !!result && typeof result === "object" && "fixture" in result,
|
|
execute: async () => { serviceCalls++; const old = await captureVerifiedMcpServiceExecution(); assert(old); await assert.rejects(root.captureMcpOperationExecutionOrigin()); return { fixture: "done" } } })
|
|
add(operationName, tool, "Owned operation root acceptance", "servers", "mm:servers:write", "write", "servers", z.object({ password: z.string() }).strict(), { version: 1,
|
|
target: () => ({ scope: "servers", serverIds: ids }), prepare: async input => ({ target: { scope: "servers", serverIds: ids }, objects: ids.map(serverId => ({ kind: "server", id: String(serverId), serverId })), snapshots: [], before: { fixture: "before" }, warnings: [],
|
|
steps: [0, 1].map(index => ({ kind: "service", service: serviceName, input: { index, password: input.password } })) }) })
|
|
const cleanupPids = new Set<number>()
|
|
Client.prototype.query = function(this: InstanceType<typeof Client>, ...args: unknown[]) {
|
|
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
|
|
const key = String(values?.[0] ?? ""), opLock = text.includes("pg_try_advisory_lock") && key.startsWith("mcp-operation:")
|
|
const legacy = opLock ? "operation" : text.includes("pg_try_advisory_lock") && text.includes("mm-config:application") ? "application" : text.includes("pg_try_advisory_lock") && /^mm-config:\d+$/.test(key) ? "server" : undefined
|
|
const physical = this as unknown as { processID: number }
|
|
const run = async () => {
|
|
if (opLock) {
|
|
const id = key.slice("mcp-operation:".length); selectedOperation = id; legacyEntries++
|
|
const cap = await root.captureMcpOperationExecutionOrigin(), metadata = await root.readCurrentMcpOperationExecutionOrigin(cap), planned = expected.get(metadata.planId); assert(planned)
|
|
assert.equal(metadata.operationId, id); assert.equal(metadata.approved, planned.approved); assert.equal(metadata.ownerId, planned.ownerId)
|
|
assert.equal(metadata.evidence.steps.length, 2); assert.equal(metadata.privateInput.password, secret); assert.deepEqual(metadata.serverIds, ids); assert(!("stepIndex" in metadata))
|
|
const held = await rival.query("SELECT count(*)::int AS n FROM pg_catalog.pg_locks WHERE pid=$1 AND locktype='advisory' AND granted", [physical.processID]); assert.equal(held.rows[0].n, 0)
|
|
caps.set(id, cap); preLockCaptures++
|
|
if (mode === "hold-public") { reached(); await proceedGate }
|
|
if (mode === "rival-operation-lock") { await rival.query("SELECT pg_advisory_lock(hashtextextended($1,0))", [key]); heldRival = key }
|
|
} else if (legacy && selectedOperation) {
|
|
await assert.rejects(root.captureMcpOperationExecutionOrigin()); await root.readCurrentMcpOperationExecutionOrigin(caps.get(selectedOperation))
|
|
}
|
|
if (legacy && selectedOperation) { const set = lockKinds.get(selectedOperation) ?? new Set<string>(); set.add(legacy); lockKinds.set(selectedOperation, set) }
|
|
const result = await Reflect.apply(clientQuery, this, args)
|
|
if (opLock && mode === "callback-error") throw Error("Owned callback failure after actual operation lock ACK")
|
|
if (text.startsWith("UPDATE public.mcp_operations SET state='unknown'") && values?.length === 4) { cleanupRows = result.rowCount; cleanupPids.add(physical.processID) }
|
|
if (text === "COMMIT" && cleanupPids.delete(physical.processID)) cleanupFinished()
|
|
return result
|
|
}
|
|
// Only direct Promise driver calls are instrumented asynchronously. All
|
|
// pg-pool callback calls and their results stay completely unchanged.
|
|
if (legacy || text.startsWith("UPDATE public.mcp_operations SET state='unknown'") || text === "COMMIT" && cleanupPids.has(physical.processID)) return run()
|
|
return Reflect.apply(clientQuery, this, args)
|
|
} as typeof Client.prototype.query
|
|
pool.query = function(this: typeof pool, ...args: unknown[]) {
|
|
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
|
|
if (text.startsWith("/* mcp-operation-root:source */") && mode === "preclaim-permission" && !sourceFaultApplied) return (async () => {
|
|
const result = await Reflect.apply(query, this, args); sourceFaultApplied = true
|
|
await application.query("UPDATE public.mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [values![1]])
|
|
return result
|
|
})()
|
|
if (!text.startsWith("/* mcp-operation-root:claim */")) return Reflect.apply(query, this, args)
|
|
return (async () => {
|
|
if (mode === "dual-claim") arrivals++
|
|
if (mode === "lost-zero") { rivalNonce = randomUUID(); const won = await application.query("UPDATE public.mcp_operations SET state='running',result=jsonb_build_object('__mmappMcpOperationClaimV1',$2::text) WHERE id=$1 AND state='pending'", [values![0], rivalNonce]); assert.equal(won.rowCount, 1) }
|
|
const result = await Reflect.apply(query, this, args); claimRows.push(result.rowCount)
|
|
if (mode === "lost-own") { assert.equal(result.rowCount, 1); throw Error("Owned actual one-row claim ACK lost") }
|
|
if (mode === "lost-zero") { assert.equal(result.rowCount, 0); throw Error("Owned actual zero-row claim ACK lost") }
|
|
return result
|
|
})()
|
|
} as typeof pool.query
|
|
async function prepare(approved: boolean) {
|
|
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned operation root", apps: ["mm"], permissions: ["mm:servers:write", "mm:mcp:approve"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 300000 }
|
|
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Owned operation root", mode: approved ? "approval" : "autonomous", permissions: ["mm:servers:write"], expiresAt: new Date(actor.expiresAt).toISOString(), scope: { serverIds: ids, operationIds: [operationName, "operations.read"], allowGlobal: false },
|
|
...(approved ? {} : { dangerousPolicy: "explicit-allow", autonomousDangerousOperationIds: [operationName] }), limits: { callsPerMinute: 500, mutationsPerHour: 1000 } }), actor)
|
|
const principal = await authenticateMcpToken(access.token), plan = await invokeMcpTool(principal, tool, { password: secret }) as { planId: string }
|
|
if (approved) await approvePlan(plan.planId, actor, true)
|
|
expected.set(plan.planId, { approved, ownerId: actor.id })
|
|
return { principal, planId: plan.planId }
|
|
}
|
|
for (const approved of [true, false]) {
|
|
reset("hold-public"); const prepared = await prepare(approved), before = serviceCalls
|
|
const operation = await applyPlan(prepared.principal, prepared.planId, randomUUID()); await bounded(reachedGate)
|
|
const privateRow = (await application.query("SELECT result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0].result
|
|
const nonce = privateRow[root.MCP_OPERATION_PRIVATE_CLAIM_KEY]; assert.equal(typeof nonce, "string")
|
|
const model = await invokeMcpTool(prepared.principal, "mm_operation_get", { operationId: operation.id }), web = await applyPlan(prepared.principal, prepared.planId, randomUUID())
|
|
for (const output of [model, web]) { assert(!JSON.stringify(output).includes(nonce)); assert(!JSON.stringify(output).includes(root.MCP_OPERATION_PRIVATE_CLAIM_KEY)) }
|
|
proceed(); const finished = await terminal(operation.id); assert.equal(finished.state, "applied"); assert.equal(serviceCalls - before, 2)
|
|
assert.deepEqual([...lockKinds.get(operation.id)!].sort(), ["application", "operation", "server"])
|
|
await assert.rejects(root.readCurrentMcpOperationExecutionOrigin(caps.get(operation.id)))
|
|
}
|
|
checks.push("actual human/autonomous public fixture runner publishes full root before operation/app/server locks, retains per-step origins and hides nonce from real model/direct-web apply results")
|
|
{
|
|
const { withMcpDatabaseRequest } = await import("./database-request-lease.js")
|
|
const { SYSTEM_DATABASE_REQUEST_LOCK, withSystemDatabaseOperationLock } = await import("../../db/operation-lock.js")
|
|
reset("normal")
|
|
const prepared = await prepare(true), before = serviceCalls, key = randomUUID()
|
|
let releaseMaintenance!: () => void, enteredMaintenance!: () => void
|
|
const release = new Promise<void>(resolve => { releaseMaintenance = resolve })
|
|
const entered = new Promise<void>(resolve => { enteredMaintenance = resolve })
|
|
let maintenance: Promise<void> | undefined
|
|
try {
|
|
const admitted = await withMcpDatabaseRequest(async () => {
|
|
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
|
|
enteredMaintenance()
|
|
await release
|
|
})
|
|
void maintenance.catch(() => {})
|
|
for (const end = Date.now() + 2000; Date.now() < end;) {
|
|
const queued = await application.query(
|
|
"SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid=0 AND objid=$1 AND mode='ExclusiveLock' AND NOT granted",
|
|
[SYSTEM_DATABASE_REQUEST_LOCK])
|
|
if (queued.rowCount) return applyPlan(prepared.principal, prepared.planId, key)
|
|
await delay(10)
|
|
}
|
|
throw Error("Owned maintenance lock did not queue")
|
|
})
|
|
await bounded(entered)
|
|
await delay(350)
|
|
await application.query("UPDATE mcp_operations SET created_at=now()-interval '3 minutes' WHERE id=$1", [admitted.id])
|
|
const orphanId = randomUUID(), orphanPlanId = randomUUID()
|
|
await application.query(`INSERT INTO mcp_plans
|
|
(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at)
|
|
SELECT $1,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,'executing',now()-interval '1 minute'
|
|
FROM mcp_plans WHERE id=$2`, [orphanPlanId, prepared.planId])
|
|
await application.query(`INSERT INTO mcp_operations (id,plan_id,connection_id,idempotency_key,state,created_at)
|
|
VALUES($1,$2,$3,$1::uuid::text,'pending',now()-interval '3 minutes')`, [orphanId, orphanPlanId, prepared.principal.connectionId])
|
|
// Exercise the recovery predicate while the real runner is held at admission.
|
|
await recoverMcpOperations()
|
|
assert.equal((await application.query("SELECT state FROM mcp_operations WHERE id=$1", [admitted.id])).rows[0].state, "pending")
|
|
assert.deepEqual((await application.query("SELECT state,error_code FROM mcp_operations WHERE id=$1", [orphanId])).rows[0],
|
|
{ state: "unknown", error_code: "PROCESS_INTERRUPTED" })
|
|
assert.equal(serviceCalls, before)
|
|
releaseMaintenance()
|
|
await maintenance
|
|
assert.equal((await terminal(admitted.id)).state, "applied")
|
|
assert.equal((await applyPlan(prepared.principal, prepared.planId, key)).id, admitted.id)
|
|
assert.equal(serviceCalls - before, 2)
|
|
checks.push("recovery preserves a queued operation under its original plan deadline, closes an expired orphan, and the queued operation executes both steps once")
|
|
} finally {
|
|
releaseMaintenance()
|
|
await maintenance?.catch(() => {})
|
|
}
|
|
}
|
|
for (const fault of ["lost-own", "lost-zero", "rival-operation-lock", "callback-error", "preclaim-permission"] as const) {
|
|
reset(fault); const prepared = await prepare(false), before = serviceCalls, operation = await applyPlan(prepared.principal, prepared.planId, randomUUID())
|
|
if (fault === "lost-zero") {
|
|
await bounded(cleanupGate); assert.equal(cleanupRows, 0)
|
|
const row = (await application.query("SELECT state,result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0]
|
|
assert.equal(row.state, "running"); assert.equal(row.result[root.MCP_OPERATION_PRIVATE_CLAIM_KEY], rivalNonce)
|
|
await application.query("UPDATE public.mcp_operations SET state='unknown',result='{}'::jsonb,finished_at=now() WHERE id=$1 AND result->>'__mmappMcpOperationClaimV1'=$2", [operation.id, rivalNonce])
|
|
} else { const row = await terminal(operation.id); assert.equal(row.state, "unknown"); assert.equal(row.error_code, "PROCESS_INTERRUPTED") }
|
|
if (heldRival) { await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]); heldRival = undefined }
|
|
assert.equal(serviceCalls, before)
|
|
}
|
|
checks.push("actual one-row/zero-row ACK loss, real rival operation lock refusal and post-real-lock callback error and pre-claim permission loss: no legacy service/native replay; only own nonce becomes unknown, rival winner preserved")
|
|
reset("dual-claim"); const prepared = await prepare(false), anchor = root.beginMcpOperationAdmission(prepared.principal), wall = Date.now(), mono = performance.now()
|
|
await application.query("BEGIN")
|
|
let issued: Awaited<ReturnType<typeof root.insertMcpOperationAdmission>>
|
|
try {
|
|
const plan = (await application.query("SELECT * FROM public.mcp_plans WHERE id=$1 FOR UPDATE", [prepared.planId])).rows[0]
|
|
const grant = await readMcpTemporaryGrantOrigin(application, prepared.principal.connectionId, prepared.principal.credentialId), expires = Math.min(plan.expires_at.getTime(), Date.parse(prepared.principal.credentialExpiresAt))
|
|
issued = await root.insertMcpOperationAdmission(application, anchor, prepared.planId, randomUUID(), { ...grant, monotonicEnd: grant.expiresAt === null ? Infinity : mono + grant.expiresAt - wall, admissionExpiresAt: expires, admissionMonotonicEnd: mono + expires - wall })
|
|
await application.query("UPDATE public.mcp_plans SET state='executing' WHERE id=$1", [prepared.planId]); assert.equal((await application.query("COMMIT")).command, "COMMIT")
|
|
} catch (error) { await application.query("ROLLBACK"); throw error }
|
|
const beforeEntries = legacyEntries
|
|
const outcomes = await Promise.allSettled([0, 1].map(() => root.withMcpOperationExecutionOrigin(issued.admission, async () => { callbackContenders++ })))
|
|
const finished = await terminal(issued.operation.id)
|
|
assert.equal(arrivals, 1); assert.deepEqual(claimRows, [1]); assert.equal(outcomes.filter(value => value.status === "rejected").length, 1)
|
|
assert.equal(legacyEntries - beforeEntries + callbackContenders, 1); assert.equal(finished.state, "unknown")
|
|
checks.push("actual creating-transaction INSERT owns private one-shot admission: two pending publishers enter one callback/one real CAS; separate rival case proves actual zero-row ACK isolation")
|
|
const later = await prepare(false), lateStart = root.beginMcpOperationAdmission(later.principal), latePlan = (await application.query("SELECT expires_at FROM public.mcp_plans WHERE id=$1", [later.planId])).rows[0]
|
|
const lateGrant = await readMcpTemporaryGrantOrigin(application, later.principal.connectionId, later.principal.credentialId), end = Math.min(latePlan.expires_at.getTime(), Date.parse(later.principal.credentialExpiresAt))
|
|
await assert.rejects(root.insertMcpOperationAdmission(application, lateStart, later.planId, randomUUID(), { ...lateGrant, monotonicEnd: Infinity, admissionExpiresAt: end, admissionMonotonicEnd: performance.now() + end - Date.now() }))
|
|
assert.equal((await application.query("SELECT id FROM public.mcp_operations WHERE plan_id=$1", [later.planId])).rows.length, 0)
|
|
checks.push("later/autocommit caller cannot mint admission or orphan an INSERT; no existing-pending UPDATE mint API")
|
|
return { checks, actualPublicFixtureRunner: true, preLockCaptures, legacyEntries, serviceCalls, claimRows, actualSql: true, nativeCalls: 0, globalWriterActivated: false,
|
|
terminalRestoreRecoveryCovered: false, extraProductionPools: 0, extraProductionProcesses: 0, privateNonceExposed: false }
|
|
} finally {
|
|
proceed?.(); pool.query = query; Client.prototype.query = clientQuery
|
|
if (heldRival) await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]).catch(() => {})
|
|
application.release(true); rival.release(true)
|
|
}
|
|
}
|