Files
MikrotikManager/backend/src/modules/mcp/operation-execution-origin.sql-helper.ts
T

212 lines
20 KiB
TypeScript

/** Actual public fixture runner/admission/legacy locks, real SQL ACK faults.
* No native HTTP, production registration or default global-writer activation. */
import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { performance } from "node:perf_hooks"
import { setTimeout as delay } from "node:timers/promises"
import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js"
export async function acceptOwnedMcpOperationRoot() {
assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true")
const { pool } = await import("../../db/index.js"), { Client } = await import("pg"), { z } = await import("zod")
const { add } = await import("./actions.js"), { registerMcpService } = await import("./service-registry.js")
const { createMcpConnection, authenticateMcpToken } = await import("./store.js")
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
const { invokeMcpTool, approvePlan, applyPlan, recoverMcpOperations } = await import("./runtime.js")
const root = await import("./operation-execution-origin.js")
const { readMcpTemporaryGrantOrigin } = await import("./temporary-grant-origin.js")
const { captureVerifiedMcpServiceExecution } = await import("../../services/config-mutation-policy.js")
const operationName = "fixture.operation-root." + randomUUID(), serviceName = "fixture.operation-root-service." + randomUUID(), tool = "mm_owned_operation_root_" + randomUUID().replaceAll("-", "")
const secret = "OWNED_PRIVATE_ROOT_PAYLOAD_" + randomUUID(), application = await pool.connect(), rival = await pool.connect()
const query = pool.query, clientQuery = Client.prototype.query, ids: number[] = [], checks: string[] = []
const expected = new Map<string, { approved: boolean; ownerId: string }>(), caps = new Map<string, unknown>(), lockKinds = new Map<string, Set<string>>()
let sourceFaultApplied = false
let mode = "normal", serviceCalls = 0, preLockCaptures = 0, legacyEntries = 0, callbackContenders = 0
let selectedOperation: string | undefined, heldRival: string | undefined, cleanupRows = -1, claimRows: number[] = [], rivalNonce: string | undefined
let reached!: () => void, proceed!: () => void, cleanupFinished!: () => void
let reachedGate = Promise.resolve(), proceedGate = Promise.resolve(), cleanupGate = Promise.resolve(), arrivals = 0
const reset = (next: string) => { mode = next; sourceFaultApplied = false; selectedOperation = undefined; cleanupRows = -1; claimRows = []; rivalNonce = undefined; arrivals = 0
reachedGate = new Promise<void>(yes => { reached = yes }); proceedGate = new Promise<void>(yes => { proceed = yes }); cleanupGate = new Promise<void>(yes => { cleanupFinished = yes }) }
const bounded = async <T>(promise: Promise<T>): Promise<T> => {
let timer: ReturnType<typeof setTimeout> | undefined
try { return await Promise.race([promise, new Promise<never>((_yes, no) => { timer = setTimeout(() => no(Error("Owned operation root fixture deadline")), 20000) })]) }
finally { clearTimeout(timer) }
}
const terminal = async (id: string) => {
for (const until = Date.now() + 20000; Date.now() < until;) {
const row = (await application.query("SELECT state,result,error_code FROM public.mcp_operations WHERE id=$1", [id])).rows[0]
if (row && !["pending", "running"].includes(row.state)) return row
await delay(10)
}
throw Error("Owned operation did not reach terminal state")
}
try {
for (let i = 0; i < 2; i++) ids.push(Number((await application.query("INSERT INTO public.servers(name,host,enabled) VALUES('owned-operation-root','127.0.0.1',false) RETURNING id")).rows[0].id))
await application.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
registerMcpService(serviceName, { current: async () => ({ fixture: "before" }), verify: async (_input, result) => !!result && typeof result === "object" && "fixture" in result,
execute: async () => { serviceCalls++; const old = await captureVerifiedMcpServiceExecution(); assert(old); await assert.rejects(root.captureMcpOperationExecutionOrigin()); return { fixture: "done" } } })
add(operationName, tool, "Owned operation root acceptance", "servers", "mm:servers:write", "write", "servers", z.object({ password: z.string() }).strict(), { version: 1,
target: () => ({ scope: "servers", serverIds: ids }), prepare: async input => ({ target: { scope: "servers", serverIds: ids }, objects: ids.map(serverId => ({ kind: "server", id: String(serverId), serverId })), snapshots: [], before: { fixture: "before" }, warnings: [],
steps: [0, 1].map(index => ({ kind: "service", service: serviceName, input: { index, password: input.password } })) }) })
const cleanupPids = new Set<number>()
Client.prototype.query = function(this: InstanceType<typeof Client>, ...args: unknown[]) {
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
const key = String(values?.[0] ?? ""), opLock = text.includes("pg_try_advisory_lock") && key.startsWith("mcp-operation:")
const legacy = opLock ? "operation" : text.includes("pg_try_advisory_lock") && text.includes("mm-config:application") ? "application" : text.includes("pg_try_advisory_lock") && /^mm-config:\d+$/.test(key) ? "server" : undefined
const physical = this as unknown as { processID: number }
const run = async () => {
if (opLock) {
const id = key.slice("mcp-operation:".length); selectedOperation = id; legacyEntries++
const cap = await root.captureMcpOperationExecutionOrigin(), metadata = await root.readCurrentMcpOperationExecutionOrigin(cap), planned = expected.get(metadata.planId); assert(planned)
assert.equal(metadata.operationId, id); assert.equal(metadata.approved, planned.approved); assert.equal(metadata.ownerId, planned.ownerId)
assert.equal(metadata.evidence.steps.length, 2); assert.equal(metadata.privateInput.password, secret); assert.deepEqual(metadata.serverIds, ids); assert(!("stepIndex" in metadata))
const held = await rival.query("SELECT count(*)::int AS n FROM pg_catalog.pg_locks WHERE pid=$1 AND locktype='advisory' AND granted", [physical.processID]); assert.equal(held.rows[0].n, 0)
caps.set(id, cap); preLockCaptures++
if (mode === "hold-public") { reached(); await proceedGate }
if (mode === "rival-operation-lock") { await rival.query("SELECT pg_advisory_lock(hashtextextended($1,0))", [key]); heldRival = key }
} else if (legacy && selectedOperation) {
await assert.rejects(root.captureMcpOperationExecutionOrigin()); await root.readCurrentMcpOperationExecutionOrigin(caps.get(selectedOperation))
}
if (legacy && selectedOperation) { const set = lockKinds.get(selectedOperation) ?? new Set<string>(); set.add(legacy); lockKinds.set(selectedOperation, set) }
const result = await Reflect.apply(clientQuery, this, args)
if (opLock && mode === "callback-error") throw Error("Owned callback failure after actual operation lock ACK")
if (text.startsWith("UPDATE public.mcp_operations SET state='unknown'") && values?.length === 4) { cleanupRows = result.rowCount; cleanupPids.add(physical.processID) }
if (text === "COMMIT" && cleanupPids.delete(physical.processID)) cleanupFinished()
return result
}
// Only direct Promise driver calls are instrumented asynchronously. All
// pg-pool callback calls and their results stay completely unchanged.
if (legacy || text.startsWith("UPDATE public.mcp_operations SET state='unknown'") || text === "COMMIT" && cleanupPids.has(physical.processID)) return run()
return Reflect.apply(clientQuery, this, args)
} as typeof Client.prototype.query
pool.query = function(this: typeof pool, ...args: unknown[]) {
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string })?.text ?? "", values = (Array.isArray(args[1]) ? args[1] : (args[0] as { values?: unknown[] })?.values) as unknown[] | undefined
if (text.startsWith("/* mcp-operation-root:source */") && mode === "preclaim-permission" && !sourceFaultApplied) return (async () => {
const result = await Reflect.apply(query, this, args); sourceFaultApplied = true
await application.query("UPDATE public.mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [values![1]])
return result
})()
if (!text.startsWith("/* mcp-operation-root:claim */")) return Reflect.apply(query, this, args)
return (async () => {
if (mode === "dual-claim") arrivals++
if (mode === "lost-zero") { rivalNonce = randomUUID(); const won = await application.query("UPDATE public.mcp_operations SET state='running',result=jsonb_build_object('__mmappMcpOperationClaimV1',$2::text) WHERE id=$1 AND state='pending'", [values![0], rivalNonce]); assert.equal(won.rowCount, 1) }
const result = await Reflect.apply(query, this, args); claimRows.push(result.rowCount)
if (mode === "lost-own") { assert.equal(result.rowCount, 1); throw Error("Owned actual one-row claim ACK lost") }
if (mode === "lost-zero") { assert.equal(result.rowCount, 0); throw Error("Owned actual zero-row claim ACK lost") }
return result
})()
} as typeof pool.query
async function prepare(approved: boolean) {
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned operation root", apps: ["mm"], permissions: ["mm:servers:write", "mm:mcp:approve"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 300000 }
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Owned operation root", mode: approved ? "approval" : "autonomous", permissions: ["mm:servers:write"], expiresAt: new Date(actor.expiresAt).toISOString(), scope: { serverIds: ids, operationIds: [operationName, "operations.read"], allowGlobal: false },
...(approved ? {} : { dangerousPolicy: "explicit-allow", autonomousDangerousOperationIds: [operationName] }), limits: { callsPerMinute: 500, mutationsPerHour: 1000 } }), actor)
const principal = await authenticateMcpToken(access.token), plan = await invokeMcpTool(principal, tool, { password: secret }) as { planId: string }
if (approved) await approvePlan(plan.planId, actor, true)
expected.set(plan.planId, { approved, ownerId: actor.id })
return { principal, planId: plan.planId }
}
for (const approved of [true, false]) {
reset("hold-public"); const prepared = await prepare(approved), before = serviceCalls
const operation = await applyPlan(prepared.principal, prepared.planId, randomUUID()); await bounded(reachedGate)
const privateRow = (await application.query("SELECT result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0].result
const nonce = privateRow[root.MCP_OPERATION_PRIVATE_CLAIM_KEY]; assert.equal(typeof nonce, "string")
const model = await invokeMcpTool(prepared.principal, "mm_operation_get", { operationId: operation.id }), web = await applyPlan(prepared.principal, prepared.planId, randomUUID())
for (const output of [model, web]) { assert(!JSON.stringify(output).includes(nonce)); assert(!JSON.stringify(output).includes(root.MCP_OPERATION_PRIVATE_CLAIM_KEY)) }
proceed(); const finished = await terminal(operation.id); assert.equal(finished.state, "applied"); assert.equal(serviceCalls - before, 2)
assert.deepEqual([...lockKinds.get(operation.id)!].sort(), ["application", "operation", "server"])
await assert.rejects(root.readCurrentMcpOperationExecutionOrigin(caps.get(operation.id)))
}
checks.push("actual human/autonomous public fixture runner publishes full root before operation/app/server locks, retains per-step origins and hides nonce from real model/direct-web apply results")
{
const { withMcpDatabaseRequest } = await import("./database-request-lease.js")
const { SYSTEM_DATABASE_REQUEST_LOCK, withSystemDatabaseOperationLock } = await import("../../db/operation-lock.js")
reset("normal")
const prepared = await prepare(true), before = serviceCalls, key = randomUUID()
let releaseMaintenance!: () => void, enteredMaintenance!: () => void
const release = new Promise<void>(resolve => { releaseMaintenance = resolve })
const entered = new Promise<void>(resolve => { enteredMaintenance = resolve })
let maintenance: Promise<void> | undefined
try {
const admitted = await withMcpDatabaseRequest(async () => {
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
enteredMaintenance()
await release
})
void maintenance.catch(() => {})
for (const end = Date.now() + 2000; Date.now() < end;) {
const queued = await application.query(
"SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid=0 AND objid=$1 AND mode='ExclusiveLock' AND NOT granted",
[SYSTEM_DATABASE_REQUEST_LOCK])
if (queued.rowCount) return applyPlan(prepared.principal, prepared.planId, key)
await delay(10)
}
throw Error("Owned maintenance lock did not queue")
})
await bounded(entered)
await delay(350)
await application.query("UPDATE mcp_operations SET created_at=now()-interval '3 minutes' WHERE id=$1", [admitted.id])
const orphanId = randomUUID(), orphanPlanId = randomUUID()
await application.query(`INSERT INTO mcp_plans
(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at)
SELECT $1,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,'executing',now()-interval '1 minute'
FROM mcp_plans WHERE id=$2`, [orphanPlanId, prepared.planId])
await application.query(`INSERT INTO mcp_operations (id,plan_id,connection_id,idempotency_key,state,created_at)
VALUES($1,$2,$3,$1::uuid::text,'pending',now()-interval '3 minutes')`, [orphanId, orphanPlanId, prepared.principal.connectionId])
// Exercise the recovery predicate while the real runner is held at admission.
await recoverMcpOperations()
assert.equal((await application.query("SELECT state FROM mcp_operations WHERE id=$1", [admitted.id])).rows[0].state, "pending")
assert.deepEqual((await application.query("SELECT state,error_code FROM mcp_operations WHERE id=$1", [orphanId])).rows[0],
{ state: "unknown", error_code: "PROCESS_INTERRUPTED" })
assert.equal(serviceCalls, before)
releaseMaintenance()
await maintenance
assert.equal((await terminal(admitted.id)).state, "applied")
assert.equal((await applyPlan(prepared.principal, prepared.planId, key)).id, admitted.id)
assert.equal(serviceCalls - before, 2)
checks.push("recovery preserves a queued operation under its original plan deadline, closes an expired orphan, and the queued operation executes both steps once")
} finally {
releaseMaintenance()
await maintenance?.catch(() => {})
}
}
for (const fault of ["lost-own", "lost-zero", "rival-operation-lock", "callback-error", "preclaim-permission"] as const) {
reset(fault); const prepared = await prepare(false), before = serviceCalls, operation = await applyPlan(prepared.principal, prepared.planId, randomUUID())
if (fault === "lost-zero") {
await bounded(cleanupGate); assert.equal(cleanupRows, 0)
const row = (await application.query("SELECT state,result FROM public.mcp_operations WHERE id=$1", [operation.id])).rows[0]
assert.equal(row.state, "running"); assert.equal(row.result[root.MCP_OPERATION_PRIVATE_CLAIM_KEY], rivalNonce)
await application.query("UPDATE public.mcp_operations SET state='unknown',result='{}'::jsonb,finished_at=now() WHERE id=$1 AND result->>'__mmappMcpOperationClaimV1'=$2", [operation.id, rivalNonce])
} else { const row = await terminal(operation.id); assert.equal(row.state, "unknown"); assert.equal(row.error_code, "PROCESS_INTERRUPTED") }
if (heldRival) { await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]); heldRival = undefined }
assert.equal(serviceCalls, before)
}
checks.push("actual one-row/zero-row ACK loss, real rival operation lock refusal and post-real-lock callback error and pre-claim permission loss: no legacy service/native replay; only own nonce becomes unknown, rival winner preserved")
reset("dual-claim"); const prepared = await prepare(false), anchor = root.beginMcpOperationAdmission(prepared.principal), wall = Date.now(), mono = performance.now()
await application.query("BEGIN")
let issued: Awaited<ReturnType<typeof root.insertMcpOperationAdmission>>
try {
const plan = (await application.query("SELECT * FROM public.mcp_plans WHERE id=$1 FOR UPDATE", [prepared.planId])).rows[0]
const grant = await readMcpTemporaryGrantOrigin(application, prepared.principal.connectionId, prepared.principal.credentialId), expires = Math.min(plan.expires_at.getTime(), Date.parse(prepared.principal.credentialExpiresAt))
issued = await root.insertMcpOperationAdmission(application, anchor, prepared.planId, randomUUID(), { ...grant, monotonicEnd: grant.expiresAt === null ? Infinity : mono + grant.expiresAt - wall, admissionExpiresAt: expires, admissionMonotonicEnd: mono + expires - wall })
await application.query("UPDATE public.mcp_plans SET state='executing' WHERE id=$1", [prepared.planId]); assert.equal((await application.query("COMMIT")).command, "COMMIT")
} catch (error) { await application.query("ROLLBACK"); throw error }
const beforeEntries = legacyEntries
const outcomes = await Promise.allSettled([0, 1].map(() => root.withMcpOperationExecutionOrigin(issued.admission, async () => { callbackContenders++ })))
const finished = await terminal(issued.operation.id)
assert.equal(arrivals, 1); assert.deepEqual(claimRows, [1]); assert.equal(outcomes.filter(value => value.status === "rejected").length, 1)
assert.equal(legacyEntries - beforeEntries + callbackContenders, 1); assert.equal(finished.state, "unknown")
checks.push("actual creating-transaction INSERT owns private one-shot admission: two pending publishers enter one callback/one real CAS; separate rival case proves actual zero-row ACK isolation")
const later = await prepare(false), lateStart = root.beginMcpOperationAdmission(later.principal), latePlan = (await application.query("SELECT expires_at FROM public.mcp_plans WHERE id=$1", [later.planId])).rows[0]
const lateGrant = await readMcpTemporaryGrantOrigin(application, later.principal.connectionId, later.principal.credentialId), end = Math.min(latePlan.expires_at.getTime(), Date.parse(later.principal.credentialExpiresAt))
await assert.rejects(root.insertMcpOperationAdmission(application, lateStart, later.planId, randomUUID(), { ...lateGrant, monotonicEnd: Infinity, admissionExpiresAt: end, admissionMonotonicEnd: performance.now() + end - Date.now() }))
assert.equal((await application.query("SELECT id FROM public.mcp_operations WHERE plan_id=$1", [later.planId])).rows.length, 0)
checks.push("later/autocommit caller cannot mint admission or orphan an INSERT; no existing-pending UPDATE mint API")
return { checks, actualPublicFixtureRunner: true, preLockCaptures, legacyEntries, serviceCalls, claimRows, actualSql: true, nativeCalls: 0, globalWriterActivated: false,
terminalRestoreRecoveryCovered: false, extraProductionPools: 0, extraProductionProcesses: 0, privateNonceExposed: false }
} finally {
proceed?.(); pool.query = query; Client.prototype.query = clientQuery
if (heldRival) await rival.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", [heldRival]).catch(() => {})
application.release(true); rival.release(true)
}
}