fix(mcp): обновить проверенный снимок маршрутизации и хранения операций

This commit is contained in:
Denozordec committed 2026-11-04 17:14:57 +07:00
1 parent f1af1a85e5
commit 5ce56b5168
117 files changed
+9960 -182

No files matched your search

+7 -1
View File
@@ -14,6 +14,7 @@ import { logFiltersSchema, logSettingsPatchSchema } from "@mmapp/contracts"
import { appUserCreateSchema, appUserUpdateSchema, appUserRoleSchema, sectionPermSchema, serverPermSchema, userBindingCreateSchema } from "@mmapp/contracts/users"
import type { McpOperationDescriptor } from "@mmapp/contracts/mcp"
import { hasPermission } from "../../lib/permissions.js"
import { assertOperationReplayScope, findOperationReplayTombstone, operationHistoryExpired } from "./operation-replay-tombstones.js"
import { pool } from "../../db/index.js"
import { getServerReadById } from "../servers/service/servers-service.js"
import { listUsers, getUserById, listInterfaceCatalog, getInterfaceCatalogState } from "../users/service/users-service.js"
@@ -473,7 +474,12 @@ add("changes.cancel", "mm_change_cancel", "Отменить свой незап
add("operations.read", "mm_operation_get", "Результат своей операции", "mcp", null, "read", "connection", z.object({ operationId: z.string().uuid() }).strict(), {
read: async (input, context) => {
const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND connection_id=$2", [input.operationId, context.connection.id])
if (!operation) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
if (!operation) {
const tombstone = await findOperationReplayTombstone(pool, context.connection.id, String(input.operationId))
if (!tombstone) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
assertOperationReplayScope(tombstone, context.connection)
operationHistoryExpired()
}
await assertPlanReadScope(await (await import("./runtime.js")).loadPlan(operation.plan_id), context)
const { rows: steps } = await pool.query("SELECT * FROM mcp_operation_steps WHERE operation_id=$1 ORDER BY step_index", [input.operationId])
return { ...operation, steps }
@@ -115,3 +115,9 @@ export const createDiagnosticCancellationSourceForTests = sourceLoader
/** Historical authentication only; confers no execution/cancellation grant. */
export const readServerProbeExecutionSource = readSourceLoader(productionCrypto)
export const createReadServerProbeExecutionSourceForTests = readSourceLoader
/** Exact previously locked source projection for private history conversion. */
export function authenticateServerProbeExecutionSourceRow(row: Record<string, unknown>, operationId: string,
owner: Readonly<DiagnosticCancellationOwner>) {
checkLookup(operationId, owner)
return validateSourceRow(row, operationId, owner, productionCrypto)
}
@@ -0,0 +1,73 @@
import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { test } from "node:test"
import type { McpAction, ActionContext } from "./action-types.js"
import { registerGlobalReconciliationActionsForTests } from "./global-reconciliation-actions.js"
type Register = Parameters<typeof registerGlobalReconciliationActionsForTests>[0]
type Dependencies = Parameters<typeof registerGlobalReconciliationActionsForTests>[1]
function fixture() {
const connectionId = randomUUID(), operationId = randomUUID(), ownerId = "current-retired-owner"
const context = { principal: { connectionId }, connection: { id: connectionId, owner_id: ownerId, owner_permissions: ["mm:servers:write"],
policy: { permissions: ["mm:servers:write"], scope: { allowGlobal: true, operationIds: ["servers.poll", "operations.global.reconcile"], serverIds: [1, 2] } } } } as ActionContext
let action!: Pick<McpAction, "target" | "read">
let history: { affectedServerIds: number[] } | undefined = { affectedServerIds: [1, 2] }
let revoke = false, retiredCalls = 0, live = false
const deps = {
connection: async () => {
if (revoke) throw Error("revoked-current-grant")
return structuredClone(context.connection)
},
query: async (sql: string) => {
if (sql.startsWith("SELECT id FROM servers")) return [{ id: 1 }]
if (live) return [{ id: operationId, state: "applied", operation_id: "servers.poll", evidence: { sealed: "live" } }]
return []
},
operation: (id: string) => ({ id, scope: "global", requiredPermissions: ["mm:servers:write"] }),
service: () => ({ reconcile: async () => ({ matchesDesiredState: true, matchesOriginalState: false }) }),
decrypt: async () => JSON.stringify({ target: { scope: "global" }, objects: [{ serverId: 1 }], steps: [{ kind: "service", service: "servers.poll", input: {} }] }),
retired: async (identity, authorize) => {
retiredCalls++
assert.deepEqual(identity, { operationId, connectionId, ownerId })
await authorize()
return history
},
} satisfies Omit<Dependencies, "operation" | "service"> & Partial<Record<"operation" | "service", unknown>>
registerGlobalReconciliationActionsForTests(((...args: Parameters<Register>) => { action = args[8] }) as Register, deps as unknown as Dependencies)
return { context, operationId, history: (value: typeof history) => { history = value }, live: () => { live = true },
revoke: () => { revoke = true }, calls: () => retiredCalls, read: () => action.read!({ operationId }, context),
target: () => action.target({ operationId }, context) }
}
test("global action reconciles retired poll only under current ownership and returns no archived material", async () => {
const f = fixture()
assert.deepEqual(await f.target(), { scope: "global" })
const result = await f.read() as { observedAt: string }
const { observedAt, ...rest } = result
assert(Number.isFinite(Date.parse(observedAt)))
assert.deepEqual(rest, { operationId: f.operationId, state: "applied", automaticRetry: false,
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }] })
assert.equal(f.calls(), 2)
})
test("global action refuses missing or inaccessible retired history", async () => {
const f = fixture()
f.history(undefined)
await assert.rejects(f.read(), { code: "NOT_FOUND" })
f.history({ affectedServerIds: [1, 2] })
f.context.connection.policy.scope.serverIds = [1]
await assert.rejects(f.read(), { code: "SERVER_DENIED" })
f.context.connection.policy.scope.serverIds = [1, 2]
f.context.connection.policy.scope.operationIds = ["operations.global.reconcile"]
await assert.rejects(f.read(), { code: "PERMISSION_DENIED" })
f.revoke()
await assert.rejects(f.read(), /revoked-current-grant/)
})
test("global action uses the live reconciliation path while original source exists", async () => {
const f = fixture()
f.live()
await f.read()
assert.equal(f.calls(), 0)
})
@@ -1,5 +1,6 @@
import { z } from "zod"
import { pool } from "../../db/index.js"
import { readRetiredServerPollHistory } from "../../services/server-poll-history-reader.js"
import { hasPermission } from "../../lib/permissions.js"
import { effectiveMcpConnection } from "./store.js"
import { decryptSecret, fingerprint, McpError } from "./security.js"
@@ -7,40 +8,84 @@ import { findMcpOperation } from "./registry.js"
import { findMcpService } from "./service-registry.js"
import type { add } from "./actions.js"
import type { ActionContext, ChangeEvidence } from "./action-types.js"
import type { ConnectionRow } from "./store.js"
async function authorizedSource(operationId: string, context: ActionContext) {
const production = {
connection: effectiveMcpConnection,
query: async (sql: string, values: unknown[] = []) => (await pool.query(sql, values)).rows,
operation: findMcpOperation,
service: findMcpService,
decrypt: decryptSecret,
retired: readRetiredServerPollHistory,
}
type Dependencies = typeof production
const notFound = () => new McpError("NOT_FOUND", "Операция не найдена", 404)
async function currentAccess(context: ActionContext, deps: Dependencies) {
context.signal?.throwIfAborted()
const current = await effectiveMcpConnection(context.principal)
const current = await deps.connection(context.principal)
if (!current.policy.scope.allowGlobal || fingerprint(current.policy) !== fingerprint(context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(context.connection.owner_permissions)) throw new McpError("PERMISSION_DENIED", "Глобальный допуск сверки изменился", 403)
const ids = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).rows.map(row => Number(row.id))
const ids = (await deps.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).map(row => Number(row.id))
if (ids.length > 1000) throw new McpError("LIMIT_EXCEEDED", "Каталог глобальной сверки превышает лимит", 409)
if (ids.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Глобальная сверка требует доступа ко всему каталогу серверов", 403)
const row = (await pool.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id])).rows[0]
if (!row) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
const original = findMcpOperation(row.operation_id)
return current
}
function requireOriginal(actionId: string, current: ConnectionRow, deps: Dependencies) {
const original = deps.operation(actionId)
if (!original || original.scope !== "global" || !current.policy.scope.operationIds.includes(original.id) || original.requiredPermissions.some(permission => !hasPermission(current.policy.permissions, permission) || !hasPermission(current.owner_permissions, permission))) throw new McpError("PERMISSION_DENIED", "Утрачено право на исходную глобальную операцию", 403)
const evidence = JSON.parse(await decryptSecret(row.evidence.sealed)) as ChangeEvidence
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !findMcpService(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
const declared = evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId])
}
function requireServerScope(declared: number[], current: ConnectionRow) {
if (declared.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Зависимость операции вне текущего глобального допуска", 403)
}
async function authorizedSource(operationId: string, context: ActionContext, deps: Dependencies) {
const current = await currentAccess(context, deps)
const row = (await deps.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id]))[0]
if (!row) {
const authorize = async () => {
const fresh = await currentAccess(context, deps)
if (fresh.id !== current.id || fresh.owner_id !== current.owner_id) throw notFound()
}
const archived = await deps.retired({ operationId, connectionId: current.id, ownerId: current.owner_id }, authorize,
{ deadlineAt: Date.now() + 10000, signal: context.signal })
context.signal?.throwIfAborted()
if (!archived) throw notFound()
requireOriginal("servers.poll", current, deps)
requireServerScope(archived.affectedServerIds, current)
return { kind: "retired-poll" as const }
}
requireOriginal(row.operation_id, current, deps)
const evidence = JSON.parse(await deps.decrypt(row.evidence.sealed)) as ChangeEvidence
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !deps.service(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
requireServerScope(evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId]), current)
if (["pending", "running"].includes(row.state)) throw new McpError("STATE_CONFLICT", "Операция ещё выполняется", 409)
context.signal?.throwIfAborted()
return { row, evidence }
return { kind: "live" as const, row, evidence }
}
export function registerGlobalReconciliationActions(register: typeof add) {
function registerActions(register: typeof add, deps: Dependencies) {
register("operations.global.reconcile", "mm_global_operation_reconcile", "Сверить глобальную сервисную операцию без повторной записи", "mcp", null, "read", "global", z.object({ operationId: z.string().uuid() }).strict(), {
target: async (input, context) => { await authorizedSource(String(input.operationId), context); return { scope: "global" } },
target: async (input, context) => { await authorizedSource(String(input.operationId), context, deps); return { scope: "global" } },
read: async (input, context) => {
const { row, evidence } = await authorizedSource(String(input.operationId), context)
const authorize = async () => { await authorizedSource(String(input.operationId), context) }
const operationId = String(input.operationId)
const source = await authorizedSource(operationId, context, deps)
if (source.kind === "retired-poll") return { operationId, state: "applied", observedAt: new Date().toISOString(),
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }], automaticRetry: false }
const { row, evidence } = source
const authorize = async () => { if ((await authorizedSource(operationId, context, deps)).kind !== "live") throw notFound() }
const results = []
for (const [index, step] of evidence.steps.entries()) {
if (step.kind !== "service") throw new McpError("UNSUPPORTED_CAPABILITY", "Нативная глобальная сверка недоступна", 409)
await authorize()
results.push({ stepIndex: index, service: step.service, ...await findMcpService(step.service)!.reconcile!(step.input, authorize) })
results.push({ stepIndex: index, service: step.service, ...await deps.service(step.service)!.reconcile!(step.input, authorize) })
}
await authorize()
return { operationId: row.id, state: row.state, observedAt: new Date().toISOString(), results, automaticRetry: false }
},
})
}
export const registerGlobalReconciliationActions = (register: typeof add) => registerActions(register, production)
export const registerGlobalReconciliationActionsForTests = registerActions
@@ -5,6 +5,8 @@ await prepareOwnedBandwidthState()
const { pool, closePool } = await import("../../db/index.js")
try {
const { applySqlMigrations } = await import("../../db/migrate.js"); await applySqlMigrations(pool)
const { acceptOperationReplayTombstones } = await import("./operation-replay-tombstones.sql-helper.js")
await acceptOperationReplayTombstones()
const { acceptOwnedMcpOperationRoot } = await import("./operation-execution-origin.sql-helper.js")
console.log("Owned actual MCP operation root:", JSON.stringify(await acceptOwnedMcpOperationRoot()))
} finally { await closePool() }
@@ -0,0 +1,74 @@
import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js"
export async function acceptOperationReplayTombstones() {
assertOwnedBandwidthSql()
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true")
const { pool } = await import("../../db/index.js")
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
const { createMcpConnection, authenticateMcpToken } = await import("./store.js")
const { applyPlan, invokeMcpTool } = await import("./runtime.js")
const { operationIdempotencyDigest } = await import("./operation-replay-tombstones.js")
const { encryptSecret, McpError } = await import("./security.js")
const { default: Fastify } = await import("fastify")
const { default: authPlugin } = await import("../../plugins/auth.js")
const { createMcpRoutes } = await import("../../routes/mcp.js")
const { env } = await import("../../config.js")
const { SignJWT } = await import("jose")
const makeConnection = async () => {
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned replay fixture", apps: ["mm"],
permissions: ["mm:users:write", "mm:mcp:read"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 }
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval",
permissions: actor.permissions, expiresAt: new Date(actor.expiresAt).toISOString(),
scope: { serverIds: [], operationIds: ["users.create", "operations.read", "changes.apply"], allowGlobal: true },
limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor)
return { actor, principal: await authenticateMcpToken(issued.token) }
}
const app = Fastify()
const ids: string[] = []
const matches = (code: string, status: number) => (error: unknown) =>
error instanceof McpError && error.code === code && error.statusCode === status
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
try {
const owner = await makeConnection(), foreign = await makeConnection()
// A real completed plan has already initialized the owned encryption state.
// The fixture seeds its tombstone directly, so exercise the same initializer.
await encryptSecret("owned replay fixture state")
const connectionId = owner.principal.connectionId
ids.push(connectionId, foreign.principal.connectionId)
const operationId = randomUUID(), planId = randomUUID(), key = "owned-replay-" + randomUUID()
await pool.query(`INSERT INTO public.mcp_operation_replay_tombstones
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
VALUES($1,$2,$3,$4,'users.create','{}',true)`, [operationId, planId, connectionId, operationIdempotencyDigest(connectionId,key)])
await assert.rejects(applyPlan(owner.principal, planId, key), matches("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(applyPlan(owner.principal, planId, randomUUID()), matches("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(applyPlan(owner.principal, randomUUID(), key), matches("STATE_CONFLICT",409))
await assert.rejects(applyPlan(foreign.principal, planId, key), matches("NOT_FOUND",404))
await assert.rejects(invokeMcpTool(owner.principal,"mm_change_apply",{ planId,idempotencyKey:key }), matches("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(invokeMcpTool(foreign.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
await app.register(authPlugin)
await app.register(createMcpRoutes(async () => {}))
for (const subject of [owner, foreign]) {
const token = await new SignJWT({ sub: subject.actor.id, email: subject.actor.email, name: subject.actor.name,
apps: subject.actor.apps, permissions: subject.actor.permissions })
.setProtectedHeader({ alg:"HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("4m")
.sign(new TextEncoder().encode(env.jwtSecret))
const reply = await app.inject({ method: "POST", url: "/mcp/plans/"+planId+"/apply", headers: { authorization:"Bearer "+token } })
assert.equal(reply.statusCode, subject === owner ? 410 : 404)
assert.equal(reply.json().code, subject === owner ? "OPERATION_HISTORY_EXPIRED" : "NOT_FOUND")
}
await pool.query("UPDATE public.mcp_connections SET policy=jsonb_set(policy,'{scope,allowGlobal}','false') WHERE id=$1", [connectionId])
await assert.rejects(applyPlan(owner.principal, planId, key), matches("NOT_FOUND",404))
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
const { rows:[proof] } = await pool.query(`SELECT
NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE connection_id=ANY($1::uuid[])) AS no_operation,
NOT EXISTS(SELECT 1 FROM public.mcp_budget_buckets WHERE connection_id=ANY($1::uuid[]) AND kind='mutations') AS no_write_budget`, [ids])
assert.deepEqual(proof,{no_operation:true,no_write_budget:true})
console.log("Owned operation tombstones: scoped MCP and Web 410, replay conflict and no dispatch")
} finally {
await app.close()
await pool.query("DELETE FROM public.mcp_operation_replay_tombstones WHERE connection_id=ANY($1::uuid[])", [ids])
}
}
@@ -0,0 +1,64 @@
import { createHash } from "node:crypto"
import type { PoolClient } from "pg"
import { hasPermission } from "../../lib/permissions.js"
import { listActions } from "./actions.js"
import { McpError } from "./security.js"
import { effectiveMcpConnection, type ConnectionRow, type McpPrincipal } from "./store.js"
type Query = Pick<PoolClient, "query">
export interface OperationReplayTombstone {
operation_id: string
plan_id: string
connection_id: string
action_id: string
target_server_ids: string[]
target_global: boolean
}
// Idempotency tokens are caller input and already stored in plaintext on live
// operations. This domain-separated digest avoids copying that plaintext and
// survives DB restore and access-key rotation. It is not a keyed secret proof.
export function operationIdempotencyDigest(connectionId: string, key: string): string {
return createHash("sha256").update("mcp-operation-replay-v1\0").update(connectionId).update("\0").update(key).digest("hex")
}
export function operationHistoryExpired(): never {
throw new McpError("OPERATION_HISTORY_EXPIRED", "Срок хранения завершённой операции истёк. Повторное применение запрещено.", 410)
}
export function assertOperationReplayScope(row: OperationReplayTombstone, connection: ConnectionRow): void {
const scope = connection.policy.scope
if (row.connection_id !== connection.id || !scope.operationIds.includes(row.action_id)) throw operationReplayNotFound()
const action = listActions().find(candidate => candidate.operation.id === row.action_id)
if (!action || action.operation.requiredPermissions.some(permission =>
!hasPermission(connection.policy.permissions, permission) || !hasPermission(connection.owner_permissions, permission))) throw operationReplayNotFound()
if (row.target_global && !scope.allowGlobal || row.target_server_ids.some(id => !scope.serverIds.some(server => String(server) === id))) throw operationReplayNotFound()
}
function operationReplayNotFound(): McpError {
return new McpError("NOT_FOUND", "Операция не найдена", 404)
}
const projection = "t.operation_id,t.plan_id,t.connection_id,t.action_id,t.target_server_ids::text[] AS target_server_ids,t.target_global"
export async function findOperationReplayTombstone(db: Query, connectionId: string, operationId: string) {
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
WHERE t.connection_id=$1 AND t.operation_id=$2`, [connectionId, operationId])).rows[0]
}
export async function findOwnerPlanReplayTombstone(db: Query, ownerId: string, planId: string) {
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
JOIN mcp_connections c ON c.id=t.connection_id WHERE c.owner_id=$1 AND t.plan_id=$2`, [ownerId, planId])).rows[0]
}
export async function refuseExpiredOperationReplay(db: Query, principal: McpPrincipal, connectionId: string, planId: string, key: string): Promise<void> {
const rows = (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
WHERE t.connection_id=$1 AND (t.plan_id=$2 OR t.idempotency_digest=$3)`,
[connectionId, planId, operationIdempotencyDigest(connectionId, key)])).rows
if (!rows.length) return
const conflict = rows.find(row => row.plan_id !== planId.toLowerCase())
const current = await effectiveMcpConnection(principal, db)
assertOperationReplayScope(conflict ?? rows[0], current)
if (conflict) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409)
operationHistoryExpired()
}
@@ -0,0 +1,148 @@
import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { setTimeout as delay } from "node:timers/promises"
import { assertOwnedBandwidthSql,prepareOwnedBandwidthState } from "../../services/bandwidth-admission-process.sql-helper.js"
assertOwnedBandwidthSql()
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true")
await prepareOwnedBandwidthState()
const {pool,closePool}=await import("../../db/index.js")
const {applySqlMigrations}=await import("../../db/migrate.js")
const {mcpConnectionCreateSchema}=await import("@mmapp/contracts/mcp")
const {createMcpConnection,authenticateMcpToken}=await import("./store.js")
const {invokeMcpTool,approvePlan,applyPlan}=await import("./runtime.js")
const {OrdinaryRetirementCommitUnknown,retireOrdinaryMcpOperations}=await import("./ordinary-operation-retirement.js")
const {tickMcpWorkers}=await import("./workers.js")
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
const {Client}=await import("pg")
const {McpError}=await import("./security.js")
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
try {
await applySqlMigrations(pool)
await pauseSchedulerJobsAndWait()
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
async function access(){
const actor={id:randomUUID(),email:"[email protected]",name:"Owned settings retention",apps:["mm"],
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"],
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
"servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true},
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
return {actor,principal:await authenticateMcpToken(issued.token)}
}
const {actor,principal}=await access(),foreign=(await access()).principal
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
settings:Record<string,unknown>={retentionDays}){
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
let terminal
for(const deadline=Date.now()+15000;Date.now()<deadline;){
terminal=(await pool.query("SELECT state,verification,error_code FROM mcp_operations WHERE id=$1",[applied.id])).rows[0]
if(terminal&&!["pending","running"].includes(terminal.state))break
await delay(50)
}
assert.deepEqual(terminal,{state:"applied",verification:"passed",error_code:null})
// Only the disposable fixture ages rows, after real authenticated apply.
// No cutoff override or production encryption/approval bypass.
const finished=new Date(Date.now()-ageMs),created=new Date(finished.getTime()-1000)
await pool.query("UPDATE mcp_plans SET created_at=$2,approved_at=$2,expires_at=$3 WHERE id=$1",
[plan.planId,created,new Date(finished.getTime()+1000)])
await pool.query("UPDATE mcp_operations SET created_at=$2,finished_at=$3 WHERE id=$1",[applied.id,created,finished])
await pool.query("UPDATE mcp_operation_steps SET created_at=$2,finished_at=$3 WHERE operation_id=$1",[applied.id,created,finished])
return {id:applied.id,planId:plan.planId,key}
}
const age=180*86400000
const held=await operation(51,age+180000),spent=await operation(52,age+120000)
const due=await operation(53,age+60000),young=await operation(54,age-60000)
const boundary=await pool.connect()
try{
await boundary.query("BEGIN")
await boundary.query("UPDATE mcp_operations SET finished_at=transaction_timestamp()-interval '180 days' WHERE id=$1",[young.id])
await assert.rejects(boundary.query(`INSERT INTO mcp_ordinary_retirement_claims
(operation_id,plan_id,connection_id,source_proof,claim_token) VALUES($1,$2,$3,$4,$5)`,
[young.id,young.planId,principal.connectionId,"a".repeat(64),randomUUID()]),/cannot be replaced or reopened/)
}finally{await boundary.query("ROLLBACK");boundary.release()}
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[randomUUID(),held.id])
const ids=[held.id,spent.id,due.id,young.id]
const projection=async()=>(await pool.query(`SELECT i.id,o.state,p.state AS plan_state,s.state AS step_state,
k.operation_id AS checkpoint,t.operation_id AS tombstone,c.claim_token
FROM unnest($1::uuid[]) i(id)
LEFT JOIN mcp_operations o ON o.id=i.id LEFT JOIN mcp_plans p ON p.id=o.plan_id
LEFT JOIN mcp_operation_steps s ON s.operation_id=i.id LEFT JOIN mcp_operation_checkpoints k ON k.operation_id=i.id
LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=i.id
LEFT JOIN mcp_ordinary_retirement_claims c ON c.operation_id=i.id ORDER BY i.id`,[ids])).rows
const originalQuery=Client.prototype.query
let commits=0
try{
Client.prototype.query=function(this:InstanceType<typeof Client>,...args:unknown[]){
if((args[0] as {text?:string})?.text==="COMMIT"&&++commits===3)
return Promise.reject(Error("owned ordinary destructive COMMIT transport loss"))
return Reflect.apply(originalQuery,this,args)
} as typeof Client.prototype.query
await assert.rejects(retireOrdinaryMcpOperations(),
(error:unknown)=>error instanceof OrdinaryRetirementCommitUnknown&&error.automaticReplay===false)
}finally{Client.prototype.query=originalQuery}
assert.equal(commits,3)
const before=await projection(),spentBefore=before.find(row=>row.id===spent.id)
assert.equal(spentBefore?.state,"applied");assert(spentBefore?.claim_token);assert.equal(spentBefore?.tombstone,null)
await tickMcpWorkers()
const after=await projection()
assert.deepEqual(after.filter(row=>row.id!==due.id),before.filter(row=>row.id!==due.id))
const retired=after.find(row=>row.id===due.id)
assert(retired?.claim_token);assert.equal(retired?.tombstone,due.id)
for(const field of ["state","plan_state","step_state","checkpoint"])assert.equal(retired[field],null)
assert.equal((await pool.query("SELECT 1 FROM mcp_plans WHERE id=$1",[due.planId])).rows.length,0)
assert.equal((await pool.query("SELECT retention_days FROM router_log_settings WHERE id=1")).rows[0].retention_days,54)
await tickMcpWorkers()
assert.deepEqual(await projection(),after)
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(applyPlan(principal,randomUUID(),due.key),errorIs("STATE_CONFLICT",409))
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:due.id}),errorIs("NOT_FOUND",404))
const telemetry=[
{action:"traffic.settings.update",version:2,record:await operation(55,age+60000,"mm_traffic_settings_update_plan")},
{action:"uptime.settings.update",version:2,record:await operation(56,age+60000,"mm_uptime_settings_update_plan",{enabled:false,retentionDays:56})},
{action:"servers.api-ping.settings.update",version:2,record:await operation(0,age+60000,"mm_servers_api_ping_settings_update_plan",{intervalSec:123})},
{action:"internet-path.settings.update",version:1,record:await operation(57,age+60000,"mm_internet_path_settings_update_plan")},
]
const telemetryRows=async()=>(await pool.query(`SELECT p.operation_id,p.operation_version,o.id
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=ANY($1::uuid[]) ORDER BY p.operation_id`,
[telemetry.map(item=>item.record.id)])).rows
assert.deepEqual((await telemetryRows()).map(row=>[row.operation_id,row.operation_version]),
telemetry.map(item=>[item.action,item.version]).sort((a,b)=>String(a[0]).localeCompare(String(b[0]))))
const telemetrySettings=async()=>(await pool.query(`SELECT to_jsonb(t) AS traffic,to_jsonb(u) AS uptime,
to_jsonb(a) AS api_ping,to_jsonb(i) AS internet FROM traffic_settings t,uptime_settings u,
servers_api_ping_settings a,internet_path_settings i WHERE t.id=1 AND u.id=1 AND a.id=1 AND i.id=1`)).rows[0]
const settingsBefore=await telemetrySettings()
const traffic=telemetry[0].record
const {rows:[originalStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[traffic.id])
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{retentionDays}','999') WHERE operation_id=$1",[traffic.id])
await tickMcpWorkers()
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[traffic.id])).rows.length,0)
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[traffic.id])).rows.length,1)
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[traffic.id,originalStep.result])
await tickMcpWorkers()
await tickMcpWorkers()
// The keyset owner may need one tick to wrap after the rejected oldest row.
await tickMcpWorkers()
assert.deepEqual(await telemetryRows(),[])
assert.deepEqual(await telemetrySettings(),settingsBefore)
for(const {action,record} of telemetry){
const {rows:[proof]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[record.id,record.planId])
assert.deepEqual(proof,{action_id:action,target_global:true,server_ids:[String(serverId)],plan_gone:true,step_gone:true,checkpoint_gone:true})
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
}
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
}finally{resumeSchedulerJobs();await closePool()}
@@ -0,0 +1,199 @@
import { randomUUID } from "node:crypto"
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
import { z } from "zod"
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
import { operationIdempotencyDigest } from "./operation-replay-tombstones.js"
import { decryptSecretExisting, fingerprint, privateFingerprinterExisting, redact } from "./security.js"
const uuid = z.string().uuid()
const positive = z.number().int().positive().safe()
const object = z.record(z.string(),z.unknown())
const sealed = z.object({sealed:z.string().min(1).max(262144)}).strict()
const ids = z.array(positive).max(1000)
const retention = z.number().int().min(1).max(365)
const bool = z.boolean()
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
// offset. Accept that exact wire form as well as an ISO timestamptz.
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
.refine(value=>Number.isFinite(Date.parse(value.replace(" ","T").replace(/([+-]\d\d)$/,"$1:00"))))
const metadata = {
id:z.literal(1),createdAt:timestamp,updatedAt:timestamp,
lastCollectedAt:timestamp.nullable(),lastDurationMs:z.number().int().safe().nullable(),
lastError:z.string().max(16384).nullable(),
}
function telemetryFamily(version:number,domain:string,settings:z.ZodObject,hasEpoch:boolean,hasConfigurationProof:boolean){
const originalSettings=hasConfigurationProof?settings.extend({configurationProof:z.string().regex(/^[0-9a-f]{64}$/)}):settings
const effect=settings.extend({...metadata,...(hasEpoch?{pollConfigurationEpoch:uuid}:{})}).strict()
return {version,domain,patch:settings.partial().strict().refine(value=>Object.keys(value).length>0),
before:z.object({value:originalSettings.strict(),serverIds:ids}).strict(),effect}
}
const families={
"logs.settings.update":{version:1,domain:null,patch:logSettingsPatchSchema,before:logSettingsSchema.strict(),effect:logSettingsSchema.strict()},
"traffic.settings.update":telemetryFamily(2,"traffic-settings",z.object({enabled:bool,intervalSec:interval(5,3600),retentionDays:retention}),true,true),
"uptime.settings.update":telemetryFamily(2,"uptime-settings",z.object({enabled:bool,resourcesEnabled:bool,pingEnabled:bool,
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
} as const
const planEvidence = z.object({
target:z.object({scope:z.literal("global")}).strict(),
objects:z.array(z.object({kind:z.string(),id:z.string(),serverId:positive.optional()}).strict()).max(1001),
steps:z.tuple([z.object({kind:z.literal("service"),service:z.string(),input:z.object({settings:object}).strict()}).strict()]),
snapshots:z.tuple([]),before:object,warnings:z.array(z.string()).max(8),
verificationScope:z.literal("configuration").optional(),
requiresHuman:z.literal(true).optional(),
}).strict()
class RetirementRefused extends Error {}
export class OrdinaryRetirementCommitUnknown extends Error {
readonly automaticReplay = false
constructor() { super("Ordinary operation retirement outcome unknown; automatic replay forbidden") }
}
const requireThat: (value: unknown) => asserts value = value => {
if (!value) throw new RetirementRefused()
}
// Fixed reviewed settings families without artifact, historic receipt or
// inverse-plan consumers. Other action families need separate review.
const ELIGIBLE = `FROM public.mcp_operations o
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
WHERE (p.operation_id,p.operation_version) IN (
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
) AND p.state='completed'
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
AND isfinite(p.expires_at) AND p.expires_at<transaction_timestamp()
AND s.state='verified' AND s.server_id IS NULL AND isfinite(s.finished_at)
AND k.after_sealed IS NOT NULL
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
AND (SELECT count(*) FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)=1
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
const CANDIDATES = `SELECT o.id,o.connection_id,o.finished_at::text AS finished_at ${ELIGIBLE}
AND NOT EXISTS(SELECT 1 FROM public.mcp_ordinary_retirement_claims WHERE operation_id=o.id)
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
ORDER BY o.finished_at,o.id LIMIT 8`
const SOURCE = `SELECT to_jsonb(o) AS operation,to_jsonb(p) AS plan,to_jsonb(s) AS step,to_jsonb(k) AS checkpoint ${ELIGIBLE}
AND o.id=$1 AND o.connection_id=$2
AND octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)+octet_length(to_jsonb(k)::text)<=1048576
FOR UPDATE OF o,p,s,k NOWAIT`
async function transaction<T>(deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise<T>):Promise<T> {
const sql = await connectProbeStorage({deadlineAt})
let committing=false,committed=false
try {
await sql.query("BEGIN")
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
const result=await work(sql)
committing=true
await sql.query("COMMIT")
committed=true
return result
} catch(error) {
if(committing)throw new OrdinaryRetirementCommitUnknown()
try{await sql.query("ROLLBACK")}catch{/* Destroy this checkout. */}
throw error
} finally {
sql.release(!committed)
}
}
async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:string) {
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT",[connectionId])).rows.length===1)
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",["mcp-operation:"+operationId])).rows[0]?.locked===true)
const rows=(await sql.query(SOURCE,[operationId,connectionId])).rows
requireThat(rows.length===1)
const raw=rows[0], operation=object.parse(raw.operation), plan=object.parse(raw.plan)
const step=object.parse(raw.step),checkpoint=object.parse(raw.checkpoint)
const open=async(value:unknown)=>JSON.parse(await decryptSecretExisting(sealed.parse(value).sealed))
const original=planEvidence.parse(await open(plan.evidence))
const action=z.enum(Object.keys(families) as [keyof typeof families,...(keyof typeof families)[]]).parse(plan.operation_id)
const family=families[action],service=original.steps[0]
requireThat(plan.operation_version===family.version&&service.service===action)
requireThat(fingerprint(await open(plan.payload))===fingerprint(service.input))
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
const patch=object.parse(family.patch.parse(service.input.settings))
const effect=object.parse(family.effect.parse(step.result))
requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value))
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
}
const serverObjects=original.objects.filter(item=>item.kind==="server")
const serverIds=ids.parse(serverObjects.map(item=>item.serverId))
requireThat(new Set(serverIds).size===serverIds.length&&serverObjects.every(item=>item.id===String(item.serverId)))
const domainObjects=original.objects.filter(item=>item.kind!=="server")
const before=family.before.parse(original.before)
if(family.domain===null){
requireThat(domainObjects.length===0&&original.requiresHuman===undefined)
}else{
requireThat(original.requiresHuman===true&&original.verificationScope==="configuration"
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
}
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
}
async function retire(operationId:string,connectionId:string,deadlineAt:number) {
const token=randomUUID()
await transaction(deadlineAt,async sql=>{
const source=await prepare(sql,operationId,connectionId)
const claim=await sql.query(`INSERT INTO public.mcp_ordinary_retirement_claims(operation_id,plan_id,connection_id,source_proof,claim_token)
VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING RETURNING operation_id`,[operationId,source.planId,connectionId,source.proof,token])
requireThat(claim.rows.length===1&&claim.rows[0].operation_id===operationId)
})
await transaction(deadlineAt,async sql=>{
const claim=(await sql.query(`SELECT source_proof,plan_id FROM public.mcp_ordinary_retirement_claims
WHERE operation_id=$1 AND connection_id=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,[operationId,connectionId,token])).rows
requireThat(claim.length===1)
const source=await prepare(sql,operationId,connectionId)
requireThat(claim[0].source_proof===source.proof&&claim[0].plan_id===source.planId)
const tombstone=await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
VALUES($1,$2,$3,$4,$5,$6,true) RETURNING operation_id`,
[operationId,source.planId,connectionId,operationIdempotencyDigest(connectionId,source.key),source.action,source.serverIds])
requireThat(tombstone.rows.length===1&&tombstone.rows[0].operation_id===operationId)
const checkpoint=await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
const step=await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
const operation=await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id AS operation_id",[operationId,connectionId,source.planId])
const plan=await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id AS plan_id",[source.planId,connectionId])
requireThat([checkpoint,step,operation].every(result=>result.rows.length===1&&result.rows[0].operation_id===operationId)
&&plan.rows.length===1&&plan.rows[0].plan_id===source.planId)
})
}
let cursor:{finishedAt:string;operationId:string}|undefined
export function retireOrdinaryMcpOperations() {
return sharedDatabaseGate.request(async()=>{
const deadlineAt=Date.now()+15000
const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES,[cursor?.finishedAt??null,cursor?.operationId??null])).rows)
requireThat(rows.length<=8)
let scanned=0,retired=0
for(const row of rows){
if(Date.now()>=deadlineAt-2000||retired>=2)break
const id=uuid.parse(row.id),connectionId=uuid.parse(row.connection_id)
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
cursor={finishedAt:row.finished_at,operationId:id}
scanned++
try{await retire(id,connectionId,deadlineAt);retired++}
catch(error){if(error instanceof OrdinaryRetirementCommitUnknown)throw error}
}
if(scanned===rows.length&&rows.length<8)cursor=undefined
return {scanned,retired,automaticReplay:false as const}
})
}
@@ -1,5 +1,8 @@
import assert from "node:assert/strict"
import { readFile } from "node:fs/promises"
import { test } from "node:test"
import { runInNewContext } from "node:vm"
import ts from "typescript"
import type { ActionContext } from "./action-types.js"
import { loadReservedScopedRecoverySource } from "./routing-community-scoped-recovery-source.js"
import { executeReservedScopedRecovery, scopedRecoveryIntentFromCheckpoint, validateReservedInverseVerified } from "./routing-community-scoped-recovery.js"
@@ -16,6 +19,67 @@ test("a serialized allocation cannot enter the shared inverse engine", async ()
await assert.rejects(executeReservedScopedRecovery(forged, {} as Parameters<typeof executeReservedScopedRecovery>[1]), /capability/)
})
test("confirmed inverse spend quarantines the exact slot on post-spend refusal without replay", async () => {
// Execute the actual boundary body with only its I/O dependencies substituted.
// This does not mint a production capability or claim native/SQL acceptance.
const source = ts.createSourceFile("routing-community-scoped-recovery.ts",
await readFile(new URL("./routing-community-scoped-recovery.ts", import.meta.url), "utf8"), ts.ScriptTarget.ES2022)
const declaration = source.statements.find(node => ts.isFunctionDeclaration(node) && node.name?.text === "executeReservedScopedRecovery")
assert(declaration)
const compiled = ts.transpileModule(declaration.getText(source), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText
const allocation = Object.freeze({ kind: "routing-reserved-inverse-allocation" })
const execution = Object.freeze({ kind: "routing-reserved-inverse-execution" })
const principal = { connectionId: "owned", credentialId: "original" }
const original = new Error("original authority failure"), result = Object.freeze({ outcome: "recovered" })
const scenarios = {
spend: ["spend"],
inspect: ["spend", "inspect", "quarantine"],
authority: ["spend", "inspect", "authority", "quarantine"],
source: ["spend", "inspect", "authority", "source", "quarantine"],
proof: ["spend", "inspect", "authority", "source", "proof", "quarantine"],
engine: ["spend", "inspect", "authority", "source", "proof", "engine", "quarantine"],
success: ["spend", "inspect", "authority", "source", "proof", "engine"],
} as const
for (const [scenario, expected] of Object.entries(scenarios)) {
const events: string[] = [], exports: Record<string, unknown> = {}, capabilities = new WeakMap()
let request: object | undefined
const at = (phase: string) => { events.push(phase); if (phase === scenario) throw original }
runInNewContext(compiled, {
exports, reservedExecutions: capabilities, fingerprint: JSON.stringify,
createRoutingReservationStore: () => ({
async beginInverseAttempt(value: unknown) {
assert.equal(value, allocation); at("spend"); return { reservationId: "spent-slot", execution }
},
async inspectInverseExecution(value: unknown) {
assert.equal(value, execution); at("inspect")
return { principal, sourceOperationId: "source", deadline: "2026-10-29T01:00:00.000Z" }
},
async markUnknown(id: string, lane: string) {
assert.equal(id, "spent-slot"); assert.equal(lane, "inverse"); at("quarantine")
throw new Error("quarantine persistence unavailable")
},
}),
effectiveMcpConnection: async () => { at("authority"); return { owner_id: "owner", policy: {}, owner_permissions: [] } },
loadReservedScopedRecoverySource: async () => {
at("source"); return { profile: { serverId: 1 }, input: {}, configuration: { packet: {} } }
},
accessGeneration: async () => "generation",
deadline: () => Date.parse("2026-10-29T01:00:00.000Z"),
sealedProof: async () => { at("proof"); return "proof" },
executeScopedRecovery: async (value: object) => {
request = value; assert.equal(capabilities.get(value), execution); at("engine"); return result
},
})
const execute = exports.executeReservedScopedRecovery as
(allocation: unknown, expected: { sourceOperationId: string; principal: typeof principal }) => Promise<unknown>
const outcome = execute(allocation, { sourceOperationId: "source", principal })
if (scenario === "success") assert.equal(await outcome, result)
else await assert.rejects(outcome, error => error === original)
assert.deepEqual(events, expected, scenario)
if (request) assert.equal(capabilities.has(request), false)
}
})
test("configuration equality or generic VERIFIED cannot replace reserved signed receipts", async () => {
const expected = {} as Parameters<typeof validateReservedInverseVerified>[1]
for (const row of [{}, { state: "verified", phase: "RECOVERY_VERIFIED", configurationVerified: true }, { inverse_intent_sealed: "x".repeat(524289) }]) {
@@ -35,8 +35,9 @@ import { registerManagementChannel } from "./management.js"
import { isSchedulerJobsPaused, pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../../services/scheduler-running.js"
/** Disposable SQL + loopback HTTP only. Does not access or configure CHR. */
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward" } = {}) {
const sid = 131, writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward"; lateInverseAuthorityChange?: "edit" | "disable" | "grant" } = {}) {
const sid = options.lateInverseAuthorityChange ? { edit: 131, disable: 132, grant: 133 }[options.lateInverseAuthorityChange] : 131
const writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
const initial = compileManagedFilterRules([{ community: "65000:10", gateway: "192.0.2.1", outIface: "", action: "route", description: "" }, { community: "65000:20", gateway: "192.0.2.5", outIface: "", action: "route", description: "" }])
const filters = ["a", "b"].map(key => ({ ".id": "*f" + key, chain: "custom-" + key, rule: initial, disabled: "false", comment: managedComment("NEVER_SCOPED_COMMENT_" + key), opaque: "NEVER_SCOPED_OPAQUE" }))
let mode: "normal" | "ignored" | "lost" | "collateral" | "post-failed" | "post-unknown" = "normal", writes = 0, requests = 0
@@ -423,8 +424,9 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
await assertOriginalCredentialRevocation()
return {accepted:true,originalCredentialRevocation:true,unauthorized401:true,fullAdmissionRollback:true,nativeReads:0,nativeWrites:0,realSql:true,allWriterLedgerComplete:false}
}
if (options.manualInverseChargeOnly) {
if (options.manualInverseChargeOnly || options.lateInverseAuthorityChange) {
const chargeCase = options.manualInverseChargeCase ?? "positive"
if (options.lateInverseAuthorityChange) assert.equal(chargeCase, "positive")
const persistent = chargeCase !== "nonpersistent"
// Same genuine fixed-reader baseline/CAS used by the existing persistent
// acceptance below. No hand-written baseline, sealed plan or authority.
@@ -573,6 +575,48 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
realSql: true, allWriterLedgerComplete: false, reservedDispatchEnabled: false, autoRecoveryEnabled: false }
}
console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-apply" }))
const lateChange = options.lateInverseAuthorityChange
if (lateChange) {
const priorIntent = (await pool.query("SELECT coalesce(max(id),0)::text AS id FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent'", [connectionId])).rows[0].id
let writeId: string | undefined
afterIntent = async () => {
const intents = (await pool.query(`SELECT payload FROM mcp_audit WHERE connection_id=$1 AND id>$2::bigint
AND action='configuration.intent' AND payload->>'path'=$3`,
[connectionId, priorIntent, "/routing/filter/rule/" + loaded.configuration.packet.nativeId])).rows
assert.equal(intents.length, 1)
assert.equal(intents[0].payload.method, "PATCH")
assert.equal(intents[0].payload.serverId, sid)
writeId = intents[0].payload.writeId
assert.match(writeId ?? "", /^[0-9a-f-]{36}$/)
const sql = {
edit: "UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late inverse edit\"') WHERE id=$1",
disable: "UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1",
grant: "UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1",
}[lateChange]
assert.equal((await pool.query(sql, [id])).rowCount, 1)
}
const inverse = await execute(recovery.planId)
assert.equal(afterIntent, undefined)
assert.ok(writeId)
assert.notEqual(inverse.state, "applied")
assert.equal(writes, afterForward.writes)
assert.equal(inversePostChecks, 0)
const auditRows = (await pool.query(`SELECT action FROM mcp_audit WHERE connection_id=$1 AND payload->>'writeId'=$2
ORDER BY id`, [connectionId, writeId])).rows
assert.deepEqual(auditRows.map(row => row.action), ["configuration.intent", "configuration.unconfirmed"])
const refusal = (await pool.query(`SELECT payload->>'phase' AS phase, payload->>'code' AS code
FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed'`, [connectionId])).rows
assert.deepEqual(refusal, [{ phase: "before-dispatch-context", code: "STATE_CONFLICT" }])
assert.deepEqual((await pool.query(`SELECT state,verified_sealed FROM mcp_scoped_recovery_intents
WHERE source_operation_id=$1 AND recovery_operation_id=$2`, [source.id, inverse.id])).rows,
[{ state: "unknown", verified_sealed: null }])
assert.equal((await pool.query(`SELECT count(*)::int AS n FROM mcp_operation_checkpoints
WHERE operation_id=$1 AND configuration_sealed IS NOT NULL`, [inverse.id])).rows[0].n, 0)
assert.equal((await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows[0].error_code,
"MUTATION_OUTCOME_UNKNOWN")
return { accepted: true, lateInverseAuthorityChange: lateChange, durableIntents: 1, inverseNativeWrites: 0,
realSql: true, realMcpDispatcher: true, fullInverseLifecycleRepeated: false }
}
if (chargeCase === "quota" || chargeCase === "revoked") {
if (chargeCase === "revoked") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id])
const beforeOperations = (await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n
@@ -257,24 +257,30 @@ async function checkpoint(value: Session, intent: Intent, phase: Checkpoint["pha
export async function executeReservedScopedRecovery(allocation: RoutingReservedInverseAllocation, expected: { sourceOperationId: string; principal: McpPrincipal }) {
const store = createRoutingReservationStore(), spent = await store.beginInverseAttempt(allocation)
const execution = spent.execution
const binding = await store.inspectInverseExecution(execution)
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
const current = await effectiveMcpConnection(binding.principal)
const context: ActionContext = { principal: binding.principal, connection: current }
const source = await loadReservedScopedRecoverySource(execution, context)
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
input.requestProof = await sealedProof(input, "request")
reservedExecutions.set(input, execution)
try { return await executeScopedRecovery(input) }
catch (error) { await store.markUnknown(binding.reservationId, "inverse").catch(() => {}); throw error }
finally { reservedExecutions.delete(input) }
try {
const binding = await store.inspectInverseExecution(execution)
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
const current = await effectiveMcpConnection(binding.principal)
const context: ActionContext = { principal: binding.principal, connection: current }
const source = await loadReservedScopedRecoverySource(execution, context)
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
input.requestProof = await sealedProof(input, "request")
reservedExecutions.set(input, execution)
try { return await executeScopedRecovery(input) }
finally { reservedExecutions.delete(input) }
} catch (error) {
// Acknowledged spend owns this exact slot even when its authority can no
// longer be inspected. A refused or uncertain spend never enters here.
await store.markUnknown(spent.reservationId, "inverse").catch(() => {})
throw error
}
}
/** Fixed-domain receipt verifier used by the reservation store. It recognizes
@@ -14,7 +14,9 @@ import { nativeBgpRows, bgpTemplateReferences } from "../../services/bgp-export-
import { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow } from "../../services/bgp-filter-binding-model.js"
import { canonicalVrfMembers } from "../../services/vrf-members.js"
import { readOspfEffectiveDisabled } from "../../services/ospf-effective-disabled.js"
import { isConfirmedRoutingFibTable } from "../../services/routing-table-fib.js"
import * as contextModel from "./routing-community-context-model.js"
import { prepareOriginalEffectSelection } from "./routing-original-effects.js"
import { readMeasurementAddress } from "./routing-measurement-address.js"
import { routeContextClients } from "./routing-community-context-native.js"
import { pathProofTag } from "./path-proof-purpose.js"
@@ -65,9 +67,8 @@ async function fixture(autoRecovery = false) {
afterHttp?.(); afterHttp = undefined
if (afterPath?.path === path) { const run = afterPath.run; afterPath = undefined; run() }
})
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
const now = Date.now(), fingerprint = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
const endpoint = { id: 1, host: "127.0.0.1", port: (router.address() as { port: number }).port, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
const endpoint = { id: 1, host: "127.0.0.1", port: 0, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
const principal = { connectionId: randomUUID(), credentialId: randomUUID(), credentialExpiresAt: new Date(now + 600000).toISOString() }
const connection = { id: principal.connectionId, owner_id: "owner", generation: "generation", owner_permissions: ["mm:network:read"],
policy: { permissions: ["mm:network:read"], scope: { serverIds: [1] }, expiresAt: new Date(now + 600000).toISOString() }, owner_valid_until: new Date(now + 600000) }
@@ -92,6 +93,7 @@ async function fixture(autoRecovery = false) {
"./routing-community-context-native.js": { routeContextClients }, "../../services/vrf-members.js": { canonicalVrfMembers }, "../../services/ospf-effective-disabled.js": { readOspfEffectiveDisabled },
"./routing-measurement-address.js": { readMeasurementAddress } }
await loadSource("./routing-community-context.ts", { ...common, "./routing-network-native-session.js": native,
"../../services/routing-table-fib.js": { isConfirmedRoutingFibTable }, "./routing-original-effects.js": { prepareOriginalEffectSelection },
"../../services/bgp-export-model.js": { nativeBgpRows, bgpTemplateReferences }, "../../services/bgp-filter-binding-model.js": { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow },
"./routing-community-context-model.js": contextModel, "./path-proof-purpose.js": { pathProofTag },
"./management-transport.js": { assertManagementTransport: async () => { throw Error("No alternate management fixture") } } }, context)
@@ -103,14 +105,23 @@ async function fixture(autoRecovery = false) {
await loadSource("./routing-network-scoped-context.ts", { "./routing-network-native-session.js": native }, privateApi)
const create = native.createRoutingNetworkNativeSession as (options: unknown) => Promise<Session>
const abort = new AbortController()
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } })
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
endpoint.port = (router.address() as { port: number }).port
const closeRouter = () => {
router.closeAllConnections()
return new Promise<void>(resolve => router.close(() => resolve()))
}
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } }).catch(async error => {
await closeRouter()
throw error
})
const readContext = privateApi.readRoutingNetworkScopedCommunityContext as (token: Token) => Promise<Context>
return { session, readContext: () => readContext(session.sharedContext), forged: () => readContext(JSON.parse(JSON.stringify(session.sharedContext))),
readResource: native.readRoutingNetworkSharedResource as (token: Token, server: number, path: string, body?: Record<string, string>) => Promise<unknown>,
bypassTail: context.readRouteCommunityContextWithSharedNativeSession as (token: Token) => Promise<unknown>,
counters: () => ({ requests, wireBytes, wireRows, rawGetObjects }), bulk: (value: number) => { bulk = value }, padding: (value: number) => { padding = value },
revoke: () => { revoked = true }, abort: () => abort.abort(), afterHttp: (hook: () => void) => { afterHttp = hook }, afterPath: (path: string, hook: () => void) => { afterPath = { path, run: hook } },
cleanup: async () => { session.close(); router.closeAllConnections(); await new Promise<void>(resolve => router.close(() => resolve())) } }
cleanup: async () => { session.close(); await closeRouter() } }
}
test("actual source context and snapshots share one pinned HTTP request/byte/row budget including object /get", async () => {
@@ -19,7 +19,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
const { pool } = await import("../../db/index.js")
const { mcpConnectionPolicySchema } = await import("@mmapp/contracts/mcp")
const { routeAiProfileSchema } = await import("@mmapp/contracts/mcp-routing")
const { accessGeneration } = await import("./security.js")
const { accessGeneration, McpError } = await import("./security.js")
const { readRoutingEndpoints, createRoutingSourceContext } = await import("./routing-observation-source.js")
const { resolveRoutingProfileAuthority } = await import("./routing-profile-authority.js")
const { captureScopedRecoveryAuthority, scopedRecoveryDeadlineCeiling } = await import("./routing-community-scoped-recovery.js")
@@ -27,7 +27,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
assert.equal((await pool.query("SELECT current_database() AS name")).rows[0].name, "postgres")
const previousEnabled = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled as boolean
const profileIds: string[] = [], ids: number[] = []
let nativeHttpRequests = 0, callbackDispatches = 0, durableIntents = 0
let nativeHttpRequests = 0, callbackDispatches = 0, dispatchFences = 0
const listener = createServer((req, res) => { nativeHttpRequests++; req.resume(); res.setHeader("content-type", "application/json"); res.end("{}") })
try {
await new Promise<void>((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", () => { listener.off("error", reject); resolve() }) })
@@ -65,12 +65,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
const path = "/routing/filter/rule/*1"
await assert.rejects(withVerifiedConfigurationChange({ actorId: ownerId, connectionId, approved: true, serverIds: ids, serverWide: true, objects: [],
authorize: async () => { await captured.assert() }, beforeDispatch: async () => {
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent' AND payload->>'path'=$2", [connectionId, path])).rows[0].n, 1)
durableIntents++
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
await captured.assert()
dispatchFences++
},
}, () => executeGuardedConfigurationWrite(ids[0], "PATCH", path, async () => {
callbackDispatches++
@@ -78,12 +73,17 @@ export async function acceptRoutingRecoveryAuthoritySql() {
const req = request({ hostname: "127.0.0.1", port, path: "/rest" + path, method: "PATCH", timeout: 1000 }, res => { res.resume(); res.once("end", resolve) })
req.once("error", reject); req.once("timeout", () => req.destroy(new Error("Synthetic loopback timeout"))); req.end("{}")
})
})), /Постоянный допуск/)
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0)
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.unconfirmed'", [connectionId])).rows[0].n, 1)
})), (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT" && /Активный routing profile/.test(error.message))
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0); assert.equal(dispatchFences, 0)
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action IN ('configuration.intent','configuration.unconfirmed','configuration.acknowledged')", [connectionId])).rows[0].n, 0)
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
await assert.rejects(captured.assert(), /Постоянный допуск/)
}
return { cases: 3, lateMutations: ["edit", "disable", "grant"], deepCaptureCases: 3, durableIntents, callbackDispatches, nativeHttpRequests,
productionSessionAuthorityFence: true, realSql: true, productionHmac: true, fullInverseLifecycleRepeated: false, controllerEnabled: false, externalNetwork: false }
return { cases: 3, deepCaptureCases: 3, directGuardBeforeIntent: true, dispatchFences, callbackDispatches, nativeHttpRequests,
capturedAuthorityChanges: ["edit", "disable", "grant"], realSql: true, productionHmac: true, controllerEnabled: false, externalNetwork: false }
} finally {
if (profileIds.length) await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=ANY($1::uuid[])", [profileIds])
if (ids.length) await pool.query("UPDATE servers SET enabled=false WHERE id=ANY($1::int[])", [ids])
@@ -2,9 +2,26 @@ import { assertOwnedRoutingRecoveryAuthoritySql, acceptRoutingRecoveryAuthorityS
// Guard precedes imports that configure a pool or load private MCP state.
assertOwnedRoutingRecoveryAuthoritySql()
process.env.AUTH_REQUIRED = "true"
process.env.AUTH_JWT_SECRET = "owned-routing-recovery-authority-acceptance-secret"
process.env.AUTH_ISSUER = "https://owned-routing-recovery-authority.test"
const { pool, closePool } = await import("../../db/index.js")
try {
const { applySqlMigrations } = await import("../../db/migrate.js")
await applySqlMigrations(pool)
console.log("captured recovery authority SQL acceptance:", JSON.stringify(await acceptRoutingRecoveryAuthoritySql()))
} finally { await closePool() }
const { updateMcpSettings } = await import("./store.js")
await import("./actions.js")
const { acceptScopedRouteCommunitySwitch } = await import("./routing-community-scoped-acceptance.js")
const actor = { id: "owned-routing-recovery-authority", email: "[email protected]", name: "Owned recovery authority",
apps: ["mm"], permissions: ["mm:settings:admin", "mm:mcp:admin", "mm:mcp:approve", "mm:network:read", "mm:network:write", "mm:filters:write", "mm:servers:admin"],
issuedAt: Date.now() - 1000, expiresAt: Date.now() + 900000 }
await updateMcpSettings({ enabled: true }, actor)
for (const lateInverseAuthorityChange of ["edit", "disable", "grant"] as const) {
const evidence = await acceptScopedRouteCommunitySwitch(actor, { lateInverseAuthorityChange })
console.log("genuine late inverse authority SQL acceptance:", JSON.stringify(evidence))
}
} finally {
await (await import("../../services/scheduler.js")).stopSchedulerForDatabaseOperation()
await closePool()
}
@@ -352,6 +352,47 @@ export async function acceptRoutingReservationSql(options: { runtimeAdmission?:
await assert.rejects(store.beginForwardAttempt(reservation.forwardAllocation), /capability/)
names.push("spent forward/no genuine checkpoint and preterminal unused-proof refuse; unknown never replays or refunds")
}
{
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
const { saveScopedConfigurationCheckpoint, saveScopedNetworkCheckpoint } = await import("./routing-community-scoped-checkpoint.js")
const { executeReservedScopedRecovery } = await import("./routing-community-scoped-recovery.js")
await pool.query("UPDATE mcp_operations SET state='running' WHERE id=$1", [source.operationId])
const step = { kind: "service", service: "routing.community.scoped", input: source.input }
await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state) VALUES($1,0,'intent',$2::jsonb)",
[source.operationId, JSON.stringify(security.redact(step))])
await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,0,$2)",
[source.operationId, await security.encryptSecret(JSON.stringify(source.input.beforeRules))])
assert(reservation.forwardAllocation)
await store.beginForwardAttempt(reservation.forwardAllocation)
// Use the actual checkpoint writers for production crypto and SQL, but
// deliberately seed a synthetic failed-forward observation. No PATCH
// or genuine network observation is part of this owned component case.
const requestSeal = String(source.input.requestProof)
const configuration = await saveScopedConfigurationCheckpoint(source.input, requestSeal)
await saveScopedNetworkCheckpoint(source.input, requestSeal, { id: randomUUID(), candidateId: "b",
status: "failed", observedAt: configuration.observedAt })
const published = await store.recordForwardOutcome(reservation.reservationId)
assert(published.allocation)
assert.equal(published.receipt.state.inverse, "held")
const refused = { sourceOperationId: randomUUID(), principal: f.principal }
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /Reserved runner identity differs/)
const unknown = await store.read(reservation.reservationId, f.principal)
assert.deepEqual(unknown.state, { ...published.receipt.state, revision: published.receipt.state.revision + 2, inverse: "unknown" })
assert.equal(unknown.chargedMutationUnits, 2)
assert.equal(unknown.chargedProfileSlots, 2)
assert.equal(unknown.refundsAvailable, false)
const shared = (await pool.query(`SELECT state,slot_phase,inverse_intent_sealed,inverse_configuration_sealed,verified_sealed
FROM mcp_scoped_recovery_intents WHERE reservation_id=$1`, [reservation.reservationId])).rows
assert.deepEqual(shared, [{ state: "unknown", slot_phase: "reserved-attempt", inverse_intent_sealed: null,
inverse_configuration_sealed: null, verified_sealed: null }])
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /capability/)
await assert.rejects(store.recordForwardOutcome(reservation.reservationId), /republished/)
await assert.rejects(store.proveUnusedInverse(reservation.reservationId), /terminal applied/)
assert.equal(await used(f.connectionId), 2)
assert.equal((await store.readHourlyCharges(f.profileId, f.principal)).reservedCharges, 2)
assert.deepEqual((await store.read(reservation.reservationId, f.principal)).state, unknown.state)
names.push("genuine inverse capability on synthetic failed-forward checkpoints: acknowledged spend, late identity refusal, signed same-slot unknown and both charges conserved without replay or refund")
}
{
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
const saved = (await pool.query("SELECT state_proof FROM mcp_routing_reservations WHERE id=$1", [reservation.reservationId])).rows[0].state_proof
+19 -4
View File
@@ -1,4 +1,6 @@
import { pollErrorLog } from "../../services/server-poll-error-provenance.js"
import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js"
import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js"
import { preparedPlanExpiry } from "./plan-expiry.js"
import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js"
import { randomUUID, createHash } from "node:crypto"
@@ -23,6 +25,7 @@ import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOpera
import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js"
import { evaluateMcpPolicy } from "./policy.js"
import { findAction, findActionByOperation } from "./actions.js"
import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js"
import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
import { audit, getConnection, transaction, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js"
import { executeRosStep, readRosSnapshot, resourceById, routerClient, type RosSnapshot } from "./resources.js"
@@ -291,6 +294,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
reservedRouting = !!await routingReservationEvidence(original, originalEvidence)
return existing
}
await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey)
await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))")
const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')")
if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429)
@@ -359,6 +363,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise<void> {
return withMcpOperationExecutionOrigin(operationAdmission, async () => {
const lease = await pool.connect()
let terminalProbe: Awaited<ReturnType<typeof executeOperationWithLease>>
let locked = false, released = false
const ownerLifetime = new AbortController()
// This closure is the SINGLE owner of physical destruction/release.
@@ -380,7 +385,7 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
throw error
}
if (released || !locked) return
await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
() => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant))
} finally {
if (!released && locked) {
@@ -395,9 +400,14 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
lease.release()
}
}
// Capture has its own bounded checkout and may fail without rewriting the
// acknowledged terminal outcome or redispatching the diagnostic.
if (terminalProbe) {
try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ }
}
})
}
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant): Promise<void> {
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) {
const { rows: [operation] } = await pool.query<StoredOperation>("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId])
if (!operation) return
const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence
@@ -576,14 +586,18 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
})
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks())
assertOperationConfigurationLockCheckpoint()
await transaction(async client => {
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
const finishedAt = await transaction(async client => {
assertOperationConfigurationLockCheckpoint()
await client.query("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1", [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1"
+ (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""),
[operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
assertOperationConfigurationLockCheckpoint()
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id])
assertOperationConfigurationLockCheckpoint()
await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) })
assertOperationConfigurationLockCheckpoint()
return receiptId ? terminal.rows[0]?.finished_at : undefined
})
assertOperationConfigurationLockCheckpoint()
if (routingReservationId && reservedForwardPassed) {
@@ -592,6 +606,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {})
}
assertOperationConfigurationLockCheckpoint()
if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt }
} catch (error) {
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
@@ -12,9 +12,9 @@ const principal = { connectionId, credentialId, credentialExpiresAt: new Date(no
const proof = async (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
const context = { principal, connection: { id: connectionId, owner_id: "owner", owner_claims_issued_at: new Date(now - 1000) } } as ActionContext
let assertions = 0
function fixture() {
function fixture(retiredSource?: ServerProbeMcpDependencies["retiredSource"]) {
const events: string[] = [], receipts = new Map<string, ServerProbeReceipt>(), contexts: ServerProbeJournalContext[] = [], sealed = new Map<string, string>()
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now, liveHistory = true
const access: ServerProbeAuthority = { transport: "mcp", ownerId: "owner", connectionId, credentialId, expiresAt: now + 600000, generation: "generation", policyProof: "1".repeat(64), ownerProof: "2".repeat(64), active: true, operationAllowed: true, scopeServerIds: scope, ownerPermissions: ["mm:servers:write"], policyPermissions: ["mm:servers:write"] }
const readAuthority = async () => { events.push("authority"); if (revoked) throw new ServerProbeExecutionError("PERMISSION_DENIED", "PRIVATE_PROVIDER_SECRET", 403); return { ...access, expiresAt: clock + 600000, scopeServerIds: [...scope] } }
const assertProtections = async (ids: number[]) => { events.push("protections"); if (ids.some(id => !scope.includes(id))) throw new Error("out of scope") }
@@ -28,6 +28,7 @@ function fixture() {
read: async id => { events.push("journal.read"); return receipts.get(id) },
}
const deps: ServerProbeMcpDependencies = {
retiredSource,
authority: (() => ({ readAuthority, assertProtections })) as ServerProbeMcpDependencies["authority"],
historicalAuthority: ((currentPrincipal: typeof principal) => ({ readAuthority: async () => { events.push("read-authority"); return { ...await readAuthority(), connectionId: currentPrincipal.connectionId, credentialId: currentPrincipal.credentialId } }, assertProtections })) as ServerProbeMcpDependencies["historicalAuthority"],
history: dependencies => createServerProbeJournalHistoryReader({ ...dependencies, loadArchive: async binding => {
@@ -51,7 +52,7 @@ function fixture() {
journal: (ctx, dependencies, execution) => { assert.equal(execution?.deadlineAt, Date.parse(prepared().expiresAt)); contexts.push(ctx); return { ...journal, read: async (...args) => { await dependencies.assertAuthority(ctx, args[1]); return journal.read(...args) }, claim: async binding => { await dependencies.assertAuthority(ctx, binding.bindingProof); return journal.claim(binding) } } },
query: async (sql, values) => {
events.push("sql"); if (revokeOnQuery) revoked = true
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] }
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return liveHistory ? [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] : [] }
if (sql.includes("FROM servers")) { assert.ok(sql.includes("ANY($1::int[])")); assert.deepEqual(values[0], scope); return sql.includes("AND id=$2") ? [] : [{ id: 2, host: "peer", enabled: true }] }
if (!sql.includes("mcp_operation_steps")) return [{ evidence: { sealed: await seal(JSON.stringify(evidence)) } }]
assert.equal(values[0], connectionId); assert.equal(values[1], "owner"); assert.equal(values[2], "servers.probes.run")
@@ -68,8 +69,9 @@ function fixture() {
}, seal, open, proof, now: () => clock,
}
const adapter = createServerProbeMcpAdapter(deps)
const retireHistory = () => { liveHistory = false }
const prepare = async (tool: PreparedServerProbe["tool"] = "ping") => { evidence = await adapter.prepare({ serverId: 9, request: tool === "bandwidth" ? { tool, bwRemoteAddress: "peer", dstServerId: 8 } : { tool, target: "192.0.2.1" } }, context); const step = evidence.steps[0]; assert.equal(step.kind, "service"); return step.kind === "service" ? step.input : assert.fail() }
return { adapter, events, receipts, contexts, prepare, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
return { adapter, events, receipts, contexts, prepare, retireHistory, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
}
for (const tool of ["ping", "traceroute", "bandwidth", "dns", "route", "mtu"] as const) {
@@ -128,6 +130,38 @@ for (const input of [{ serverId: 9, request: { tool: "ping", target: "x", creden
await assert.rejects(f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0, signal: controller.signal })); assert.equal(f.events.filter(event => event === "execute").length, attempted); assertions += 3
}
assert.ok(!("diagnosticProgress" in serverProbeSafeMcpError(new ServerProbeExecutionError("X", "secret", 500, "unknown", 99), { receiptId, operationId, stageBudget: 1 }))); assertions++
{
const hash = "3".repeat(64), date = (value: number) => new Date(value).toISOString().replace("Z", "000Z")
const retired = { source: { version: 1 as const, operation: "servers.probes.run" as const, operationVersion: 1 as const, tool: "ping" as const,
receiptId, executionId: operationId, connectionId, ownerId: "owner", planId: randomUUID(), originalCredentialId: credentialId,
sourceServerId: 9, serverIds: [9], issuedAt: now - 1000, authorityExpiresAt: date(now + 600000), terminalAt: date(now + 600000),
activityAt: date(now + 600000), gcNotBefore: date(now + 87060000), retentionDays: 1, safetyMs: 60000,
bindingProof: hash, sourceDigest: hash, terminalDigest: hash, stageSnapshotDigest: hash, receiptProof: hash, ciphertextSha256: hash,
}, payloadPurged: false }
let lookups = 0, changeOnRecheck = false
const f = fixture(async lookup => {
lookups++
assert.deepEqual(lookup, { receiptId, operationId, connectionId, ownerId: "owner", scopeServerIds: [2, 9] })
return changeOnRecheck && lookups % 2 === 0 ? undefined : retired
})
const input = await f.prepare(), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 })
f.setResult(output); f.finishOperation()
const request = { operationId, receiptId, page: 0 }
const live = await f.adapter.readPage(request, context)
assert.equal(lookups, 0)
f.retireHistory()
const rotated = { ...context, principal: { ...principal, credentialId: randomUUID() } }
assert.deepEqual(await f.adapter.readPage(request, rotated), live)
assert.deepEqual(await f.adapter.receiptTarget(request, rotated), { scope: "servers", serverIds: [9] })
const beforeForeign = lookups
await assert.rejects(f.adapter.readPage(request, { ...rotated, connection: { ...context.connection, owner_id: "foreign" } }))
assert.equal(lookups, beforeForeign)
lookups = 0; changeOnRecheck = true
const privateReads = f.events.filter(event => event === "history.read").length
await assert.rejects(f.adapter.readPage(request, rotated))
assert.equal(f.events.filter(event => event === "history.read").length, privateReads)
assertions += 7
}
{
const f = fixture(), input = await f.prepare("bandwidth"), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 }); f.setResult(output); f.finishOperation(); f.expirePrepared()
await assert.rejects(f.adapter.service.current(input))
@@ -10,6 +10,7 @@ import { serverProbeReceiptPage } from "../../services/server-probe-public-resul
import { decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
import type { ActionContext, ChangeEvidence } from "./action-types.js"
import type { McpServiceHandler } from "./service-registry.js"
import type { findRetiredServerProbeHistory, ServerProbeRetiredLookup } from "../../services/server-probe-retired-history.js"
/** Foundations only: importing this module registers no tools or services. */
export const serverProbeActionSchema = z.object({ serverId: z.number().int().positive().safe(), request: serverProbeExecutionRequest }).strict()
@@ -32,6 +33,7 @@ export interface ServerProbeMcpDependencies {
executor: typeof createProductionServerProbeExecutor
journal: (context: ServerProbeJournalContext, dependencies: ServerProbeJournalDependencies, execution?: { signal?: AbortSignal; deadlineAt?: number }) => ServerProbeJournal
query: Query
retiredSource?: (lookup: ServerProbeRetiredLookup) => ReturnType<typeof findRetiredServerProbeHistory>
seal: (value: string) => Promise<string>
open: (value: string) => Promise<string>
proof: (value: unknown) => Promise<string>
@@ -52,7 +54,7 @@ export async function openServerProbeMcpBinding(raw: Record<string, unknown>, cr
return { input, binding }
}
const ids = (prepared: PreparedServerProbe) => [...new Set([prepared.serverId, ...(prepared.destinationServerId ? [prepared.destinationServerId] : [])])].sort((a, b) => a - b)
function metadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
export function serverProbeReceiptMetadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
const page = serverProbeReceiptPage(receipt, 0)
if (!Number.isSafeInteger(receipt.attemptedStages) || !Number.isSafeInteger(receipt.completedStages) || receipt.completedStages < 0 || receipt.completedStages > receipt.attemptedStages || receipt.attemptedStages > stageBudget) return denied()
return { operationId, receiptId: receipt.receiptId, tool: receipt.tool, status: receipt.state, httpStatus: receipt.httpStatus, stageBudget, attemptedStages: receipt.attemptedStages, completedStages: receipt.completedStages, sha256: page.sha256, totalPages: page.totalPages, bytes: page.bytes, automaticReplay: false, forwardingVerified: false }
@@ -119,7 +121,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
await owned(input, binding, row.operation_id)
await preflight(binding, signal)
if (!receipt || receipt.receiptId !== input.receiptId || receipt.serverId !== binding.prepared.serverId || receipt.destinationServerId !== binding.prepared.destinationServerId || receipt.tool !== binding.prepared.tool) return denied()
return { row, receipt, result: metadata(receipt, row.operation_id, binding.stageBudget) }
return { row, receipt, result: serverProbeReceiptMetadata(receipt, row.operation_id, binding.stageBudget) }
}
const target = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
const input = serverProbeActionSchema.parse(raw), access = deps.authority(context.principal, { signal: context.signal })
@@ -177,7 +179,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
const { executor } = execution(input, binding, row, true, context.signal)
const receipt = await executor.execute(binding.prepared, context.signal)
await preflight(binding, context.signal); await owned(input, binding, context.operationId, true); await preflight(binding, context.signal)
return metadata(receipt, context.operationId, binding.stageBudget)
return serverProbeReceiptMetadata(receipt, context.operationId, binding.stageBudget)
} catch (error) { throw serverProbeSafeMcpError(error, progress) }
},
verify: async (raw, result) => {
@@ -200,19 +202,47 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
AND j.source_server_id=ANY($6::int[]) AND (j.destination_server_id IS NULL OR j.destination_server_id=ANY($6::int[])) LIMIT 2`,
[request.receiptId, request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION, initial.scopeServerIds])
await fresh()
if (rows.length !== 1) return denied()
const row = rows[0], serverIds = [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
const loadRetired = deps.retiredSource
const readRetired = rows.length === 0 && loadRetired ? () => loadRetired({
receiptId: request.receiptId, operationId: request.operationId, connectionId: context.principal.connectionId,
ownerId: initial.ownerId, scopeServerIds: initial.scopeServerIds,
}) : undefined
const retired = readRetired ? await readRetired() : undefined
if (readRetired) await fresh()
if (rows.length !== 1 && !retired) return denied()
const row = rows[0]
const serverIds = retired ? retired.source.serverIds : [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
const fence = async () => { const current = await fresh(); if (serverIds.some(id => !current.scopeServerIds.includes(id))) return denied(); await access.assertProtections(serverIds, current); await fresh() }
await fence()
return { initial, serverIds, fence }
const retiredFence = retired && readRetired ? async () => {
await fence()
const current = await readRetired()
await fence()
if (!current || fingerprint(current) !== fingerprint(retired)) return denied()
} : undefined
return { initial, serverIds, fence, retired, retiredFence }
}
const receiptTarget = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence } = await historicalAccess(request, context)
await fence(); return { scope: "servers", serverIds }
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence, retiredFence } = await historicalAccess(request, context)
await (retiredFence ?? fence)(); return { scope: "servers", serverIds }
}
const readPage = async (raw: Record<string, unknown>, context: ActionContext) => {
try {
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence } = await historicalAccess(request, context)
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence, retired, retiredFence } = await historicalAccess(request, context)
if (retired && retiredFence) {
const source = retired.source
const originalContext: ServerProbeJournalContext = {
identity: { kind: "mcp", ownerId: source.ownerId, executionId: source.executionId, connectionId: source.connectionId, credentialId: source.originalCredentialId,
issuedAt: source.issuedAt, expiresAt: Date.parse(source.authorityExpiresAt), operationName: SERVER_PROBE_OPERATION },
sourceServerId: source.sourceServerId, ...(source.destinationServerId !== undefined ? { destinationServerId: source.destinationServerId } : {}), tool: source.tool,
}
await retiredFence()
const reader = deps.history({ assertReadAuthority: retiredFence }, { signal: context.signal })
const receipt = await reader.read({ context: originalContext, receiptId: source.receiptId, bindingProof: source.bindingProof })
const page = serverProbeReceiptPage(receipt, request.page)
await retiredFence()
return page
}
await fence()
const rows = await deps.query(`SELECT p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN mcp_connections c ON c.id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2 AND c.owner_id=$3 AND p.operation_id=$4 AND p.operation_version=1 AND octet_length(p.evidence::text)<=2097152 LIMIT 2`, [request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION])
await fence(); if (rows.length !== 1) return denied()
@@ -171,6 +171,16 @@ export async function acceptServerProbeAllToolsSql() {
assert.equal(done.result.confirmedWrites, 0); assert.equal(done.result.completedServiceSteps, 1); assert.equal(done.result.verificationScope, "acknowledgement")
const receipt = done.result.diagnosticReceipt as { receiptId: string; stageBudget: number; attemptedStages: number; completedStages: number; totalPages: number; sha256: string }
assert(receipt); assert.equal(receipt.attemptedStages, receipt.completedStages); assert(receipt.attemptedStages > 0 && receipt.attemptedStages <= receipt.stageBudget)
let captured = false
for (const until = Date.now()+2000; Date.now()<until;) {
const proof = (await pool.query(`SELECT f.source->>'terminalAt'=to_char(GREATEST(o.finished_at,p.expires_at) AT TIME ZONE 'UTC',
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS terminal_matches FROM server_probe_terminal_proofs f
JOIN mcp_operations o ON o.id=f.operation_id JOIN mcp_plans p ON p.id=o.plan_id
WHERE f.operation_id=$1 AND f.receipt_id=$2`, [done.id,receipt.receiptId])).rows[0]
if (proof) { assert.equal(proof.terminal_matches,true); captured = true; break }
await delay(20)
}
assert.equal(captured,true,"Owned successful diagnostic did not publish its terminal proof")
const step = (await pool.query("SELECT state,result FROM mcp_operation_steps WHERE operation_id=$1", [done.id])).rows
assert.equal(step.length, 1); assert.equal(step[0].state, "acknowledged")
for (const field of ["output", "data", "contentB64", "bindingCiphertext"]) assert(!Object.hasOwn(step[0].result, field))
@@ -201,6 +211,49 @@ export async function acceptServerProbeAllToolsSql() {
cases.push(tool + ": actual public plan/approval/apply, receipt metadata, full stage ledger and checksum pages without redispatch")
}
// Force a genuine capture SQL failure after the real diagnostic has
// committed. A sequence records the rejected attempt despite rollback.
const captureFault = await fixture()
const faultPlan = await prepare(captureFault, { tool: "route", target: "127.0.0.1" })
await pool.query("CREATE SEQUENCE public.owned_probe_capture_failure")
try {
await pool.query(`CREATE FUNCTION public.owned_probe_capture_failure() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NEW.source->>'connectionId'=TG_ARGV[0] THEN
PERFORM nextval('public.owned_probe_capture_failure');
RAISE EXCEPTION 'owned terminal proof insertion refused';
END IF;
RETURN NEW;
END $$`)
try {
assert.match(captureFault.principal.connectionId,/^[a-f0-9-]{36}$/)
await pool.query(`CREATE TRIGGER owned_probe_capture_failure BEFORE INSERT ON public.server_probe_terminal_proofs
FOR EACH ROW EXECUTE FUNCTION public.owned_probe_capture_failure('${captureFault.principal.connectionId}')`)
try {
const beforeFault = requests+dnsUdp+dnsTcp, faultKey=randomUUID()
const applied = await applyPlan(captureFault.principal,faultPlan,faultKey), done=await terminal(applied.id)
let attempted = false
for (const until=Date.now()+2500;Date.now()<until;) {
attempted = (await pool.query("SELECT is_called FROM public.owned_probe_capture_failure")).rows[0].is_called
if (attempted) break
await delay(20)
}
assert.equal(attempted,true,"Owned post-COMMIT capture insert must have reached the injected SQL failure")
assert.equal(done.state,"applied"); assert.equal(done.error_code,null)
assert.equal((await pool.query("SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1",[applied.id])).rows.length,0)
const originalReceipt = done.result.diagnosticReceipt as { receiptId:string; sha256:string }
const page = await invokeMcpTool(captureFault.principal,"mm_server_probe_receipt_get",
{ operationId:applied.id,receiptId:originalReceipt.receiptId,page:0 }) as {sha256:string}
assert.equal(page.sha256,originalReceipt.sha256)
assert.equal((await applyPlan(captureFault.principal,faultPlan,faultKey)).id,applied.id)
assert.equal((await terminal(applied.id)).state,"applied")
assert.equal(requests+dnsUdp+dnsTcp,beforeFault+1)
assert.equal((await pool.query("SELECT last_value::int AS attempts FROM public.owned_probe_capture_failure")).rows[0].attempts,1)
} finally { await pool.query("DROP TRIGGER owned_probe_capture_failure ON public.server_probe_terminal_proofs") }
} finally { await pool.query("DROP FUNCTION public.owned_probe_capture_failure()") }
} finally { await pool.query("DROP SEQUENCE public.owned_probe_capture_failure") }
cases.push("terminal proof SQL refusal: acknowledged result remains applied and readable, exactly one loopback measurement and no retry")
// Pause only the one runner's pending->running SQL BEFORE sending it. This
// leaves a real pending approved source, allowing genuine durable cancel.
// A server-table lock after 'running' would have no intent step yet and the
@@ -209,8 +262,9 @@ export async function acceptServerProbeAllToolsSql() {
const entered = gate(), resume = gate(); releasePending = resume.resolve
let heldOperation = "", held = false
pool.query = (async function (...args: unknown[]) {
if (!held && typeof args[0] === "string" && args[0].startsWith("UPDATE mcp_operations SET state='running'")) {
held = true; heldOperation = String((args[1] as unknown[])[0]); entered.resolve(); await resume.promise
const query = args[0] as { text?: string; values?: unknown[] } | undefined
if (!held && query?.text?.startsWith("/* mcp-operation-root:claim */") && typeof query.values?.[0] === "string") {
held = true; heldOperation = query.values[0]; entered.resolve(); await resume.promise
}
return Reflect.apply(originalQuery, pool, args)
}) as typeof pool.query
@@ -2,6 +2,7 @@
import { createMcpServerProbeAuthority, createMcpServerProbeReadAuthority } from "../../services/server-probe-authority.js"
import { createProductionServerProbeExecutor } from "../../services/server-probe-execution.js"
import { createServerProbeStorageHistoryReader } from "../../services/server-probe-storage-history.js"
import { findRetiredServerProbeHistory } from "../../services/server-probe-retired-history.js"
import { createSqlServerProbeSourceReaders } from "../../services/server-probe-storage-source.js"
import { createProbeStorageHostIo } from "../../services/server-probe-storage-host-io.js"
import { assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js"
@@ -28,6 +29,7 @@ export function createStorageServerProbeMcpAdapter(limits: ProbeStorageLimits =
},
}
const adapter = createServerProbeMcpAdapter({ query: io.rows, journal: io.journal,
retiredSource: lookup => findRetiredServerProbeHistory(io.rows, lookup),
authority: (principal, options) => createMcpServerProbeAuthority(principal, options, authorityDependencies),
historicalAuthority: (principal, options) => createMcpServerProbeReadAuthority(principal, options, authorityDependencies),
history: (dependencies, execution) => createServerProbeStorageHistoryReader({ assertReadAuthority: dependencies.assertReadAuthority,
+20 -3
View File
@@ -1,10 +1,13 @@
import { randomUUID } from "node:crypto"
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
import { pool } from "../../db/index.js"
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
import { recoverMcpOperations, verifyStep } from "./runtime.js"
import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js"
import { audit, getMcpSettings } from "./store.js"
import { accessGeneration, decryptSecret, redact } from "./security.js"
import { tickRoutingProfiles } from "./routing-profiles.js"
@@ -48,11 +51,22 @@ export async function detectMcpDrift() {
}
}
const mcpRetentionBatchSize = 1000
async function retainMcpData() {
const settings = await getMcpSettings()
await pool.query("DELETE FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')", [settings.requestsRetentionDays])
await pool.query("DELETE FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)", [settings.auditRetentionDays])
await pool.query("DELETE FROM mcp_artifacts WHERE expires_at<now()")
await pool.query(`WITH expired AS (
SELECT id FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')
ORDER BY created_at,id LIMIT $2 FOR UPDATE
) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [settings.requestsRetentionDays, mcpRetentionBatchSize])
await pool.query(`WITH expired AS (
SELECT id FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)
ORDER BY created_at,id LIMIT $2 FOR UPDATE
) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [settings.auditRetentionDays, mcpRetentionBatchSize])
await pool.query(`WITH expired AS (
SELECT id FROM mcp_artifacts WHERE expires_at<now()
ORDER BY expires_at,id LIMIT $1 FOR UPDATE
) DELETE FROM mcp_artifacts USING expired WHERE mcp_artifacts.id=expired.id`, [mcpRetentionBatchSize])
await pool.query("DELETE FROM mcp_oauth_requests WHERE expires_at<now()-interval '1 day'")
await pool.query("DELETE FROM mcp_budget_buckets WHERE bucket<now()-interval '2 days'")
await routingReservationTransaction(db => retainManualInverseChargesWithinOwner(db, settings.requestsRetentionDays))
@@ -79,8 +93,11 @@ export async function tickMcpWorkers() {
await retainMcpData()
await retainServerProbeStorage()
await compactCompletedServerPollStorage()
await retireCompletedServerPollOperations()
await collectExpiredServerPollStorage()
await retireOrdinaryMcpOperations()
if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() }
await maintainServerProbeOperationRetirement()
} finally {
if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true }
lease.release(unsafe)