fix(mcp): обновить проверенный снимок маршрутизации и хранения операций
This commit is contained in:
1 parent
f1af1a85e5
commit
5ce56b5168
117 files changed
+9960
-182
No files matched your search
@@ -14,6 +14,7 @@ import { logFiltersSchema, logSettingsPatchSchema } from "@mmapp/contracts"
|
||||
import { appUserCreateSchema, appUserUpdateSchema, appUserRoleSchema, sectionPermSchema, serverPermSchema, userBindingCreateSchema } from "@mmapp/contracts/users"
|
||||
import type { McpOperationDescriptor } from "@mmapp/contracts/mcp"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { assertOperationReplayScope, findOperationReplayTombstone, operationHistoryExpired } from "./operation-replay-tombstones.js"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { getServerReadById } from "../servers/service/servers-service.js"
|
||||
import { listUsers, getUserById, listInterfaceCatalog, getInterfaceCatalogState } from "../users/service/users-service.js"
|
||||
@@ -473,7 +474,12 @@ add("changes.cancel", "mm_change_cancel", "Отменить свой незап
|
||||
add("operations.read", "mm_operation_get", "Результат своей операции", "mcp", null, "read", "connection", z.object({ operationId: z.string().uuid() }).strict(), {
|
||||
read: async (input, context) => {
|
||||
const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND connection_id=$2", [input.operationId, context.connection.id])
|
||||
if (!operation) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
if (!operation) {
|
||||
const tombstone = await findOperationReplayTombstone(pool, context.connection.id, String(input.operationId))
|
||||
if (!tombstone) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
assertOperationReplayScope(tombstone, context.connection)
|
||||
operationHistoryExpired()
|
||||
}
|
||||
await assertPlanReadScope(await (await import("./runtime.js")).loadPlan(operation.plan_id), context)
|
||||
const { rows: steps } = await pool.query("SELECT * FROM mcp_operation_steps WHERE operation_id=$1 ORDER BY step_index", [input.operationId])
|
||||
return { ...operation, steps }
|
||||
|
||||
@@ -115,3 +115,9 @@ export const createDiagnosticCancellationSourceForTests = sourceLoader
|
||||
/** Historical authentication only; confers no execution/cancellation grant. */
|
||||
export const readServerProbeExecutionSource = readSourceLoader(productionCrypto)
|
||||
export const createReadServerProbeExecutionSourceForTests = readSourceLoader
|
||||
/** Exact previously locked source projection for private history conversion. */
|
||||
export function authenticateServerProbeExecutionSourceRow(row: Record<string, unknown>, operationId: string,
|
||||
owner: Readonly<DiagnosticCancellationOwner>) {
|
||||
checkLookup(operationId, owner)
|
||||
return validateSourceRow(row, operationId, owner, productionCrypto)
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { test } from "node:test"
|
||||
import type { McpAction, ActionContext } from "./action-types.js"
|
||||
import { registerGlobalReconciliationActionsForTests } from "./global-reconciliation-actions.js"
|
||||
|
||||
type Register = Parameters<typeof registerGlobalReconciliationActionsForTests>[0]
|
||||
type Dependencies = Parameters<typeof registerGlobalReconciliationActionsForTests>[1]
|
||||
|
||||
function fixture() {
|
||||
const connectionId = randomUUID(), operationId = randomUUID(), ownerId = "current-retired-owner"
|
||||
const context = { principal: { connectionId }, connection: { id: connectionId, owner_id: ownerId, owner_permissions: ["mm:servers:write"],
|
||||
policy: { permissions: ["mm:servers:write"], scope: { allowGlobal: true, operationIds: ["servers.poll", "operations.global.reconcile"], serverIds: [1, 2] } } } } as ActionContext
|
||||
let action!: Pick<McpAction, "target" | "read">
|
||||
let history: { affectedServerIds: number[] } | undefined = { affectedServerIds: [1, 2] }
|
||||
let revoke = false, retiredCalls = 0, live = false
|
||||
const deps = {
|
||||
connection: async () => {
|
||||
if (revoke) throw Error("revoked-current-grant")
|
||||
return structuredClone(context.connection)
|
||||
},
|
||||
query: async (sql: string) => {
|
||||
if (sql.startsWith("SELECT id FROM servers")) return [{ id: 1 }]
|
||||
if (live) return [{ id: operationId, state: "applied", operation_id: "servers.poll", evidence: { sealed: "live" } }]
|
||||
return []
|
||||
},
|
||||
operation: (id: string) => ({ id, scope: "global", requiredPermissions: ["mm:servers:write"] }),
|
||||
service: () => ({ reconcile: async () => ({ matchesDesiredState: true, matchesOriginalState: false }) }),
|
||||
decrypt: async () => JSON.stringify({ target: { scope: "global" }, objects: [{ serverId: 1 }], steps: [{ kind: "service", service: "servers.poll", input: {} }] }),
|
||||
retired: async (identity, authorize) => {
|
||||
retiredCalls++
|
||||
assert.deepEqual(identity, { operationId, connectionId, ownerId })
|
||||
await authorize()
|
||||
return history
|
||||
},
|
||||
} satisfies Omit<Dependencies, "operation" | "service"> & Partial<Record<"operation" | "service", unknown>>
|
||||
registerGlobalReconciliationActionsForTests(((...args: Parameters<Register>) => { action = args[8] }) as Register, deps as unknown as Dependencies)
|
||||
return { context, operationId, history: (value: typeof history) => { history = value }, live: () => { live = true },
|
||||
revoke: () => { revoke = true }, calls: () => retiredCalls, read: () => action.read!({ operationId }, context),
|
||||
target: () => action.target({ operationId }, context) }
|
||||
}
|
||||
|
||||
test("global action reconciles retired poll only under current ownership and returns no archived material", async () => {
|
||||
const f = fixture()
|
||||
assert.deepEqual(await f.target(), { scope: "global" })
|
||||
const result = await f.read() as { observedAt: string }
|
||||
const { observedAt, ...rest } = result
|
||||
assert(Number.isFinite(Date.parse(observedAt)))
|
||||
assert.deepEqual(rest, { operationId: f.operationId, state: "applied", automaticRetry: false,
|
||||
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }] })
|
||||
assert.equal(f.calls(), 2)
|
||||
})
|
||||
|
||||
test("global action refuses missing or inaccessible retired history", async () => {
|
||||
const f = fixture()
|
||||
f.history(undefined)
|
||||
await assert.rejects(f.read(), { code: "NOT_FOUND" })
|
||||
f.history({ affectedServerIds: [1, 2] })
|
||||
f.context.connection.policy.scope.serverIds = [1]
|
||||
await assert.rejects(f.read(), { code: "SERVER_DENIED" })
|
||||
f.context.connection.policy.scope.serverIds = [1, 2]
|
||||
f.context.connection.policy.scope.operationIds = ["operations.global.reconcile"]
|
||||
await assert.rejects(f.read(), { code: "PERMISSION_DENIED" })
|
||||
f.revoke()
|
||||
await assert.rejects(f.read(), /revoked-current-grant/)
|
||||
})
|
||||
|
||||
test("global action uses the live reconciliation path while original source exists", async () => {
|
||||
const f = fixture()
|
||||
f.live()
|
||||
await f.read()
|
||||
assert.equal(f.calls(), 0)
|
||||
})
|
||||
@@ -1,5 +1,6 @@
|
||||
import { z } from "zod"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { readRetiredServerPollHistory } from "../../services/server-poll-history-reader.js"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { effectiveMcpConnection } from "./store.js"
|
||||
import { decryptSecret, fingerprint, McpError } from "./security.js"
|
||||
@@ -7,40 +8,84 @@ import { findMcpOperation } from "./registry.js"
|
||||
import { findMcpService } from "./service-registry.js"
|
||||
import type { add } from "./actions.js"
|
||||
import type { ActionContext, ChangeEvidence } from "./action-types.js"
|
||||
import type { ConnectionRow } from "./store.js"
|
||||
|
||||
async function authorizedSource(operationId: string, context: ActionContext) {
|
||||
const production = {
|
||||
connection: effectiveMcpConnection,
|
||||
query: async (sql: string, values: unknown[] = []) => (await pool.query(sql, values)).rows,
|
||||
operation: findMcpOperation,
|
||||
service: findMcpService,
|
||||
decrypt: decryptSecret,
|
||||
retired: readRetiredServerPollHistory,
|
||||
}
|
||||
type Dependencies = typeof production
|
||||
const notFound = () => new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
|
||||
async function currentAccess(context: ActionContext, deps: Dependencies) {
|
||||
context.signal?.throwIfAborted()
|
||||
const current = await effectiveMcpConnection(context.principal)
|
||||
const current = await deps.connection(context.principal)
|
||||
if (!current.policy.scope.allowGlobal || fingerprint(current.policy) !== fingerprint(context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(context.connection.owner_permissions)) throw new McpError("PERMISSION_DENIED", "Глобальный допуск сверки изменился", 403)
|
||||
const ids = (await pool.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).rows.map(row => Number(row.id))
|
||||
const ids = (await deps.query("SELECT id FROM servers ORDER BY id LIMIT 1001")).map(row => Number(row.id))
|
||||
if (ids.length > 1000) throw new McpError("LIMIT_EXCEEDED", "Каталог глобальной сверки превышает лимит", 409)
|
||||
if (ids.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Глобальная сверка требует доступа ко всему каталогу серверов", 403)
|
||||
const row = (await pool.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id])).rows[0]
|
||||
if (!row) throw new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
const original = findMcpOperation(row.operation_id)
|
||||
return current
|
||||
}
|
||||
|
||||
function requireOriginal(actionId: string, current: ConnectionRow, deps: Dependencies) {
|
||||
const original = deps.operation(actionId)
|
||||
if (!original || original.scope !== "global" || !current.policy.scope.operationIds.includes(original.id) || original.requiredPermissions.some(permission => !hasPermission(current.policy.permissions, permission) || !hasPermission(current.owner_permissions, permission))) throw new McpError("PERMISSION_DENIED", "Утрачено право на исходную глобальную операцию", 403)
|
||||
const evidence = JSON.parse(await decryptSecret(row.evidence.sealed)) as ChangeEvidence
|
||||
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !findMcpService(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
|
||||
const declared = evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId])
|
||||
}
|
||||
|
||||
function requireServerScope(declared: number[], current: ConnectionRow) {
|
||||
if (declared.some(id => !current.policy.scope.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Зависимость операции вне текущего глобального допуска", 403)
|
||||
}
|
||||
|
||||
async function authorizedSource(operationId: string, context: ActionContext, deps: Dependencies) {
|
||||
const current = await currentAccess(context, deps)
|
||||
const row = (await deps.query("SELECT o.id,o.state,p.operation_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2", [operationId, current.id]))[0]
|
||||
if (!row) {
|
||||
const authorize = async () => {
|
||||
const fresh = await currentAccess(context, deps)
|
||||
if (fresh.id !== current.id || fresh.owner_id !== current.owner_id) throw notFound()
|
||||
}
|
||||
const archived = await deps.retired({ operationId, connectionId: current.id, ownerId: current.owner_id }, authorize,
|
||||
{ deadlineAt: Date.now() + 10000, signal: context.signal })
|
||||
context.signal?.throwIfAborted()
|
||||
if (!archived) throw notFound()
|
||||
requireOriginal("servers.poll", current, deps)
|
||||
requireServerScope(archived.affectedServerIds, current)
|
||||
return { kind: "retired-poll" as const }
|
||||
}
|
||||
requireOriginal(row.operation_id, current, deps)
|
||||
const evidence = JSON.parse(await deps.decrypt(row.evidence.sealed)) as ChangeEvidence
|
||||
if (evidence.target.scope !== "global" || !evidence.steps.length || evidence.steps.some(step => step.kind !== "service" || !deps.service(step.service)?.reconcile)) throw new McpError("UNSUPPORTED_CAPABILITY", "Для глобальной операции нет безопасной read-only сверки", 409)
|
||||
requireServerScope(evidence.objects.flatMap(object => object.serverId === undefined ? [] : [object.serverId]), current)
|
||||
if (["pending", "running"].includes(row.state)) throw new McpError("STATE_CONFLICT", "Операция ещё выполняется", 409)
|
||||
context.signal?.throwIfAborted()
|
||||
return { row, evidence }
|
||||
return { kind: "live" as const, row, evidence }
|
||||
}
|
||||
export function registerGlobalReconciliationActions(register: typeof add) {
|
||||
|
||||
function registerActions(register: typeof add, deps: Dependencies) {
|
||||
register("operations.global.reconcile", "mm_global_operation_reconcile", "Сверить глобальную сервисную операцию без повторной записи", "mcp", null, "read", "global", z.object({ operationId: z.string().uuid() }).strict(), {
|
||||
target: async (input, context) => { await authorizedSource(String(input.operationId), context); return { scope: "global" } },
|
||||
target: async (input, context) => { await authorizedSource(String(input.operationId), context, deps); return { scope: "global" } },
|
||||
read: async (input, context) => {
|
||||
const { row, evidence } = await authorizedSource(String(input.operationId), context)
|
||||
const authorize = async () => { await authorizedSource(String(input.operationId), context) }
|
||||
const operationId = String(input.operationId)
|
||||
const source = await authorizedSource(operationId, context, deps)
|
||||
if (source.kind === "retired-poll") return { operationId, state: "applied", observedAt: new Date().toISOString(),
|
||||
results: [{ stepIndex: 0, service: "servers.poll", matchesDesiredState: true, matchesOriginalState: false }], automaticRetry: false }
|
||||
const { row, evidence } = source
|
||||
const authorize = async () => { if ((await authorizedSource(operationId, context, deps)).kind !== "live") throw notFound() }
|
||||
const results = []
|
||||
for (const [index, step] of evidence.steps.entries()) {
|
||||
if (step.kind !== "service") throw new McpError("UNSUPPORTED_CAPABILITY", "Нативная глобальная сверка недоступна", 409)
|
||||
await authorize()
|
||||
results.push({ stepIndex: index, service: step.service, ...await findMcpService(step.service)!.reconcile!(step.input, authorize) })
|
||||
results.push({ stepIndex: index, service: step.service, ...await deps.service(step.service)!.reconcile!(step.input, authorize) })
|
||||
}
|
||||
await authorize()
|
||||
return { operationId: row.id, state: row.state, observedAt: new Date().toISOString(), results, automaticRetry: false }
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
export const registerGlobalReconciliationActions = (register: typeof add) => registerActions(register, production)
|
||||
export const registerGlobalReconciliationActionsForTests = registerActions
|
||||
@@ -5,6 +5,8 @@ await prepareOwnedBandwidthState()
|
||||
const { pool, closePool } = await import("../../db/index.js")
|
||||
try {
|
||||
const { applySqlMigrations } = await import("../../db/migrate.js"); await applySqlMigrations(pool)
|
||||
const { acceptOperationReplayTombstones } = await import("./operation-replay-tombstones.sql-helper.js")
|
||||
await acceptOperationReplayTombstones()
|
||||
const { acceptOwnedMcpOperationRoot } = await import("./operation-execution-origin.sql-helper.js")
|
||||
console.log("Owned actual MCP operation root:", JSON.stringify(await acceptOwnedMcpOperationRoot()))
|
||||
} finally { await closePool() }
|
||||
@@ -0,0 +1,74 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { assertOwnedBandwidthSql } from "../../services/bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
export async function acceptOperationReplayTombstones() {
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN, "true")
|
||||
const { pool } = await import("../../db/index.js")
|
||||
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
||||
const { createMcpConnection, authenticateMcpToken } = await import("./store.js")
|
||||
const { applyPlan, invokeMcpTool } = await import("./runtime.js")
|
||||
const { operationIdempotencyDigest } = await import("./operation-replay-tombstones.js")
|
||||
const { encryptSecret, McpError } = await import("./security.js")
|
||||
const { default: Fastify } = await import("fastify")
|
||||
const { default: authPlugin } = await import("../../plugins/auth.js")
|
||||
const { createMcpRoutes } = await import("../../routes/mcp.js")
|
||||
const { env } = await import("../../config.js")
|
||||
const { SignJWT } = await import("jose")
|
||||
const makeConnection = async () => {
|
||||
const actor = { id: randomUUID(), email: "[email protected]", name: "Owned replay fixture", apps: ["mm"],
|
||||
permissions: ["mm:users:write", "mm:mcp:read"], issuedAt: Date.now()-1000, expiresAt: Date.now()+300000 }
|
||||
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: actor.name, mode: "approval",
|
||||
permissions: actor.permissions, expiresAt: new Date(actor.expiresAt).toISOString(),
|
||||
scope: { serverIds: [], operationIds: ["users.create", "operations.read", "changes.apply"], allowGlobal: true },
|
||||
limits: { callsPerMinute: 100, mutationsPerHour: 10 } }), actor)
|
||||
return { actor, principal: await authenticateMcpToken(issued.token) }
|
||||
}
|
||||
const app = Fastify()
|
||||
const ids: string[] = []
|
||||
const matches = (code: string, status: number) => (error: unknown) =>
|
||||
error instanceof McpError && error.code === code && error.statusCode === status
|
||||
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
||||
try {
|
||||
const owner = await makeConnection(), foreign = await makeConnection()
|
||||
// A real completed plan has already initialized the owned encryption state.
|
||||
// The fixture seeds its tombstone directly, so exercise the same initializer.
|
||||
await encryptSecret("owned replay fixture state")
|
||||
const connectionId = owner.principal.connectionId
|
||||
ids.push(connectionId, foreign.principal.connectionId)
|
||||
const operationId = randomUUID(), planId = randomUUID(), key = "owned-replay-" + randomUUID()
|
||||
await pool.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,'users.create','{}',true)`, [operationId, planId, connectionId, operationIdempotencyDigest(connectionId,key)])
|
||||
await assert.rejects(applyPlan(owner.principal, planId, key), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(owner.principal, planId, randomUUID()), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(owner.principal, randomUUID(), key), matches("STATE_CONFLICT",409))
|
||||
await assert.rejects(applyPlan(foreign.principal, planId, key), matches("NOT_FOUND",404))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_change_apply",{ planId,idempotencyKey:key }), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(foreign.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
|
||||
await app.register(authPlugin)
|
||||
await app.register(createMcpRoutes(async () => {}))
|
||||
for (const subject of [owner, foreign]) {
|
||||
const token = await new SignJWT({ sub: subject.actor.id, email: subject.actor.email, name: subject.actor.name,
|
||||
apps: subject.actor.apps, permissions: subject.actor.permissions })
|
||||
.setProtectedHeader({ alg:"HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("4m")
|
||||
.sign(new TextEncoder().encode(env.jwtSecret))
|
||||
const reply = await app.inject({ method: "POST", url: "/mcp/plans/"+planId+"/apply", headers: { authorization:"Bearer "+token } })
|
||||
assert.equal(reply.statusCode, subject === owner ? 410 : 404)
|
||||
assert.equal(reply.json().code, subject === owner ? "OPERATION_HISTORY_EXPIRED" : "NOT_FOUND")
|
||||
}
|
||||
await pool.query("UPDATE public.mcp_connections SET policy=jsonb_set(policy,'{scope,allowGlobal}','false') WHERE id=$1", [connectionId])
|
||||
await assert.rejects(applyPlan(owner.principal, planId, key), matches("NOT_FOUND",404))
|
||||
await assert.rejects(invokeMcpTool(owner.principal,"mm_operation_get",{ operationId }), matches("NOT_FOUND",404))
|
||||
const { rows:[proof] } = await pool.query(`SELECT
|
||||
NOT EXISTS(SELECT 1 FROM public.mcp_operations WHERE connection_id=ANY($1::uuid[])) AS no_operation,
|
||||
NOT EXISTS(SELECT 1 FROM public.mcp_budget_buckets WHERE connection_id=ANY($1::uuid[]) AND kind='mutations') AS no_write_budget`, [ids])
|
||||
assert.deepEqual(proof,{no_operation:true,no_write_budget:true})
|
||||
console.log("Owned operation tombstones: scoped MCP and Web 410, replay conflict and no dispatch")
|
||||
} finally {
|
||||
await app.close()
|
||||
await pool.query("DELETE FROM public.mcp_operation_replay_tombstones WHERE connection_id=ANY($1::uuid[])", [ids])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
import { createHash } from "node:crypto"
|
||||
import type { PoolClient } from "pg"
|
||||
import { hasPermission } from "../../lib/permissions.js"
|
||||
import { listActions } from "./actions.js"
|
||||
import { McpError } from "./security.js"
|
||||
import { effectiveMcpConnection, type ConnectionRow, type McpPrincipal } from "./store.js"
|
||||
|
||||
type Query = Pick<PoolClient, "query">
|
||||
export interface OperationReplayTombstone {
|
||||
operation_id: string
|
||||
plan_id: string
|
||||
connection_id: string
|
||||
action_id: string
|
||||
target_server_ids: string[]
|
||||
target_global: boolean
|
||||
}
|
||||
|
||||
// Idempotency tokens are caller input and already stored in plaintext on live
|
||||
// operations. This domain-separated digest avoids copying that plaintext and
|
||||
// survives DB restore and access-key rotation. It is not a keyed secret proof.
|
||||
export function operationIdempotencyDigest(connectionId: string, key: string): string {
|
||||
return createHash("sha256").update("mcp-operation-replay-v1\0").update(connectionId).update("\0").update(key).digest("hex")
|
||||
}
|
||||
|
||||
export function operationHistoryExpired(): never {
|
||||
throw new McpError("OPERATION_HISTORY_EXPIRED", "Срок хранения завершённой операции истёк. Повторное применение запрещено.", 410)
|
||||
}
|
||||
|
||||
export function assertOperationReplayScope(row: OperationReplayTombstone, connection: ConnectionRow): void {
|
||||
const scope = connection.policy.scope
|
||||
if (row.connection_id !== connection.id || !scope.operationIds.includes(row.action_id)) throw operationReplayNotFound()
|
||||
const action = listActions().find(candidate => candidate.operation.id === row.action_id)
|
||||
if (!action || action.operation.requiredPermissions.some(permission =>
|
||||
!hasPermission(connection.policy.permissions, permission) || !hasPermission(connection.owner_permissions, permission))) throw operationReplayNotFound()
|
||||
if (row.target_global && !scope.allowGlobal || row.target_server_ids.some(id => !scope.serverIds.some(server => String(server) === id))) throw operationReplayNotFound()
|
||||
}
|
||||
|
||||
function operationReplayNotFound(): McpError {
|
||||
return new McpError("NOT_FOUND", "Операция не найдена", 404)
|
||||
}
|
||||
|
||||
const projection = "t.operation_id,t.plan_id,t.connection_id,t.action_id,t.target_server_ids::text[] AS target_server_ids,t.target_global"
|
||||
|
||||
export async function findOperationReplayTombstone(db: Query, connectionId: string, operationId: string) {
|
||||
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
WHERE t.connection_id=$1 AND t.operation_id=$2`, [connectionId, operationId])).rows[0]
|
||||
}
|
||||
|
||||
export async function findOwnerPlanReplayTombstone(db: Query, ownerId: string, planId: string) {
|
||||
return (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
JOIN mcp_connections c ON c.id=t.connection_id WHERE c.owner_id=$1 AND t.plan_id=$2`, [ownerId, planId])).rows[0]
|
||||
}
|
||||
|
||||
export async function refuseExpiredOperationReplay(db: Query, principal: McpPrincipal, connectionId: string, planId: string, key: string): Promise<void> {
|
||||
const rows = (await db.query<OperationReplayTombstone>(`SELECT ${projection} FROM mcp_operation_replay_tombstones t
|
||||
WHERE t.connection_id=$1 AND (t.plan_id=$2 OR t.idempotency_digest=$3)`,
|
||||
[connectionId, planId, operationIdempotencyDigest(connectionId, key)])).rows
|
||||
if (!rows.length) return
|
||||
const conflict = rows.find(row => row.plan_id !== planId.toLowerCase())
|
||||
const current = await effectiveMcpConnection(principal, db)
|
||||
assertOperationReplayScope(conflict ?? rows[0], current)
|
||||
if (conflict) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409)
|
||||
operationHistoryExpired()
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import { assertOwnedBandwidthSql,prepareOwnedBandwidthState } from "../../services/bandwidth-admission-process.sql-helper.js"
|
||||
|
||||
assertOwnedBandwidthSql()
|
||||
assert.equal(process.env.MMAPP_OWN_MCP_OPERATION_ORIGIN,"true")
|
||||
await prepareOwnedBandwidthState()
|
||||
const {pool,closePool}=await import("../../db/index.js")
|
||||
const {applySqlMigrations}=await import("../../db/migrate.js")
|
||||
const {mcpConnectionCreateSchema}=await import("@mmapp/contracts/mcp")
|
||||
const {createMcpConnection,authenticateMcpToken}=await import("./store.js")
|
||||
const {invokeMcpTool,approvePlan,applyPlan}=await import("./runtime.js")
|
||||
const {OrdinaryRetirementCommitUnknown,retireOrdinaryMcpOperations}=await import("./ordinary-operation-retirement.js")
|
||||
const {tickMcpWorkers}=await import("./workers.js")
|
||||
const {pauseSchedulerJobsAndWait,resumeSchedulerJobs}=await import("../../services/scheduler-running.js")
|
||||
const {Client}=await import("pg")
|
||||
const {McpError}=await import("./security.js")
|
||||
const errorIs=(code:string,status:number)=>(error:unknown)=>error instanceof McpError&&error.code===code&&error.statusCode===status
|
||||
|
||||
try {
|
||||
await applySqlMigrations(pool)
|
||||
await pauseSchedulerJobsAndWait()
|
||||
await pool.query("UPDATE public.mcp_settings SET enabled=true WHERE id=1")
|
||||
const serverId=Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned logs retention','127.0.0.1',false) RETURNING id")).rows[0].id)
|
||||
async function access(){
|
||||
const actor={id:randomUUID(),email:"[email protected]",name:"Owned settings retention",apps:["mm"],
|
||||
permissions:["mm:settings:admin","mm:mcp:approve","mm:traffic:write","mm:uptime:write","mm:servers:write","mm:network:write"],
|
||||
issuedAt:Date.now()-1000,expiresAt:Date.now()+300000}
|
||||
const issued=await createMcpConnection(mcpConnectionCreateSchema.parse({name:actor.name,mode:"approval",
|
||||
permissions:actor.permissions,expiresAt:new Date(actor.expiresAt).toISOString(),
|
||||
scope:{serverIds:[serverId],operationIds:["logs.settings.update","traffic.settings.update","uptime.settings.update",
|
||||
"servers.api-ping.settings.update","internet-path.settings.update","operations.read","changes.apply"],allowGlobal:true},
|
||||
limits:{callsPerMinute:100,mutationsPerHour:10}}),actor)
|
||||
return {actor,principal:await authenticateMcpToken(issued.token)}
|
||||
}
|
||||
const {actor,principal}=await access(),foreign=(await access()).principal
|
||||
async function operation(retentionDays:number,ageMs:number,tool="mm_logs_settings_update_plan",
|
||||
settings:Record<string,unknown>={retentionDays}){
|
||||
const plan=await invokeMcpTool(principal,tool,{settings}) as {planId:string;state:string}
|
||||
if(plan.state==="awaiting-approval")await approvePlan(plan.planId,actor,true)
|
||||
const key=randomUUID(),applied=await applyPlan(principal,plan.planId,key)
|
||||
let terminal
|
||||
for(const deadline=Date.now()+15000;Date.now()<deadline;){
|
||||
terminal=(await pool.query("SELECT state,verification,error_code FROM mcp_operations WHERE id=$1",[applied.id])).rows[0]
|
||||
if(terminal&&!["pending","running"].includes(terminal.state))break
|
||||
await delay(50)
|
||||
}
|
||||
assert.deepEqual(terminal,{state:"applied",verification:"passed",error_code:null})
|
||||
// Only the disposable fixture ages rows, after real authenticated apply.
|
||||
// No cutoff override or production encryption/approval bypass.
|
||||
const finished=new Date(Date.now()-ageMs),created=new Date(finished.getTime()-1000)
|
||||
await pool.query("UPDATE mcp_plans SET created_at=$2,approved_at=$2,expires_at=$3 WHERE id=$1",
|
||||
[plan.planId,created,new Date(finished.getTime()+1000)])
|
||||
await pool.query("UPDATE mcp_operations SET created_at=$2,finished_at=$3 WHERE id=$1",[applied.id,created,finished])
|
||||
await pool.query("UPDATE mcp_operation_steps SET created_at=$2,finished_at=$3 WHERE operation_id=$1",[applied.id,created,finished])
|
||||
return {id:applied.id,planId:plan.planId,key}
|
||||
}
|
||||
const age=180*86400000
|
||||
const held=await operation(51,age+180000),spent=await operation(52,age+120000)
|
||||
const due=await operation(53,age+60000),young=await operation(54,age-60000)
|
||||
const boundary=await pool.connect()
|
||||
try{
|
||||
await boundary.query("BEGIN")
|
||||
await boundary.query("UPDATE mcp_operations SET finished_at=transaction_timestamp()-interval '180 days' WHERE id=$1",[young.id])
|
||||
await assert.rejects(boundary.query(`INSERT INTO mcp_ordinary_retirement_claims
|
||||
(operation_id,plan_id,connection_id,source_proof,claim_token) VALUES($1,$2,$3,$4,$5)`,
|
||||
[young.id,young.planId,principal.connectionId,"a".repeat(64),randomUUID()]),/cannot be replaced or reopened/)
|
||||
}finally{await boundary.query("ROLLBACK");boundary.release()}
|
||||
await pool.query("INSERT INTO mcp_drift(id,operation_id,expected,observed) VALUES($1,$2,'{}','{}')",[randomUUID(),held.id])
|
||||
const ids=[held.id,spent.id,due.id,young.id]
|
||||
const projection=async()=>(await pool.query(`SELECT i.id,o.state,p.state AS plan_state,s.state AS step_state,
|
||||
k.operation_id AS checkpoint,t.operation_id AS tombstone,c.claim_token
|
||||
FROM unnest($1::uuid[]) i(id)
|
||||
LEFT JOIN mcp_operations o ON o.id=i.id LEFT JOIN mcp_plans p ON p.id=o.plan_id
|
||||
LEFT JOIN mcp_operation_steps s ON s.operation_id=i.id LEFT JOIN mcp_operation_checkpoints k ON k.operation_id=i.id
|
||||
LEFT JOIN mcp_operation_replay_tombstones t ON t.operation_id=i.id
|
||||
LEFT JOIN mcp_ordinary_retirement_claims c ON c.operation_id=i.id ORDER BY i.id`,[ids])).rows
|
||||
const originalQuery=Client.prototype.query
|
||||
let commits=0
|
||||
try{
|
||||
Client.prototype.query=function(this:InstanceType<typeof Client>,...args:unknown[]){
|
||||
if((args[0] as {text?:string})?.text==="COMMIT"&&++commits===3)
|
||||
return Promise.reject(Error("owned ordinary destructive COMMIT transport loss"))
|
||||
return Reflect.apply(originalQuery,this,args)
|
||||
} as typeof Client.prototype.query
|
||||
await assert.rejects(retireOrdinaryMcpOperations(),
|
||||
(error:unknown)=>error instanceof OrdinaryRetirementCommitUnknown&&error.automaticReplay===false)
|
||||
}finally{Client.prototype.query=originalQuery}
|
||||
assert.equal(commits,3)
|
||||
const before=await projection(),spentBefore=before.find(row=>row.id===spent.id)
|
||||
assert.equal(spentBefore?.state,"applied");assert(spentBefore?.claim_token);assert.equal(spentBefore?.tombstone,null)
|
||||
await tickMcpWorkers()
|
||||
const after=await projection()
|
||||
assert.deepEqual(after.filter(row=>row.id!==due.id),before.filter(row=>row.id!==due.id))
|
||||
const retired=after.find(row=>row.id===due.id)
|
||||
assert(retired?.claim_token);assert.equal(retired?.tombstone,due.id)
|
||||
for(const field of ["state","plan_state","step_state","checkpoint"])assert.equal(retired[field],null)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_plans WHERE id=$1",[due.planId])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT retention_days FROM router_log_settings WHERE id=1")).rows[0].retention_days,54)
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await projection(),after)
|
||||
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(applyPlan(principal,randomUUID(),due.key),errorIs("STATE_CONFLICT",409))
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:due.id}),errorIs("NOT_FOUND",404))
|
||||
const telemetry=[
|
||||
{action:"traffic.settings.update",version:2,record:await operation(55,age+60000,"mm_traffic_settings_update_plan")},
|
||||
{action:"uptime.settings.update",version:2,record:await operation(56,age+60000,"mm_uptime_settings_update_plan",{enabled:false,retentionDays:56})},
|
||||
{action:"servers.api-ping.settings.update",version:2,record:await operation(0,age+60000,"mm_servers_api_ping_settings_update_plan",{intervalSec:123})},
|
||||
{action:"internet-path.settings.update",version:1,record:await operation(57,age+60000,"mm_internet_path_settings_update_plan")},
|
||||
]
|
||||
const telemetryRows=async()=>(await pool.query(`SELECT p.operation_id,p.operation_version,o.id
|
||||
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=ANY($1::uuid[]) ORDER BY p.operation_id`,
|
||||
[telemetry.map(item=>item.record.id)])).rows
|
||||
assert.deepEqual((await telemetryRows()).map(row=>[row.operation_id,row.operation_version]),
|
||||
telemetry.map(item=>[item.action,item.version]).sort((a,b)=>String(a[0]).localeCompare(String(b[0]))))
|
||||
const telemetrySettings=async()=>(await pool.query(`SELECT to_jsonb(t) AS traffic,to_jsonb(u) AS uptime,
|
||||
to_jsonb(a) AS api_ping,to_jsonb(i) AS internet FROM traffic_settings t,uptime_settings u,
|
||||
servers_api_ping_settings a,internet_path_settings i WHERE t.id=1 AND u.id=1 AND a.id=1 AND i.id=1`)).rows[0]
|
||||
const settingsBefore=await telemetrySettings()
|
||||
const traffic=telemetry[0].record
|
||||
const {rows:[originalStep]}=await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1",[traffic.id])
|
||||
await pool.query("UPDATE mcp_operation_steps SET result=jsonb_set(result,'{retentionDays}','999') WHERE operation_id=$1",[traffic.id])
|
||||
await tickMcpWorkers()
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_ordinary_retirement_claims WHERE operation_id=$1",[traffic.id])).rows.length,0)
|
||||
assert.equal((await pool.query("SELECT 1 FROM mcp_operations WHERE id=$1",[traffic.id])).rows.length,1)
|
||||
await pool.query("UPDATE mcp_operation_steps SET result=$2 WHERE operation_id=$1",[traffic.id,originalStep.result])
|
||||
await tickMcpWorkers()
|
||||
await tickMcpWorkers()
|
||||
// The keyset owner may need one tick to wrap after the rejected oldest row.
|
||||
await tickMcpWorkers()
|
||||
assert.deepEqual(await telemetryRows(),[])
|
||||
assert.deepEqual(await telemetrySettings(),settingsBefore)
|
||||
for(const {action,record} of telemetry){
|
||||
const {rows:[proof]}=await pool.query(`SELECT t.action_id,t.target_global,t.target_server_ids::text[] AS server_ids,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_plans WHERE id=$2) AS plan_gone,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_steps WHERE operation_id=$1) AS step_gone,
|
||||
NOT EXISTS(SELECT 1 FROM mcp_operation_checkpoints WHERE operation_id=$1) AS checkpoint_gone
|
||||
FROM mcp_operation_replay_tombstones t WHERE t.operation_id=$1 AND t.plan_id=$2`,[record.id,record.planId])
|
||||
assert.deepEqual(proof,{action_id:action,target_global:true,server_ids:[String(serverId)],plan_gone:true,step_gone:true,checkpoint_gone:true})
|
||||
await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("OPERATION_HISTORY_EXPIRED",410))
|
||||
await assert.rejects(invokeMcpTool(foreign,"mm_operation_get",{operationId:record.id}),errorIs("NOT_FOUND",404))
|
||||
}
|
||||
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{scope,serverIds}','[]') WHERE id=$1",[principal.connectionId])
|
||||
await assert.rejects(applyPlan(principal,due.planId,due.key),errorIs("NOT_FOUND",404))
|
||||
for(const {record} of telemetry)await assert.rejects(applyPlan(principal,record.planId,record.key),errorIs("NOT_FOUND",404))
|
||||
console.log("Owned settings operations: five exact families, authenticated apply, physical 180-day purge, preserved settings and scoped replay")
|
||||
}finally{resumeSchedulerJobs();await closePool()}
|
||||
@@ -0,0 +1,199 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { logSettingsPatchSchema, logSettingsSchema } from "@mmapp/contracts"
|
||||
import { z } from "zod"
|
||||
import { connectProbeStorage } from "../../services/server-probe-storage-connection.js"
|
||||
import type { ProbeStorageSql } from "../../services/server-probe-journal-maintenance.js"
|
||||
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
||||
import { operationIdempotencyDigest } from "./operation-replay-tombstones.js"
|
||||
import { decryptSecretExisting, fingerprint, privateFingerprinterExisting, redact } from "./security.js"
|
||||
|
||||
const uuid = z.string().uuid()
|
||||
const positive = z.number().int().positive().safe()
|
||||
const object = z.record(z.string(),z.unknown())
|
||||
const sealed = z.object({sealed:z.string().min(1).max(262144)}).strict()
|
||||
const ids = z.array(positive).max(1000)
|
||||
const retention = z.number().int().min(1).max(365)
|
||||
const bool = z.boolean()
|
||||
const interval = (min:number,max:number)=>z.number().int().min(min).max(max)
|
||||
// Drizzle's string-mode timestamptz keeps PostgreSQL's space and hour-only
|
||||
// offset. Accept that exact wire form as well as an ISO timestamptz.
|
||||
const timestamp = z.string().regex(/^\d{4}-\d\d-\d\d[ T]\d\d:\d\d:\d\d(?:\.\d{1,6})?(?:Z|[+-]\d\d(?::\d\d)?)$/)
|
||||
.refine(value=>Number.isFinite(Date.parse(value.replace(" ","T").replace(/([+-]\d\d)$/,"$1:00"))))
|
||||
const metadata = {
|
||||
id:z.literal(1),createdAt:timestamp,updatedAt:timestamp,
|
||||
lastCollectedAt:timestamp.nullable(),lastDurationMs:z.number().int().safe().nullable(),
|
||||
lastError:z.string().max(16384).nullable(),
|
||||
}
|
||||
function telemetryFamily(version:number,domain:string,settings:z.ZodObject,hasEpoch:boolean,hasConfigurationProof:boolean){
|
||||
const originalSettings=hasConfigurationProof?settings.extend({configurationProof:z.string().regex(/^[0-9a-f]{64}$/)}):settings
|
||||
const effect=settings.extend({...metadata,...(hasEpoch?{pollConfigurationEpoch:uuid}:{})}).strict()
|
||||
return {version,domain,patch:settings.partial().strict().refine(value=>Object.keys(value).length>0),
|
||||
before:z.object({value:originalSettings.strict(),serverIds:ids}).strict(),effect}
|
||||
}
|
||||
const families={
|
||||
"logs.settings.update":{version:1,domain:null,patch:logSettingsPatchSchema,before:logSettingsSchema.strict(),effect:logSettingsSchema.strict()},
|
||||
"traffic.settings.update":telemetryFamily(2,"traffic-settings",z.object({enabled:bool,intervalSec:interval(5,3600),retentionDays:retention}),true,true),
|
||||
"uptime.settings.update":telemetryFamily(2,"uptime-settings",z.object({enabled:bool,resourcesEnabled:bool,pingEnabled:bool,
|
||||
speedEnabled:bool,intervalSec:interval(5,86400),probeIntervalSec:interval(5,86400),speedIntervalSec:interval(10,86400),retentionDays:retention}),true,true),
|
||||
"servers.api-ping.settings.update":telemetryFamily(2,"servers-api-ping-settings",z.object({enabled:bool,intervalSec:interval(10,86400)}),false,true),
|
||||
"internet-path.settings.update":telemetryFamily(1,"internet-path-settings",z.object({enabled:bool,intervalSec:interval(30,86400),retentionDays:retention}),true,false),
|
||||
} as const
|
||||
const planEvidence = z.object({
|
||||
target:z.object({scope:z.literal("global")}).strict(),
|
||||
objects:z.array(z.object({kind:z.string(),id:z.string(),serverId:positive.optional()}).strict()).max(1001),
|
||||
steps:z.tuple([z.object({kind:z.literal("service"),service:z.string(),input:z.object({settings:object}).strict()}).strict()]),
|
||||
snapshots:z.tuple([]),before:object,warnings:z.array(z.string()).max(8),
|
||||
verificationScope:z.literal("configuration").optional(),
|
||||
requiresHuman:z.literal(true).optional(),
|
||||
}).strict()
|
||||
|
||||
class RetirementRefused extends Error {}
|
||||
export class OrdinaryRetirementCommitUnknown extends Error {
|
||||
readonly automaticReplay = false
|
||||
constructor() { super("Ordinary operation retirement outcome unknown; automatic replay forbidden") }
|
||||
}
|
||||
const requireThat: (value: unknown) => asserts value = value => {
|
||||
if (!value) throw new RetirementRefused()
|
||||
}
|
||||
|
||||
// Fixed reviewed settings families without artifact, historic receipt or
|
||||
// inverse-plan consumers. Other action families need separate review.
|
||||
const ELIGIBLE = `FROM public.mcp_operations o
|
||||
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
||||
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
||||
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=0
|
||||
WHERE (p.operation_id,p.operation_version) IN (
|
||||
('logs.settings.update',1),('traffic.settings.update',2),('uptime.settings.update',2),
|
||||
('servers.api-ping.settings.update',2),('internet-path.settings.update',1)
|
||||
) AND p.state='completed'
|
||||
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL
|
||||
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
||||
AND isfinite(p.expires_at) AND p.expires_at<transaction_timestamp()
|
||||
AND s.state='verified' AND s.server_id IS NULL AND isfinite(s.finished_at)
|
||||
AND k.after_sealed IS NOT NULL
|
||||
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
|
||||
AND (SELECT count(*) FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)=1
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
|
||||
const CANDIDATES = `SELECT o.id,o.connection_id,o.finished_at::text AS finished_at ${ELIGIBLE}
|
||||
AND NOT EXISTS(SELECT 1 FROM public.mcp_ordinary_retirement_claims WHERE operation_id=o.id)
|
||||
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
|
||||
ORDER BY o.finished_at,o.id LIMIT 8`
|
||||
const SOURCE = `SELECT to_jsonb(o) AS operation,to_jsonb(p) AS plan,to_jsonb(s) AS step,to_jsonb(k) AS checkpoint ${ELIGIBLE}
|
||||
AND o.id=$1 AND o.connection_id=$2
|
||||
AND octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)+octet_length(to_jsonb(k)::text)<=1048576
|
||||
FOR UPDATE OF o,p,s,k NOWAIT`
|
||||
|
||||
async function transaction<T>(deadlineAt:number, work:(sql:ProbeStorageSql)=>Promise<T>):Promise<T> {
|
||||
const sql = await connectProbeStorage({deadlineAt})
|
||||
let committing=false,committed=false
|
||||
try {
|
||||
await sql.query("BEGIN")
|
||||
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
|
||||
const result=await work(sql)
|
||||
committing=true
|
||||
await sql.query("COMMIT")
|
||||
committed=true
|
||||
return result
|
||||
} catch(error) {
|
||||
if(committing)throw new OrdinaryRetirementCommitUnknown()
|
||||
try{await sql.query("ROLLBACK")}catch{/* Destroy this checkout. */}
|
||||
throw error
|
||||
} finally {
|
||||
sql.release(!committed)
|
||||
}
|
||||
}
|
||||
|
||||
async function prepare(sql:ProbeStorageSql, operationId:string, connectionId:string) {
|
||||
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT",[connectionId])).rows.length===1)
|
||||
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",["mcp-operation:"+operationId])).rows[0]?.locked===true)
|
||||
const rows=(await sql.query(SOURCE,[operationId,connectionId])).rows
|
||||
requireThat(rows.length===1)
|
||||
const raw=rows[0], operation=object.parse(raw.operation), plan=object.parse(raw.plan)
|
||||
const step=object.parse(raw.step),checkpoint=object.parse(raw.checkpoint)
|
||||
const open=async(value:unknown)=>JSON.parse(await decryptSecretExisting(sealed.parse(value).sealed))
|
||||
const original=planEvidence.parse(await open(plan.evidence))
|
||||
const action=z.enum(Object.keys(families) as [keyof typeof families,...(keyof typeof families)[]]).parse(plan.operation_id)
|
||||
const family=families[action],service=original.steps[0]
|
||||
requireThat(plan.operation_version===family.version&&service.service===action)
|
||||
requireThat(fingerprint(await open(plan.payload))===fingerprint(service.input))
|
||||
requireThat(fingerprint(step.intended_state)===fingerprint(redact(service)))
|
||||
const patch=object.parse(family.patch.parse(service.input.settings))
|
||||
const effect=object.parse(family.effect.parse(step.result))
|
||||
requireThat(Object.entries(patch).every(([key,value])=>effect[key]===value))
|
||||
requireThat(fingerprint(operation.result)===fingerprint({confirmedWrites:1,verificationScope:"configuration"}))
|
||||
for(const value of [checkpoint.before_sealed,checkpoint.after_sealed]){
|
||||
requireThat(typeof value==="string"&&value.length<=1024&&fingerprint(await open({sealed:value}))===fingerprint([]))
|
||||
}
|
||||
const serverObjects=original.objects.filter(item=>item.kind==="server")
|
||||
const serverIds=ids.parse(serverObjects.map(item=>item.serverId))
|
||||
requireThat(new Set(serverIds).size===serverIds.length&&serverObjects.every(item=>item.id===String(item.serverId)))
|
||||
const domainObjects=original.objects.filter(item=>item.kind!=="server")
|
||||
const before=family.before.parse(original.before)
|
||||
if(family.domain===null){
|
||||
requireThat(domainObjects.length===0&&original.requiresHuman===undefined)
|
||||
}else{
|
||||
requireThat(original.requiresHuman===true&&original.verificationScope==="configuration"
|
||||
&&domainObjects.length===1&&domainObjects[0].kind===family.domain&&domainObjects[0].id==="1"&&domainObjects[0].serverId===undefined)
|
||||
requireThat("serverIds" in before&&fingerprint(before.serverIds)===fingerprint(serverIds))
|
||||
}
|
||||
const signer=await privateFingerprinterExisting("mcp-ordinary-retirement-source-v1")
|
||||
return {planId:uuid.parse(plan.id),action,serverIds,key:z.string().min(1).max(200).parse(operation.idempotency_key),proof:signer(fingerprint(raw))}
|
||||
}
|
||||
|
||||
async function retire(operationId:string,connectionId:string,deadlineAt:number) {
|
||||
const token=randomUUID()
|
||||
await transaction(deadlineAt,async sql=>{
|
||||
const source=await prepare(sql,operationId,connectionId)
|
||||
const claim=await sql.query(`INSERT INTO public.mcp_ordinary_retirement_claims(operation_id,plan_id,connection_id,source_proof,claim_token)
|
||||
VALUES($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING RETURNING operation_id`,[operationId,source.planId,connectionId,source.proof,token])
|
||||
requireThat(claim.rows.length===1&&claim.rows[0].operation_id===operationId)
|
||||
})
|
||||
await transaction(deadlineAt,async sql=>{
|
||||
const claim=(await sql.query(`SELECT source_proof,plan_id FROM public.mcp_ordinary_retirement_claims
|
||||
WHERE operation_id=$1 AND connection_id=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,[operationId,connectionId,token])).rows
|
||||
requireThat(claim.length===1)
|
||||
const source=await prepare(sql,operationId,connectionId)
|
||||
requireThat(claim[0].source_proof===source.proof&&claim[0].plan_id===source.planId)
|
||||
const tombstone=await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
||||
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
||||
VALUES($1,$2,$3,$4,$5,$6,true) RETURNING operation_id`,
|
||||
[operationId,source.planId,connectionId,operationIdempotencyDigest(connectionId,source.key),source.action,source.serverIds])
|
||||
requireThat(tombstone.rows.length===1&&tombstone.rows[0].operation_id===operationId)
|
||||
const checkpoint=await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
||||
const step=await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
||||
const operation=await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id AS operation_id",[operationId,connectionId,source.planId])
|
||||
const plan=await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id AS plan_id",[source.planId,connectionId])
|
||||
requireThat([checkpoint,step,operation].every(result=>result.rows.length===1&&result.rows[0].operation_id===operationId)
|
||||
&&plan.rows.length===1&&plan.rows[0].plan_id===source.planId)
|
||||
})
|
||||
}
|
||||
|
||||
let cursor:{finishedAt:string;operationId:string}|undefined
|
||||
export function retireOrdinaryMcpOperations() {
|
||||
return sharedDatabaseGate.request(async()=>{
|
||||
const deadlineAt=Date.now()+15000
|
||||
const rows=await transaction(deadlineAt,async sql=>(await sql.query(CANDIDATES,[cursor?.finishedAt??null,cursor?.operationId??null])).rows)
|
||||
requireThat(rows.length<=8)
|
||||
let scanned=0,retired=0
|
||||
for(const row of rows){
|
||||
if(Date.now()>=deadlineAt-2000||retired>=2)break
|
||||
const id=uuid.parse(row.id),connectionId=uuid.parse(row.connection_id)
|
||||
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
|
||||
cursor={finishedAt:row.finished_at,operationId:id}
|
||||
scanned++
|
||||
try{await retire(id,connectionId,deadlineAt);retired++}
|
||||
catch(error){if(error instanceof OrdinaryRetirementCommitUnknown)throw error}
|
||||
}
|
||||
if(scanned===rows.length&&rows.length<8)cursor=undefined
|
||||
return {scanned,retired,automaticReplay:false as const}
|
||||
})
|
||||
}
|
||||
@@ -1,5 +1,8 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { readFile } from "node:fs/promises"
|
||||
import { test } from "node:test"
|
||||
import { runInNewContext } from "node:vm"
|
||||
import ts from "typescript"
|
||||
import type { ActionContext } from "./action-types.js"
|
||||
import { loadReservedScopedRecoverySource } from "./routing-community-scoped-recovery-source.js"
|
||||
import { executeReservedScopedRecovery, scopedRecoveryIntentFromCheckpoint, validateReservedInverseVerified } from "./routing-community-scoped-recovery.js"
|
||||
@@ -16,6 +19,67 @@ test("a serialized allocation cannot enter the shared inverse engine", async ()
|
||||
await assert.rejects(executeReservedScopedRecovery(forged, {} as Parameters<typeof executeReservedScopedRecovery>[1]), /capability/)
|
||||
})
|
||||
|
||||
test("confirmed inverse spend quarantines the exact slot on post-spend refusal without replay", async () => {
|
||||
// Execute the actual boundary body with only its I/O dependencies substituted.
|
||||
// This does not mint a production capability or claim native/SQL acceptance.
|
||||
const source = ts.createSourceFile("routing-community-scoped-recovery.ts",
|
||||
await readFile(new URL("./routing-community-scoped-recovery.ts", import.meta.url), "utf8"), ts.ScriptTarget.ES2022)
|
||||
const declaration = source.statements.find(node => ts.isFunctionDeclaration(node) && node.name?.text === "executeReservedScopedRecovery")
|
||||
assert(declaration)
|
||||
const compiled = ts.transpileModule(declaration.getText(source), { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2022 } }).outputText
|
||||
const allocation = Object.freeze({ kind: "routing-reserved-inverse-allocation" })
|
||||
const execution = Object.freeze({ kind: "routing-reserved-inverse-execution" })
|
||||
const principal = { connectionId: "owned", credentialId: "original" }
|
||||
const original = new Error("original authority failure"), result = Object.freeze({ outcome: "recovered" })
|
||||
const scenarios = {
|
||||
spend: ["spend"],
|
||||
inspect: ["spend", "inspect", "quarantine"],
|
||||
authority: ["spend", "inspect", "authority", "quarantine"],
|
||||
source: ["spend", "inspect", "authority", "source", "quarantine"],
|
||||
proof: ["spend", "inspect", "authority", "source", "proof", "quarantine"],
|
||||
engine: ["spend", "inspect", "authority", "source", "proof", "engine", "quarantine"],
|
||||
success: ["spend", "inspect", "authority", "source", "proof", "engine"],
|
||||
} as const
|
||||
for (const [scenario, expected] of Object.entries(scenarios)) {
|
||||
const events: string[] = [], exports: Record<string, unknown> = {}, capabilities = new WeakMap()
|
||||
let request: object | undefined
|
||||
const at = (phase: string) => { events.push(phase); if (phase === scenario) throw original }
|
||||
runInNewContext(compiled, {
|
||||
exports, reservedExecutions: capabilities, fingerprint: JSON.stringify,
|
||||
createRoutingReservationStore: () => ({
|
||||
async beginInverseAttempt(value: unknown) {
|
||||
assert.equal(value, allocation); at("spend"); return { reservationId: "spent-slot", execution }
|
||||
},
|
||||
async inspectInverseExecution(value: unknown) {
|
||||
assert.equal(value, execution); at("inspect")
|
||||
return { principal, sourceOperationId: "source", deadline: "2026-10-29T01:00:00.000Z" }
|
||||
},
|
||||
async markUnknown(id: string, lane: string) {
|
||||
assert.equal(id, "spent-slot"); assert.equal(lane, "inverse"); at("quarantine")
|
||||
throw new Error("quarantine persistence unavailable")
|
||||
},
|
||||
}),
|
||||
effectiveMcpConnection: async () => { at("authority"); return { owner_id: "owner", policy: {}, owner_permissions: [] } },
|
||||
loadReservedScopedRecoverySource: async () => {
|
||||
at("source"); return { profile: { serverId: 1 }, input: {}, configuration: { packet: {} } }
|
||||
},
|
||||
accessGeneration: async () => "generation",
|
||||
deadline: () => Date.parse("2026-10-29T01:00:00.000Z"),
|
||||
sealedProof: async () => { at("proof"); return "proof" },
|
||||
executeScopedRecovery: async (value: object) => {
|
||||
request = value; assert.equal(capabilities.get(value), execution); at("engine"); return result
|
||||
},
|
||||
})
|
||||
const execute = exports.executeReservedScopedRecovery as
|
||||
(allocation: unknown, expected: { sourceOperationId: string; principal: typeof principal }) => Promise<unknown>
|
||||
const outcome = execute(allocation, { sourceOperationId: "source", principal })
|
||||
if (scenario === "success") assert.equal(await outcome, result)
|
||||
else await assert.rejects(outcome, error => error === original)
|
||||
assert.deepEqual(events, expected, scenario)
|
||||
if (request) assert.equal(capabilities.has(request), false)
|
||||
}
|
||||
})
|
||||
|
||||
test("configuration equality or generic VERIFIED cannot replace reserved signed receipts", async () => {
|
||||
const expected = {} as Parameters<typeof validateReservedInverseVerified>[1]
|
||||
for (const row of [{}, { state: "verified", phase: "RECOVERY_VERIFIED", configurationVerified: true }, { inverse_intent_sealed: "x".repeat(524289) }]) {
|
||||
|
||||
@@ -35,8 +35,9 @@ import { registerManagementChannel } from "./management.js"
|
||||
import { isSchedulerJobsPaused, pauseSchedulerJobsAndWait, resumeSchedulerJobs } from "../../services/scheduler-running.js"
|
||||
|
||||
/** Disposable SQL + loopback HTTP only. Does not access or configure CHR. */
|
||||
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward" } = {}) {
|
||||
const sid = 131, writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
|
||||
export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options: { originalEffectsOnly?: boolean; originalCredentialRevocationOnly?: boolean; forwardRoundsOnly?: boolean; manualInverseChargeOnly?: boolean; manualInverseChargeCase?: "positive" | "quota" | "concurrent" | "unknown" | "revoked" | "nonpersistent" | "mixed-forward"; lateInverseAuthorityChange?: "edit" | "disable" | "grant" } = {}) {
|
||||
const sid = options.lateInverseAuthorityChange ? { edit: 131, disable: 132, grant: 133 }[options.lateInverseAuthorityChange] : 131
|
||||
const writer: AuthUser = { ...actor, permissions: [...new Set([...actor.permissions, "mm:filters:write"])] }
|
||||
const initial = compileManagedFilterRules([{ community: "65000:10", gateway: "192.0.2.1", outIface: "", action: "route", description: "" }, { community: "65000:20", gateway: "192.0.2.5", outIface: "", action: "route", description: "" }])
|
||||
const filters = ["a", "b"].map(key => ({ ".id": "*f" + key, chain: "custom-" + key, rule: initial, disabled: "false", comment: managedComment("NEVER_SCOPED_COMMENT_" + key), opaque: "NEVER_SCOPED_OPAQUE" }))
|
||||
let mode: "normal" | "ignored" | "lost" | "collateral" | "post-failed" | "post-unknown" = "normal", writes = 0, requests = 0
|
||||
@@ -423,8 +424,9 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
|
||||
await assertOriginalCredentialRevocation()
|
||||
return {accepted:true,originalCredentialRevocation:true,unauthorized401:true,fullAdmissionRollback:true,nativeReads:0,nativeWrites:0,realSql:true,allWriterLedgerComplete:false}
|
||||
}
|
||||
if (options.manualInverseChargeOnly) {
|
||||
if (options.manualInverseChargeOnly || options.lateInverseAuthorityChange) {
|
||||
const chargeCase = options.manualInverseChargeCase ?? "positive"
|
||||
if (options.lateInverseAuthorityChange) assert.equal(chargeCase, "positive")
|
||||
const persistent = chargeCase !== "nonpersistent"
|
||||
// Same genuine fixed-reader baseline/CAS used by the existing persistent
|
||||
// acceptance below. No hand-written baseline, sealed plan or authority.
|
||||
@@ -573,6 +575,48 @@ export async function acceptScopedRouteCommunitySwitch(actor: AuthUser, options:
|
||||
realSql: true, allWriterLedgerComplete: false, reservedDispatchEnabled: false, autoRecoveryEnabled: false }
|
||||
}
|
||||
console.log(JSON.stringify({ stage: "manual-inverse:human-recovery-apply" }))
|
||||
const lateChange = options.lateInverseAuthorityChange
|
||||
if (lateChange) {
|
||||
const priorIntent = (await pool.query("SELECT coalesce(max(id),0)::text AS id FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent'", [connectionId])).rows[0].id
|
||||
let writeId: string | undefined
|
||||
afterIntent = async () => {
|
||||
const intents = (await pool.query(`SELECT payload FROM mcp_audit WHERE connection_id=$1 AND id>$2::bigint
|
||||
AND action='configuration.intent' AND payload->>'path'=$3`,
|
||||
[connectionId, priorIntent, "/routing/filter/rule/" + loaded.configuration.packet.nativeId])).rows
|
||||
assert.equal(intents.length, 1)
|
||||
assert.equal(intents[0].payload.method, "PATCH")
|
||||
assert.equal(intents[0].payload.serverId, sid)
|
||||
writeId = intents[0].payload.writeId
|
||||
assert.match(writeId ?? "", /^[0-9a-f-]{36}$/)
|
||||
const sql = {
|
||||
edit: "UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late inverse edit\"') WHERE id=$1",
|
||||
disable: "UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1",
|
||||
grant: "UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1",
|
||||
}[lateChange]
|
||||
assert.equal((await pool.query(sql, [id])).rowCount, 1)
|
||||
}
|
||||
const inverse = await execute(recovery.planId)
|
||||
assert.equal(afterIntent, undefined)
|
||||
assert.ok(writeId)
|
||||
assert.notEqual(inverse.state, "applied")
|
||||
assert.equal(writes, afterForward.writes)
|
||||
assert.equal(inversePostChecks, 0)
|
||||
const auditRows = (await pool.query(`SELECT action FROM mcp_audit WHERE connection_id=$1 AND payload->>'writeId'=$2
|
||||
ORDER BY id`, [connectionId, writeId])).rows
|
||||
assert.deepEqual(auditRows.map(row => row.action), ["configuration.intent", "configuration.unconfirmed"])
|
||||
const refusal = (await pool.query(`SELECT payload->>'phase' AS phase, payload->>'code' AS code
|
||||
FROM mcp_audit WHERE connection_id=$1 AND action='routing.scoped.recovery-unconfirmed'`, [connectionId])).rows
|
||||
assert.deepEqual(refusal, [{ phase: "before-dispatch-context", code: "STATE_CONFLICT" }])
|
||||
assert.deepEqual((await pool.query(`SELECT state,verified_sealed FROM mcp_scoped_recovery_intents
|
||||
WHERE source_operation_id=$1 AND recovery_operation_id=$2`, [source.id, inverse.id])).rows,
|
||||
[{ state: "unknown", verified_sealed: null }])
|
||||
assert.equal((await pool.query(`SELECT count(*)::int AS n FROM mcp_operation_checkpoints
|
||||
WHERE operation_id=$1 AND configuration_sealed IS NOT NULL`, [inverse.id])).rows[0].n, 0)
|
||||
assert.equal((await pool.query("SELECT error_code FROM mcp_operations WHERE id=$1", [inverse.id])).rows[0].error_code,
|
||||
"MUTATION_OUTCOME_UNKNOWN")
|
||||
return { accepted: true, lateInverseAuthorityChange: lateChange, durableIntents: 1, inverseNativeWrites: 0,
|
||||
realSql: true, realMcpDispatcher: true, fullInverseLifecycleRepeated: false }
|
||||
}
|
||||
if (chargeCase === "quota" || chargeCase === "revoked") {
|
||||
if (chargeCase === "revoked") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [id])
|
||||
const beforeOperations = (await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE connection_id=$1", [connectionId])).rows[0].n
|
||||
|
||||
@@ -257,24 +257,30 @@ async function checkpoint(value: Session, intent: Intent, phase: Checkpoint["pha
|
||||
export async function executeReservedScopedRecovery(allocation: RoutingReservedInverseAllocation, expected: { sourceOperationId: string; principal: McpPrincipal }) {
|
||||
const store = createRoutingReservationStore(), spent = await store.beginInverseAttempt(allocation)
|
||||
const execution = spent.execution
|
||||
const binding = await store.inspectInverseExecution(execution)
|
||||
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
|
||||
const current = await effectiveMcpConnection(binding.principal)
|
||||
const context: ActionContext = { principal: binding.principal, connection: current }
|
||||
const source = await loadReservedScopedRecoverySource(execution, context)
|
||||
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
|
||||
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
|
||||
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
|
||||
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
|
||||
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
|
||||
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
|
||||
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
|
||||
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
|
||||
input.requestProof = await sealedProof(input, "request")
|
||||
reservedExecutions.set(input, execution)
|
||||
try { return await executeScopedRecovery(input) }
|
||||
catch (error) { await store.markUnknown(binding.reservationId, "inverse").catch(() => {}); throw error }
|
||||
finally { reservedExecutions.delete(input) }
|
||||
try {
|
||||
const binding = await store.inspectInverseExecution(execution)
|
||||
if (binding.sourceOperationId !== expected.sourceOperationId || fingerprint(binding.principal) !== fingerprint(expected.principal)) throw fail("Reserved runner identity differs")
|
||||
const current = await effectiveMcpConnection(binding.principal)
|
||||
const context: ActionContext = { principal: binding.principal, connection: current }
|
||||
const source = await loadReservedScopedRecoverySource(execution, context)
|
||||
const channel: ManagementTransportBinding | undefined = source.input.managementBindingCiphertext == null ? undefined
|
||||
: JSON.parse(await decryptSecret(String(source.input.managementBindingCiphertext)))
|
||||
const input: Record<string, unknown> = { mode: "reserved", serverId: source.profile.serverId, sourceOperationId: binding.sourceOperationId,
|
||||
sourceProof: source.sourceProof, configurationProof: binding.configurationProof, sourceReceiptId: binding.sourceReceiptId,
|
||||
profile: source.profile, candidateId: source.configuration.packet.originalCandidateId, rules: source.input.beforeRules, beforeRules: source.input.rules,
|
||||
principal: binding.principal, ownerId: current.owner_id, ownerProof: fingerprint(current.owner_permissions), policyProof: fingerprint(current.policy), connectionGeneration: current.generation,
|
||||
generation: await accessGeneration(), receiptId: binding.inverseReceiptId, proofRevision: 1,
|
||||
validUntil: new Date(Math.min(Date.parse(binding.deadline), deadline(source, context, channel?.expiresAt))).toISOString(), managementBindingCiphertext: source.input.managementBindingCiphertext ?? null }
|
||||
input.requestProof = await sealedProof(input, "request")
|
||||
reservedExecutions.set(input, execution)
|
||||
try { return await executeScopedRecovery(input) }
|
||||
finally { reservedExecutions.delete(input) }
|
||||
} catch (error) {
|
||||
// Acknowledged spend owns this exact slot even when its authority can no
|
||||
// longer be inspected. A refused or uncertain spend never enters here.
|
||||
await store.markUnknown(spent.reservationId, "inverse").catch(() => {})
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Fixed-domain receipt verifier used by the reservation store. It recognizes
|
||||
|
||||
@@ -14,7 +14,9 @@ import { nativeBgpRows, bgpTemplateReferences } from "../../services/bgp-export-
|
||||
import { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow } from "../../services/bgp-filter-binding-model.js"
|
||||
import { canonicalVrfMembers } from "../../services/vrf-members.js"
|
||||
import { readOspfEffectiveDisabled } from "../../services/ospf-effective-disabled.js"
|
||||
import { isConfirmedRoutingFibTable } from "../../services/routing-table-fib.js"
|
||||
import * as contextModel from "./routing-community-context-model.js"
|
||||
import { prepareOriginalEffectSelection } from "./routing-original-effects.js"
|
||||
import { readMeasurementAddress } from "./routing-measurement-address.js"
|
||||
import { routeContextClients } from "./routing-community-context-native.js"
|
||||
import { pathProofTag } from "./path-proof-purpose.js"
|
||||
@@ -65,9 +67,8 @@ async function fixture(autoRecovery = false) {
|
||||
afterHttp?.(); afterHttp = undefined
|
||||
if (afterPath?.path === path) { const run = afterPath.run; afterPath = undefined; run() }
|
||||
})
|
||||
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
||||
const now = Date.now(), fingerprint = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
|
||||
const endpoint = { id: 1, host: "127.0.0.1", port: (router.address() as { port: number }).port, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
|
||||
const endpoint = { id: 1, host: "127.0.0.1", port: 0, username: "fixture", password: "private", useSsl: false, verifySsl: true, enabled: true, routingEndpointEpoch: randomUUID() }
|
||||
const principal = { connectionId: randomUUID(), credentialId: randomUUID(), credentialExpiresAt: new Date(now + 600000).toISOString() }
|
||||
const connection = { id: principal.connectionId, owner_id: "owner", generation: "generation", owner_permissions: ["mm:network:read"],
|
||||
policy: { permissions: ["mm:network:read"], scope: { serverIds: [1] }, expiresAt: new Date(now + 600000).toISOString() }, owner_valid_until: new Date(now + 600000) }
|
||||
@@ -92,6 +93,7 @@ async function fixture(autoRecovery = false) {
|
||||
"./routing-community-context-native.js": { routeContextClients }, "../../services/vrf-members.js": { canonicalVrfMembers }, "../../services/ospf-effective-disabled.js": { readOspfEffectiveDisabled },
|
||||
"./routing-measurement-address.js": { readMeasurementAddress } }
|
||||
await loadSource("./routing-community-context.ts", { ...common, "./routing-network-native-session.js": native,
|
||||
"../../services/routing-table-fib.js": { isConfirmedRoutingFibTable }, "./routing-original-effects.js": { prepareOriginalEffectSelection },
|
||||
"../../services/bgp-export-model.js": { nativeBgpRows, bgpTemplateReferences }, "../../services/bgp-filter-binding-model.js": { bgpFilterName, effectiveBgpFilter, stableBgpPolicyRow },
|
||||
"./routing-community-context-model.js": contextModel, "./path-proof-purpose.js": { pathProofTag },
|
||||
"./management-transport.js": { assertManagementTransport: async () => { throw Error("No alternate management fixture") } } }, context)
|
||||
@@ -103,14 +105,23 @@ async function fixture(autoRecovery = false) {
|
||||
await loadSource("./routing-network-scoped-context.ts", { "./routing-network-native-session.js": native }, privateApi)
|
||||
const create = native.createRoutingNetworkNativeSession as (options: unknown) => Promise<Session>
|
||||
const abort = new AbortController()
|
||||
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } })
|
||||
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
||||
endpoint.port = (router.address() as { port: number }).port
|
||||
const closeRouter = () => {
|
||||
router.closeAllConnections()
|
||||
return new Promise<void>(resolve => router.close(() => resolve()))
|
||||
}
|
||||
const session = await create({ sourceOperationId: row.id, context: { principal, connection, signal: abort.signal }, authorize: authority, deadline: now + 119000, manifest: { servers: [1], probes: [] } }).catch(async error => {
|
||||
await closeRouter()
|
||||
throw error
|
||||
})
|
||||
const readContext = privateApi.readRoutingNetworkScopedCommunityContext as (token: Token) => Promise<Context>
|
||||
return { session, readContext: () => readContext(session.sharedContext), forged: () => readContext(JSON.parse(JSON.stringify(session.sharedContext))),
|
||||
readResource: native.readRoutingNetworkSharedResource as (token: Token, server: number, path: string, body?: Record<string, string>) => Promise<unknown>,
|
||||
bypassTail: context.readRouteCommunityContextWithSharedNativeSession as (token: Token) => Promise<unknown>,
|
||||
counters: () => ({ requests, wireBytes, wireRows, rawGetObjects }), bulk: (value: number) => { bulk = value }, padding: (value: number) => { padding = value },
|
||||
revoke: () => { revoked = true }, abort: () => abort.abort(), afterHttp: (hook: () => void) => { afterHttp = hook }, afterPath: (path: string, hook: () => void) => { afterPath = { path, run: hook } },
|
||||
cleanup: async () => { session.close(); router.closeAllConnections(); await new Promise<void>(resolve => router.close(() => resolve())) } }
|
||||
cleanup: async () => { session.close(); await closeRouter() } }
|
||||
}
|
||||
|
||||
test("actual source context and snapshots share one pinned HTTP request/byte/row budget including object /get", async () => {
|
||||
|
||||
@@ -19,7 +19,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const { pool } = await import("../../db/index.js")
|
||||
const { mcpConnectionPolicySchema } = await import("@mmapp/contracts/mcp")
|
||||
const { routeAiProfileSchema } = await import("@mmapp/contracts/mcp-routing")
|
||||
const { accessGeneration } = await import("./security.js")
|
||||
const { accessGeneration, McpError } = await import("./security.js")
|
||||
const { readRoutingEndpoints, createRoutingSourceContext } = await import("./routing-observation-source.js")
|
||||
const { resolveRoutingProfileAuthority } = await import("./routing-profile-authority.js")
|
||||
const { captureScopedRecoveryAuthority, scopedRecoveryDeadlineCeiling } = await import("./routing-community-scoped-recovery.js")
|
||||
@@ -27,7 +27,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
assert.equal((await pool.query("SELECT current_database() AS name")).rows[0].name, "postgres")
|
||||
const previousEnabled = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled as boolean
|
||||
const profileIds: string[] = [], ids: number[] = []
|
||||
let nativeHttpRequests = 0, callbackDispatches = 0, durableIntents = 0
|
||||
let nativeHttpRequests = 0, callbackDispatches = 0, dispatchFences = 0
|
||||
const listener = createServer((req, res) => { nativeHttpRequests++; req.resume(); res.setHeader("content-type", "application/json"); res.end("{}") })
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => { listener.once("error", reject); listener.listen(0, "127.0.0.1", () => { listener.off("error", reject); resolve() }) })
|
||||
@@ -65,12 +65,7 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const path = "/routing/filter/rule/*1"
|
||||
await assert.rejects(withVerifiedConfigurationChange({ actorId: ownerId, connectionId, approved: true, serverIds: ids, serverWide: true, objects: [],
|
||||
authorize: async () => { await captured.assert() }, beforeDispatch: async () => {
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.intent' AND payload->>'path'=$2", [connectionId, path])).rows[0].n, 1)
|
||||
durableIntents++
|
||||
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
|
||||
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
|
||||
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
|
||||
await captured.assert()
|
||||
dispatchFences++
|
||||
},
|
||||
}, () => executeGuardedConfigurationWrite(ids[0], "PATCH", path, async () => {
|
||||
callbackDispatches++
|
||||
@@ -78,12 +73,17 @@ export async function acceptRoutingRecoveryAuthoritySql() {
|
||||
const req = request({ hostname: "127.0.0.1", port, path: "/rest" + path, method: "PATCH", timeout: 1000 }, res => { res.resume(); res.once("end", resolve) })
|
||||
req.once("error", reject); req.once("timeout", () => req.destroy(new Error("Synthetic loopback timeout"))); req.end("{}")
|
||||
})
|
||||
})), /Постоянный допуск/)
|
||||
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action='configuration.unconfirmed'", [connectionId])).rows[0].n, 1)
|
||||
})), (error: unknown) => error instanceof McpError && error.code === "STATE_CONFLICT" && /Активный routing profile/.test(error.message))
|
||||
assert.equal(callbackDispatches, 0); assert.equal(nativeHttpRequests, 0); assert.equal(dispatchFences, 0)
|
||||
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_audit WHERE connection_id=$1 AND action IN ('configuration.intent','configuration.unconfirmed','configuration.acknowledged')", [connectionId])).rows[0].n, 0)
|
||||
|
||||
if (change === "edit") await pool.query("UPDATE mcp_routing_profiles SET version=version+1,config=jsonb_set(config,'{name}','\"late edit\"') WHERE id=$1", [profileId])
|
||||
if (change === "disable") await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=$1", [profileId])
|
||||
if (change === "grant") await pool.query("UPDATE mcp_routing_profiles SET grant_expires_at=clock_timestamp()-interval '1 second' WHERE id=$1", [profileId])
|
||||
await assert.rejects(captured.assert(), /Постоянный допуск/)
|
||||
}
|
||||
return { cases: 3, lateMutations: ["edit", "disable", "grant"], deepCaptureCases: 3, durableIntents, callbackDispatches, nativeHttpRequests,
|
||||
productionSessionAuthorityFence: true, realSql: true, productionHmac: true, fullInverseLifecycleRepeated: false, controllerEnabled: false, externalNetwork: false }
|
||||
return { cases: 3, deepCaptureCases: 3, directGuardBeforeIntent: true, dispatchFences, callbackDispatches, nativeHttpRequests,
|
||||
capturedAuthorityChanges: ["edit", "disable", "grant"], realSql: true, productionHmac: true, controllerEnabled: false, externalNetwork: false }
|
||||
} finally {
|
||||
if (profileIds.length) await pool.query("UPDATE mcp_routing_profiles SET enabled=false WHERE id=ANY($1::uuid[])", [profileIds])
|
||||
if (ids.length) await pool.query("UPDATE servers SET enabled=false WHERE id=ANY($1::int[])", [ids])
|
||||
|
||||
@@ -2,9 +2,26 @@ import { assertOwnedRoutingRecoveryAuthoritySql, acceptRoutingRecoveryAuthorityS
|
||||
|
||||
// Guard precedes imports that configure a pool or load private MCP state.
|
||||
assertOwnedRoutingRecoveryAuthoritySql()
|
||||
process.env.AUTH_REQUIRED = "true"
|
||||
process.env.AUTH_JWT_SECRET = "owned-routing-recovery-authority-acceptance-secret"
|
||||
process.env.AUTH_ISSUER = "https://owned-routing-recovery-authority.test"
|
||||
const { pool, closePool } = await import("../../db/index.js")
|
||||
try {
|
||||
const { applySqlMigrations } = await import("../../db/migrate.js")
|
||||
await applySqlMigrations(pool)
|
||||
console.log("captured recovery authority SQL acceptance:", JSON.stringify(await acceptRoutingRecoveryAuthoritySql()))
|
||||
} finally { await closePool() }
|
||||
const { updateMcpSettings } = await import("./store.js")
|
||||
await import("./actions.js")
|
||||
const { acceptScopedRouteCommunitySwitch } = await import("./routing-community-scoped-acceptance.js")
|
||||
const actor = { id: "owned-routing-recovery-authority", email: "[email protected]", name: "Owned recovery authority",
|
||||
apps: ["mm"], permissions: ["mm:settings:admin", "mm:mcp:admin", "mm:mcp:approve", "mm:network:read", "mm:network:write", "mm:filters:write", "mm:servers:admin"],
|
||||
issuedAt: Date.now() - 1000, expiresAt: Date.now() + 900000 }
|
||||
await updateMcpSettings({ enabled: true }, actor)
|
||||
for (const lateInverseAuthorityChange of ["edit", "disable", "grant"] as const) {
|
||||
const evidence = await acceptScopedRouteCommunitySwitch(actor, { lateInverseAuthorityChange })
|
||||
console.log("genuine late inverse authority SQL acceptance:", JSON.stringify(evidence))
|
||||
}
|
||||
} finally {
|
||||
await (await import("../../services/scheduler.js")).stopSchedulerForDatabaseOperation()
|
||||
await closePool()
|
||||
}
|
||||
@@ -352,6 +352,47 @@ export async function acceptRoutingReservationSql(options: { runtimeAdmission?:
|
||||
await assert.rejects(store.beginForwardAttempt(reservation.forwardAllocation), /capability/)
|
||||
names.push("spent forward/no genuine checkpoint and preterminal unused-proof refuse; unknown never replays or refunds")
|
||||
}
|
||||
{
|
||||
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
|
||||
const { saveScopedConfigurationCheckpoint, saveScopedNetworkCheckpoint } = await import("./routing-community-scoped-checkpoint.js")
|
||||
const { executeReservedScopedRecovery } = await import("./routing-community-scoped-recovery.js")
|
||||
await pool.query("UPDATE mcp_operations SET state='running' WHERE id=$1", [source.operationId])
|
||||
const step = { kind: "service", service: "routing.community.scoped", input: source.input }
|
||||
await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,state,intended_state) VALUES($1,0,'intent',$2::jsonb)",
|
||||
[source.operationId, JSON.stringify(security.redact(step))])
|
||||
await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,0,$2)",
|
||||
[source.operationId, await security.encryptSecret(JSON.stringify(source.input.beforeRules))])
|
||||
assert(reservation.forwardAllocation)
|
||||
await store.beginForwardAttempt(reservation.forwardAllocation)
|
||||
// Use the actual checkpoint writers for production crypto and SQL, but
|
||||
// deliberately seed a synthetic failed-forward observation. No PATCH
|
||||
// or genuine network observation is part of this owned component case.
|
||||
const requestSeal = String(source.input.requestProof)
|
||||
const configuration = await saveScopedConfigurationCheckpoint(source.input, requestSeal)
|
||||
await saveScopedNetworkCheckpoint(source.input, requestSeal, { id: randomUUID(), candidateId: "b",
|
||||
status: "failed", observedAt: configuration.observedAt })
|
||||
const published = await store.recordForwardOutcome(reservation.reservationId)
|
||||
assert(published.allocation)
|
||||
assert.equal(published.receipt.state.inverse, "held")
|
||||
const refused = { sourceOperationId: randomUUID(), principal: f.principal }
|
||||
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /Reserved runner identity differs/)
|
||||
const unknown = await store.read(reservation.reservationId, f.principal)
|
||||
assert.deepEqual(unknown.state, { ...published.receipt.state, revision: published.receipt.state.revision + 2, inverse: "unknown" })
|
||||
assert.equal(unknown.chargedMutationUnits, 2)
|
||||
assert.equal(unknown.chargedProfileSlots, 2)
|
||||
assert.equal(unknown.refundsAvailable, false)
|
||||
const shared = (await pool.query(`SELECT state,slot_phase,inverse_intent_sealed,inverse_configuration_sealed,verified_sealed
|
||||
FROM mcp_scoped_recovery_intents WHERE reservation_id=$1`, [reservation.reservationId])).rows
|
||||
assert.deepEqual(shared, [{ state: "unknown", slot_phase: "reserved-attempt", inverse_intent_sealed: null,
|
||||
inverse_configuration_sealed: null, verified_sealed: null }])
|
||||
await assert.rejects(executeReservedScopedRecovery(published.allocation, refused), /capability/)
|
||||
await assert.rejects(store.recordForwardOutcome(reservation.reservationId), /republished/)
|
||||
await assert.rejects(store.proveUnusedInverse(reservation.reservationId), /terminal applied/)
|
||||
assert.equal(await used(f.connectionId), 2)
|
||||
assert.equal((await store.readHourlyCharges(f.profileId, f.principal)).reservedCharges, 2)
|
||||
assert.deepEqual((await store.read(reservation.reservationId, f.principal)).state, unknown.state)
|
||||
names.push("genuine inverse capability on synthetic failed-forward checkpoints: acknowledged spend, late identity refusal, signed same-slot unknown and both charges conserved without replay or refund")
|
||||
}
|
||||
{
|
||||
const f = await fixture(), source = await f.makeSource(), reservation = await store.reserve(source.operationId)
|
||||
const saved = (await pool.query("SELECT state_proof FROM mcp_routing_reservations WHERE id=$1", [reservation.reservationId])).rows[0].state_proof
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { pollErrorLog } from "../../services/server-poll-error-provenance.js"
|
||||
import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js"
|
||||
import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js"
|
||||
import { preparedPlanExpiry } from "./plan-expiry.js"
|
||||
import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js"
|
||||
import { randomUUID, createHash } from "node:crypto"
|
||||
@@ -23,6 +25,7 @@ import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOpera
|
||||
import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js"
|
||||
import { evaluateMcpPolicy } from "./policy.js"
|
||||
import { findAction, findActionByOperation } from "./actions.js"
|
||||
import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js"
|
||||
import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
|
||||
import { audit, getConnection, transaction, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js"
|
||||
import { executeRosStep, readRosSnapshot, resourceById, routerClient, type RosSnapshot } from "./resources.js"
|
||||
@@ -291,6 +294,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
|
||||
reservedRouting = !!await routingReservationEvidence(original, originalEvidence)
|
||||
return existing
|
||||
}
|
||||
await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey)
|
||||
await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))")
|
||||
const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')")
|
||||
if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429)
|
||||
@@ -359,6 +363,7 @@ export async function applyPlan(principal: McpPrincipal, planId: string, idempot
|
||||
async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise<void> {
|
||||
return withMcpOperationExecutionOrigin(operationAdmission, async () => {
|
||||
const lease = await pool.connect()
|
||||
let terminalProbe: Awaited<ReturnType<typeof executeOperationWithLease>>
|
||||
let locked = false, released = false
|
||||
const ownerLifetime = new AbortController()
|
||||
// This closure is the SINGLE owner of physical destruction/release.
|
||||
@@ -380,7 +385,7 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
|
||||
throw error
|
||||
}
|
||||
if (released || !locked) return
|
||||
await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
|
||||
terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
|
||||
() => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant))
|
||||
} finally {
|
||||
if (!released && locked) {
|
||||
@@ -395,9 +400,14 @@ async function executeOperation(operationId: string, principal: McpPrincipal, ro
|
||||
lease.release()
|
||||
}
|
||||
}
|
||||
// Capture has its own bounded checkout and may fail without rewriting the
|
||||
// acknowledged terminal outcome or redispatching the diagnostic.
|
||||
if (terminalProbe) {
|
||||
try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ }
|
||||
}
|
||||
})
|
||||
}
|
||||
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant): Promise<void> {
|
||||
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) {
|
||||
const { rows: [operation] } = await pool.query<StoredOperation>("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId])
|
||||
if (!operation) return
|
||||
const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence
|
||||
@@ -576,14 +586,18 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
})
|
||||
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks())
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await transaction(async client => {
|
||||
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
|
||||
const finishedAt = await transaction(async client => {
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await client.query("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1", [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
|
||||
const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1"
|
||||
+ (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""),
|
||||
[operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id])
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) })
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
return receiptId ? terminal.rows[0]?.finished_at : undefined
|
||||
})
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
if (routingReservationId && reservedForwardPassed) {
|
||||
@@ -592,6 +606,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {})
|
||||
}
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt }
|
||||
} catch (error) {
|
||||
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
|
||||
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
|
||||
|
||||
@@ -12,9 +12,9 @@ const principal = { connectionId, credentialId, credentialExpiresAt: new Date(no
|
||||
const proof = async (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex")
|
||||
const context = { principal, connection: { id: connectionId, owner_id: "owner", owner_claims_issued_at: new Date(now - 1000) } } as ActionContext
|
||||
let assertions = 0
|
||||
function fixture() {
|
||||
function fixture(retiredSource?: ServerProbeMcpDependencies["retiredSource"]) {
|
||||
const events: string[] = [], receipts = new Map<string, ServerProbeReceipt>(), contexts: ServerProbeJournalContext[] = [], sealed = new Map<string, string>()
|
||||
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now
|
||||
let evidence: ChangeEvidence, result: unknown = null, stepState = "intent", operationState = "running", scope = [2, 9], revoked = false, sqlTamper: string | undefined, dispatchFailure: unknown, preparedTool: PreparedServerProbe["tool"] = "ping", mismatchDestination = false, nextCipher = 0, revokeOnQuery = false, expiredPrepared = false, clock = now, liveHistory = true
|
||||
const access: ServerProbeAuthority = { transport: "mcp", ownerId: "owner", connectionId, credentialId, expiresAt: now + 600000, generation: "generation", policyProof: "1".repeat(64), ownerProof: "2".repeat(64), active: true, operationAllowed: true, scopeServerIds: scope, ownerPermissions: ["mm:servers:write"], policyPermissions: ["mm:servers:write"] }
|
||||
const readAuthority = async () => { events.push("authority"); if (revoked) throw new ServerProbeExecutionError("PERMISSION_DENIED", "PRIVATE_PROVIDER_SECRET", 403); return { ...access, expiresAt: clock + 600000, scopeServerIds: [...scope] } }
|
||||
const assertProtections = async (ids: number[]) => { events.push("protections"); if (ids.some(id => !scope.includes(id))) throw new Error("out of scope") }
|
||||
@@ -28,6 +28,7 @@ function fixture() {
|
||||
read: async id => { events.push("journal.read"); return receipts.get(id) },
|
||||
}
|
||||
const deps: ServerProbeMcpDependencies = {
|
||||
retiredSource,
|
||||
authority: (() => ({ readAuthority, assertProtections })) as ServerProbeMcpDependencies["authority"],
|
||||
historicalAuthority: ((currentPrincipal: typeof principal) => ({ readAuthority: async () => { events.push("read-authority"); return { ...await readAuthority(), connectionId: currentPrincipal.connectionId, credentialId: currentPrincipal.credentialId } }, assertProtections })) as ServerProbeMcpDependencies["historicalAuthority"],
|
||||
history: dependencies => createServerProbeJournalHistoryReader({ ...dependencies, loadArchive: async binding => {
|
||||
@@ -51,7 +52,7 @@ function fixture() {
|
||||
journal: (ctx, dependencies, execution) => { assert.equal(execution?.deadlineAt, Date.parse(prepared().expiresAt)); contexts.push(ctx); return { ...journal, read: async (...args) => { await dependencies.assertAuthority(ctx, args[1]); return journal.read(...args) }, claim: async binding => { await dependencies.assertAuthority(ctx, binding.bindingProof); return journal.claim(binding) } } },
|
||||
query: async (sql, values) => {
|
||||
events.push("sql"); if (revokeOnQuery) revoked = true
|
||||
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] }
|
||||
if (sql.includes("probe-history:scope")) { assert.equal(values[2], connectionId); assert.equal(values[3], "owner"); return liveHistory ? [{ source_server_id: 9, destination_server_id: preparedTool === "bandwidth" ? 2 : null }] : [] }
|
||||
if (sql.includes("FROM servers")) { assert.ok(sql.includes("ANY($1::int[])")); assert.deepEqual(values[0], scope); return sql.includes("AND id=$2") ? [] : [{ id: 2, host: "peer", enabled: true }] }
|
||||
if (!sql.includes("mcp_operation_steps")) return [{ evidence: { sealed: await seal(JSON.stringify(evidence)) } }]
|
||||
assert.equal(values[0], connectionId); assert.equal(values[1], "owner"); assert.equal(values[2], "servers.probes.run")
|
||||
@@ -68,8 +69,9 @@ function fixture() {
|
||||
}, seal, open, proof, now: () => clock,
|
||||
}
|
||||
const adapter = createServerProbeMcpAdapter(deps)
|
||||
const retireHistory = () => { liveHistory = false }
|
||||
const prepare = async (tool: PreparedServerProbe["tool"] = "ping") => { evidence = await adapter.prepare({ serverId: 9, request: tool === "bandwidth" ? { tool, bwRemoteAddress: "peer", dstServerId: 8 } : { tool, target: "192.0.2.1" } }, context); const step = evidence.steps[0]; assert.equal(step.kind, "service"); return step.kind === "service" ? step.input : assert.fail() }
|
||||
return { adapter, events, receipts, contexts, prepare, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
|
||||
return { adapter, events, receipts, contexts, prepare, retireHistory, setResult: (value: unknown) => { result = value; stepState = "applied" }, finishOperation: () => { operationState = "applied"; stepState = "acknowledged" }, setTamper: (value?: string) => { sqlTamper = value }, revoke: () => { revoked = true }, setScope: (value: number[]) => { scope = value }, fail: (value: unknown) => { dispatchFailure = value }, revokeOnQuery: () => { revokeOnQuery = true }, evidence: () => evidence, mismatchDestination: () => { mismatchDestination = true }, expirePrepared: () => { expiredPrepared = true; clock = now + 3600000 } }
|
||||
}
|
||||
|
||||
for (const tool of ["ping", "traceroute", "bandwidth", "dns", "route", "mtu"] as const) {
|
||||
@@ -128,6 +130,38 @@ for (const input of [{ serverId: 9, request: { tool: "ping", target: "x", creden
|
||||
await assert.rejects(f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0, signal: controller.signal })); assert.equal(f.events.filter(event => event === "execute").length, attempted); assertions += 3
|
||||
}
|
||||
assert.ok(!("diagnosticProgress" in serverProbeSafeMcpError(new ServerProbeExecutionError("X", "secret", 500, "unknown", 99), { receiptId, operationId, stageBudget: 1 }))); assertions++
|
||||
{
|
||||
const hash = "3".repeat(64), date = (value: number) => new Date(value).toISOString().replace("Z", "000Z")
|
||||
const retired = { source: { version: 1 as const, operation: "servers.probes.run" as const, operationVersion: 1 as const, tool: "ping" as const,
|
||||
receiptId, executionId: operationId, connectionId, ownerId: "owner", planId: randomUUID(), originalCredentialId: credentialId,
|
||||
sourceServerId: 9, serverIds: [9], issuedAt: now - 1000, authorityExpiresAt: date(now + 600000), terminalAt: date(now + 600000),
|
||||
activityAt: date(now + 600000), gcNotBefore: date(now + 87060000), retentionDays: 1, safetyMs: 60000,
|
||||
bindingProof: hash, sourceDigest: hash, terminalDigest: hash, stageSnapshotDigest: hash, receiptProof: hash, ciphertextSha256: hash,
|
||||
}, payloadPurged: false }
|
||||
let lookups = 0, changeOnRecheck = false
|
||||
const f = fixture(async lookup => {
|
||||
lookups++
|
||||
assert.deepEqual(lookup, { receiptId, operationId, connectionId, ownerId: "owner", scopeServerIds: [2, 9] })
|
||||
return changeOnRecheck && lookups % 2 === 0 ? undefined : retired
|
||||
})
|
||||
const input = await f.prepare(), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 })
|
||||
f.setResult(output); f.finishOperation()
|
||||
const request = { operationId, receiptId, page: 0 }
|
||||
const live = await f.adapter.readPage(request, context)
|
||||
assert.equal(lookups, 0)
|
||||
f.retireHistory()
|
||||
const rotated = { ...context, principal: { ...principal, credentialId: randomUUID() } }
|
||||
assert.deepEqual(await f.adapter.readPage(request, rotated), live)
|
||||
assert.deepEqual(await f.adapter.receiptTarget(request, rotated), { scope: "servers", serverIds: [9] })
|
||||
const beforeForeign = lookups
|
||||
await assert.rejects(f.adapter.readPage(request, { ...rotated, connection: { ...context.connection, owner_id: "foreign" } }))
|
||||
assert.equal(lookups, beforeForeign)
|
||||
lookups = 0; changeOnRecheck = true
|
||||
const privateReads = f.events.filter(event => event === "history.read").length
|
||||
await assert.rejects(f.adapter.readPage(request, rotated))
|
||||
assert.equal(f.events.filter(event => event === "history.read").length, privateReads)
|
||||
assertions += 7
|
||||
}
|
||||
{
|
||||
const f = fixture(), input = await f.prepare("bandwidth"), output = await f.adapter.service.execute(input, { operationId, phase: "forward", stepIndex: 0 }); f.setResult(output); f.finishOperation(); f.expirePrepared()
|
||||
await assert.rejects(f.adapter.service.current(input))
|
||||
|
||||
@@ -10,6 +10,7 @@ import { serverProbeReceiptPage } from "../../services/server-probe-public-resul
|
||||
import { decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
|
||||
import type { ActionContext, ChangeEvidence } from "./action-types.js"
|
||||
import type { McpServiceHandler } from "./service-registry.js"
|
||||
import type { findRetiredServerProbeHistory, ServerProbeRetiredLookup } from "../../services/server-probe-retired-history.js"
|
||||
|
||||
/** Foundations only: importing this module registers no tools or services. */
|
||||
export const serverProbeActionSchema = z.object({ serverId: z.number().int().positive().safe(), request: serverProbeExecutionRequest }).strict()
|
||||
@@ -32,6 +33,7 @@ export interface ServerProbeMcpDependencies {
|
||||
executor: typeof createProductionServerProbeExecutor
|
||||
journal: (context: ServerProbeJournalContext, dependencies: ServerProbeJournalDependencies, execution?: { signal?: AbortSignal; deadlineAt?: number }) => ServerProbeJournal
|
||||
query: Query
|
||||
retiredSource?: (lookup: ServerProbeRetiredLookup) => ReturnType<typeof findRetiredServerProbeHistory>
|
||||
seal: (value: string) => Promise<string>
|
||||
open: (value: string) => Promise<string>
|
||||
proof: (value: unknown) => Promise<string>
|
||||
@@ -52,7 +54,7 @@ export async function openServerProbeMcpBinding(raw: Record<string, unknown>, cr
|
||||
return { input, binding }
|
||||
}
|
||||
const ids = (prepared: PreparedServerProbe) => [...new Set([prepared.serverId, ...(prepared.destinationServerId ? [prepared.destinationServerId] : [])])].sort((a, b) => a - b)
|
||||
function metadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
|
||||
export function serverProbeReceiptMetadata(receipt: ServerProbeReceipt, operationId: string, stageBudget: number) {
|
||||
const page = serverProbeReceiptPage(receipt, 0)
|
||||
if (!Number.isSafeInteger(receipt.attemptedStages) || !Number.isSafeInteger(receipt.completedStages) || receipt.completedStages < 0 || receipt.completedStages > receipt.attemptedStages || receipt.attemptedStages > stageBudget) return denied()
|
||||
return { operationId, receiptId: receipt.receiptId, tool: receipt.tool, status: receipt.state, httpStatus: receipt.httpStatus, stageBudget, attemptedStages: receipt.attemptedStages, completedStages: receipt.completedStages, sha256: page.sha256, totalPages: page.totalPages, bytes: page.bytes, automaticReplay: false, forwardingVerified: false }
|
||||
@@ -119,7 +121,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
await owned(input, binding, row.operation_id)
|
||||
await preflight(binding, signal)
|
||||
if (!receipt || receipt.receiptId !== input.receiptId || receipt.serverId !== binding.prepared.serverId || receipt.destinationServerId !== binding.prepared.destinationServerId || receipt.tool !== binding.prepared.tool) return denied()
|
||||
return { row, receipt, result: metadata(receipt, row.operation_id, binding.stageBudget) }
|
||||
return { row, receipt, result: serverProbeReceiptMetadata(receipt, row.operation_id, binding.stageBudget) }
|
||||
}
|
||||
const target = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
|
||||
const input = serverProbeActionSchema.parse(raw), access = deps.authority(context.principal, { signal: context.signal })
|
||||
@@ -177,7 +179,7 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
const { executor } = execution(input, binding, row, true, context.signal)
|
||||
const receipt = await executor.execute(binding.prepared, context.signal)
|
||||
await preflight(binding, context.signal); await owned(input, binding, context.operationId, true); await preflight(binding, context.signal)
|
||||
return metadata(receipt, context.operationId, binding.stageBudget)
|
||||
return serverProbeReceiptMetadata(receipt, context.operationId, binding.stageBudget)
|
||||
} catch (error) { throw serverProbeSafeMcpError(error, progress) }
|
||||
},
|
||||
verify: async (raw, result) => {
|
||||
@@ -200,19 +202,47 @@ export function createServerProbeMcpAdapter(overrides: Partial<ServerProbeMcpDep
|
||||
AND j.source_server_id=ANY($6::int[]) AND (j.destination_server_id IS NULL OR j.destination_server_id=ANY($6::int[])) LIMIT 2`,
|
||||
[request.receiptId, request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION, initial.scopeServerIds])
|
||||
await fresh()
|
||||
if (rows.length !== 1) return denied()
|
||||
const row = rows[0], serverIds = [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
|
||||
const loadRetired = deps.retiredSource
|
||||
const readRetired = rows.length === 0 && loadRetired ? () => loadRetired({
|
||||
receiptId: request.receiptId, operationId: request.operationId, connectionId: context.principal.connectionId,
|
||||
ownerId: initial.ownerId, scopeServerIds: initial.scopeServerIds,
|
||||
}) : undefined
|
||||
const retired = readRetired ? await readRetired() : undefined
|
||||
if (readRetired) await fresh()
|
||||
if (rows.length !== 1 && !retired) return denied()
|
||||
const row = rows[0]
|
||||
const serverIds = retired ? retired.source.serverIds : [...new Set([z.number().int().positive().safe().parse(row.source_server_id), ...(row.destination_server_id === null ? [] : [z.number().int().positive().safe().parse(row.destination_server_id)])])].sort((a, b) => a - b)
|
||||
const fence = async () => { const current = await fresh(); if (serverIds.some(id => !current.scopeServerIds.includes(id))) return denied(); await access.assertProtections(serverIds, current); await fresh() }
|
||||
await fence()
|
||||
return { initial, serverIds, fence }
|
||||
const retiredFence = retired && readRetired ? async () => {
|
||||
await fence()
|
||||
const current = await readRetired()
|
||||
await fence()
|
||||
if (!current || fingerprint(current) !== fingerprint(retired)) return denied()
|
||||
} : undefined
|
||||
return { initial, serverIds, fence, retired, retiredFence }
|
||||
}
|
||||
const receiptTarget = async (raw: Record<string, unknown>, context: ActionContext): Promise<ChangeEvidence["target"]> => {
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence } = await historicalAccess(request, context)
|
||||
await fence(); return { scope: "servers", serverIds }
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { serverIds, fence, retiredFence } = await historicalAccess(request, context)
|
||||
await (retiredFence ?? fence)(); return { scope: "servers", serverIds }
|
||||
}
|
||||
const readPage = async (raw: Record<string, unknown>, context: ActionContext) => {
|
||||
try {
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence } = await historicalAccess(request, context)
|
||||
const request = serverProbeReceiptReadSchema.parse(raw), { initial, serverIds, fence, retired, retiredFence } = await historicalAccess(request, context)
|
||||
if (retired && retiredFence) {
|
||||
const source = retired.source
|
||||
const originalContext: ServerProbeJournalContext = {
|
||||
identity: { kind: "mcp", ownerId: source.ownerId, executionId: source.executionId, connectionId: source.connectionId, credentialId: source.originalCredentialId,
|
||||
issuedAt: source.issuedAt, expiresAt: Date.parse(source.authorityExpiresAt), operationName: SERVER_PROBE_OPERATION },
|
||||
sourceServerId: source.sourceServerId, ...(source.destinationServerId !== undefined ? { destinationServerId: source.destinationServerId } : {}), tool: source.tool,
|
||||
}
|
||||
await retiredFence()
|
||||
const reader = deps.history({ assertReadAuthority: retiredFence }, { signal: context.signal })
|
||||
const receipt = await reader.read({ context: originalContext, receiptId: source.receiptId, bindingProof: source.bindingProof })
|
||||
const page = serverProbeReceiptPage(receipt, request.page)
|
||||
await retiredFence()
|
||||
return page
|
||||
}
|
||||
await fence()
|
||||
const rows = await deps.query(`SELECT p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id JOIN mcp_connections c ON c.id=o.connection_id WHERE o.id=$1 AND o.connection_id=$2 AND c.owner_id=$3 AND p.operation_id=$4 AND p.operation_version=1 AND octet_length(p.evidence::text)<=2097152 LIMIT 2`, [request.operationId, context.principal.connectionId, initial.ownerId, SERVER_PROBE_OPERATION])
|
||||
await fence(); if (rows.length !== 1) return denied()
|
||||
|
||||
@@ -171,6 +171,16 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
assert.equal(done.result.confirmedWrites, 0); assert.equal(done.result.completedServiceSteps, 1); assert.equal(done.result.verificationScope, "acknowledgement")
|
||||
const receipt = done.result.diagnosticReceipt as { receiptId: string; stageBudget: number; attemptedStages: number; completedStages: number; totalPages: number; sha256: string }
|
||||
assert(receipt); assert.equal(receipt.attemptedStages, receipt.completedStages); assert(receipt.attemptedStages > 0 && receipt.attemptedStages <= receipt.stageBudget)
|
||||
let captured = false
|
||||
for (const until = Date.now()+2000; Date.now()<until;) {
|
||||
const proof = (await pool.query(`SELECT f.source->>'terminalAt'=to_char(GREATEST(o.finished_at,p.expires_at) AT TIME ZONE 'UTC',
|
||||
'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS terminal_matches FROM server_probe_terminal_proofs f
|
||||
JOIN mcp_operations o ON o.id=f.operation_id JOIN mcp_plans p ON p.id=o.plan_id
|
||||
WHERE f.operation_id=$1 AND f.receipt_id=$2`, [done.id,receipt.receiptId])).rows[0]
|
||||
if (proof) { assert.equal(proof.terminal_matches,true); captured = true; break }
|
||||
await delay(20)
|
||||
}
|
||||
assert.equal(captured,true,"Owned successful diagnostic did not publish its terminal proof")
|
||||
const step = (await pool.query("SELECT state,result FROM mcp_operation_steps WHERE operation_id=$1", [done.id])).rows
|
||||
assert.equal(step.length, 1); assert.equal(step[0].state, "acknowledged")
|
||||
for (const field of ["output", "data", "contentB64", "bindingCiphertext"]) assert(!Object.hasOwn(step[0].result, field))
|
||||
@@ -201,6 +211,49 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
cases.push(tool + ": actual public plan/approval/apply, receipt metadata, full stage ledger and checksum pages without redispatch")
|
||||
}
|
||||
|
||||
// Force a genuine capture SQL failure after the real diagnostic has
|
||||
// committed. A sequence records the rejected attempt despite rollback.
|
||||
const captureFault = await fixture()
|
||||
const faultPlan = await prepare(captureFault, { tool: "route", target: "127.0.0.1" })
|
||||
await pool.query("CREATE SEQUENCE public.owned_probe_capture_failure")
|
||||
try {
|
||||
await pool.query(`CREATE FUNCTION public.owned_probe_capture_failure() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NEW.source->>'connectionId'=TG_ARGV[0] THEN
|
||||
PERFORM nextval('public.owned_probe_capture_failure');
|
||||
RAISE EXCEPTION 'owned terminal proof insertion refused';
|
||||
END IF;
|
||||
RETURN NEW;
|
||||
END $$`)
|
||||
try {
|
||||
assert.match(captureFault.principal.connectionId,/^[a-f0-9-]{36}$/)
|
||||
await pool.query(`CREATE TRIGGER owned_probe_capture_failure BEFORE INSERT ON public.server_probe_terminal_proofs
|
||||
FOR EACH ROW EXECUTE FUNCTION public.owned_probe_capture_failure('${captureFault.principal.connectionId}')`)
|
||||
try {
|
||||
const beforeFault = requests+dnsUdp+dnsTcp, faultKey=randomUUID()
|
||||
const applied = await applyPlan(captureFault.principal,faultPlan,faultKey), done=await terminal(applied.id)
|
||||
let attempted = false
|
||||
for (const until=Date.now()+2500;Date.now()<until;) {
|
||||
attempted = (await pool.query("SELECT is_called FROM public.owned_probe_capture_failure")).rows[0].is_called
|
||||
if (attempted) break
|
||||
await delay(20)
|
||||
}
|
||||
assert.equal(attempted,true,"Owned post-COMMIT capture insert must have reached the injected SQL failure")
|
||||
assert.equal(done.state,"applied"); assert.equal(done.error_code,null)
|
||||
assert.equal((await pool.query("SELECT 1 FROM server_probe_terminal_proofs WHERE operation_id=$1",[applied.id])).rows.length,0)
|
||||
const originalReceipt = done.result.diagnosticReceipt as { receiptId:string; sha256:string }
|
||||
const page = await invokeMcpTool(captureFault.principal,"mm_server_probe_receipt_get",
|
||||
{ operationId:applied.id,receiptId:originalReceipt.receiptId,page:0 }) as {sha256:string}
|
||||
assert.equal(page.sha256,originalReceipt.sha256)
|
||||
assert.equal((await applyPlan(captureFault.principal,faultPlan,faultKey)).id,applied.id)
|
||||
assert.equal((await terminal(applied.id)).state,"applied")
|
||||
assert.equal(requests+dnsUdp+dnsTcp,beforeFault+1)
|
||||
assert.equal((await pool.query("SELECT last_value::int AS attempts FROM public.owned_probe_capture_failure")).rows[0].attempts,1)
|
||||
} finally { await pool.query("DROP TRIGGER owned_probe_capture_failure ON public.server_probe_terminal_proofs") }
|
||||
} finally { await pool.query("DROP FUNCTION public.owned_probe_capture_failure()") }
|
||||
} finally { await pool.query("DROP SEQUENCE public.owned_probe_capture_failure") }
|
||||
cases.push("terminal proof SQL refusal: acknowledged result remains applied and readable, exactly one loopback measurement and no retry")
|
||||
|
||||
// Pause only the one runner's pending->running SQL BEFORE sending it. This
|
||||
// leaves a real pending approved source, allowing genuine durable cancel.
|
||||
// A server-table lock after 'running' would have no intent step yet and the
|
||||
@@ -209,8 +262,9 @@ export async function acceptServerProbeAllToolsSql() {
|
||||
const entered = gate(), resume = gate(); releasePending = resume.resolve
|
||||
let heldOperation = "", held = false
|
||||
pool.query = (async function (...args: unknown[]) {
|
||||
if (!held && typeof args[0] === "string" && args[0].startsWith("UPDATE mcp_operations SET state='running'")) {
|
||||
held = true; heldOperation = String((args[1] as unknown[])[0]); entered.resolve(); await resume.promise
|
||||
const query = args[0] as { text?: string; values?: unknown[] } | undefined
|
||||
if (!held && query?.text?.startsWith("/* mcp-operation-root:claim */") && typeof query.values?.[0] === "string") {
|
||||
held = true; heldOperation = query.values[0]; entered.resolve(); await resume.promise
|
||||
}
|
||||
return Reflect.apply(originalQuery, pool, args)
|
||||
}) as typeof pool.query
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import { createMcpServerProbeAuthority, createMcpServerProbeReadAuthority } from "../../services/server-probe-authority.js"
|
||||
import { createProductionServerProbeExecutor } from "../../services/server-probe-execution.js"
|
||||
import { createServerProbeStorageHistoryReader } from "../../services/server-probe-storage-history.js"
|
||||
import { findRetiredServerProbeHistory } from "../../services/server-probe-retired-history.js"
|
||||
import { createSqlServerProbeSourceReaders } from "../../services/server-probe-storage-source.js"
|
||||
import { createProbeStorageHostIo } from "../../services/server-probe-storage-host-io.js"
|
||||
import { assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js"
|
||||
@@ -28,6 +29,7 @@ export function createStorageServerProbeMcpAdapter(limits: ProbeStorageLimits =
|
||||
},
|
||||
}
|
||||
const adapter = createServerProbeMcpAdapter({ query: io.rows, journal: io.journal,
|
||||
retiredSource: lookup => findRetiredServerProbeHistory(io.rows, lookup),
|
||||
authority: (principal, options) => createMcpServerProbeAuthority(principal, options, authorityDependencies),
|
||||
historicalAuthority: (principal, options) => createMcpServerProbeReadAuthority(principal, options, authorityDependencies),
|
||||
history: (dependencies, execution) => createServerProbeStorageHistoryReader({ assertReadAuthority: dependencies.assertReadAuthority,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { randomUUID } from "node:crypto"
|
||||
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
|
||||
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
|
||||
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
|
||||
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
|
||||
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
|
||||
import { pool } from "../../db/index.js"
|
||||
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
||||
import { recoverMcpOperations, verifyStep } from "./runtime.js"
|
||||
import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js"
|
||||
import { audit, getMcpSettings } from "./store.js"
|
||||
import { accessGeneration, decryptSecret, redact } from "./security.js"
|
||||
import { tickRoutingProfiles } from "./routing-profiles.js"
|
||||
@@ -48,11 +51,22 @@ export async function detectMcpDrift() {
|
||||
}
|
||||
}
|
||||
|
||||
const mcpRetentionBatchSize = 1000
|
||||
|
||||
async function retainMcpData() {
|
||||
const settings = await getMcpSettings()
|
||||
await pool.query("DELETE FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')", [settings.requestsRetentionDays])
|
||||
await pool.query("DELETE FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)", [settings.auditRetentionDays])
|
||||
await pool.query("DELETE FROM mcp_artifacts WHERE expires_at<now()")
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')
|
||||
ORDER BY created_at,id LIMIT $2 FOR UPDATE
|
||||
) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [settings.requestsRetentionDays, mcpRetentionBatchSize])
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)
|
||||
ORDER BY created_at,id LIMIT $2 FOR UPDATE
|
||||
) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [settings.auditRetentionDays, mcpRetentionBatchSize])
|
||||
await pool.query(`WITH expired AS (
|
||||
SELECT id FROM mcp_artifacts WHERE expires_at<now()
|
||||
ORDER BY expires_at,id LIMIT $1 FOR UPDATE
|
||||
) DELETE FROM mcp_artifacts USING expired WHERE mcp_artifacts.id=expired.id`, [mcpRetentionBatchSize])
|
||||
await pool.query("DELETE FROM mcp_oauth_requests WHERE expires_at<now()-interval '1 day'")
|
||||
await pool.query("DELETE FROM mcp_budget_buckets WHERE bucket<now()-interval '2 days'")
|
||||
await routingReservationTransaction(db => retainManualInverseChargesWithinOwner(db, settings.requestsRetentionDays))
|
||||
@@ -79,8 +93,11 @@ export async function tickMcpWorkers() {
|
||||
await retainMcpData()
|
||||
await retainServerProbeStorage()
|
||||
await compactCompletedServerPollStorage()
|
||||
await retireCompletedServerPollOperations()
|
||||
await collectExpiredServerPollStorage()
|
||||
await retireOrdinaryMcpOperations()
|
||||
if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() }
|
||||
await maintainServerProbeOperationRetirement()
|
||||
} finally {
|
||||
if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true }
|
||||
lease.release(unsafe)
|
||||
|
||||
Reference in new issue
Block a user