Files
MikrotikManager/backend/src/modules/mcp/runtime.ts
T

895 lines
76 KiB
TypeScript

import { partitionWriteFailureDiagnostic, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js"
import { pollErrorLog } from "../../services/server-poll-error-provenance.js"
import { manualRoutingPredispatchDiagnostic } from "./routing-community-predispatch-diagnostic.js"
import { vrfPreparationDiagnostic } from "./vrf-preparation-diagnostic.js"
import { createVrfRuntimeDiagnosticBoundary } from "../../services/vrf-execution-diagnostic.js"
import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js"
import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js"
import { preparedPlanExpiry } from "./plan-expiry.js"
import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js"
import { randomUUID, createHash } from "node:crypto"
import { performance } from "node:perf_hooks"
import { readMcpTemporaryGrantOrigin,assertOriginalMcpTemporaryGrant,type OriginalServiceGrant } from "./temporary-grant-origin.js"
import type { PoolClient } from "pg"
import { mcpProtectedObjectSchema, mcpTemporaryGrantSchema, type McpPolicyPreviewRequest } from "@mmapp/contracts/mcp"
import { appUserCreateSchema, appUserUpdateSchema, userBindingCreateSchema } from "@mmapp/contracts/users"
import { logSettingsPatchSchema } from "@mmapp/contracts"
import { pool } from "../../db/index.js"
import { hasPermission, type AuthUser } from "../../lib/permissions.js"
import { createUser, updateUser, deleteUser, addBinding, removeBinding, getUserById } from "../users/service/users-service.js"
import { normalizeBindingPeer } from "../users/peer-bind.js"
import { getLogSettings, updateLogSettings } from "../../services/router-logs-collector.js"
import { refreshScheduler } from "../../services/scheduler.js"
import { withApplicationMutationLock, withConfigMutationLock } from "../../services/config-mutation-lock.js"
import { withOperationConfigurationLockClient, assertOperationConfigurationLockScope, assertOperationConfigurationLockCheckpoint } from "../../services/operation-config-lock-lease.js"
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
import { mcpDatabaseRequestSignal, withMcpDatabaseRequest, withMcpRunnerDatabaseRequest } from "./database-request-lease.js"
import { setTimeout as waitForRunnerAdmission } from "node:timers/promises"
import { withRuntimePollPrincipalSource, executeRuntimePollPolicyTransaction, type RuntimePollPolicyLoan } from "./runtime-poll-principal-boundary.js"
import { readPollRuntimePolicySource, type PollRuntimePolicySource } from "./poll-runtime-policy-source.js"
import { withVerifiedConfigurationChange, assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js"
import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOperationAdmission, withMcpOperationExecutionOrigin, stripMcpOperationExecutionPrivateResult, type McpOperationAdmission } from "./operation-execution-origin.js"
import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js"
import { evaluateMcpPolicy } from "./policy.js"
import { findAction, findActionByOperation } from "./actions.js"
import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js"
import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js"
import { audit, getConnection, transaction, requestAccountingWrite, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js"
import { executeRosStep, readRosSnapshot, routerClient, type RosSnapshot } from "./resources.js"
import { verifyHealthChecks } from "./routing-health-checks.js"
import { verifyRosStepSnapshots } from "./ros-step-verification.js"
import { createRosPrefix } from "./ros-prefix.js"
import { checkRpfPolicy } from "./routing-controls.js"
import type { ActionContext, ChangeEvidence, ChangeStep } from "./action-types.js"
import { logsFollowCancellation } from "./logs-follow-cancellation.js"
import { findMcpService, type McpServiceExecutionContext, type RoutingReservedNetworkSession } from "./service-registry.js"
import { withDiagnosticCancellation } from "./diagnostic-cancellation.js"
import { z } from "zod"
import { verifyManagementChannel } from "./management.js"
import { intersectGrant } from "./store.js"
import { changeMutationCost } from "./change-cost.js"
import { verifyRoutingObservationChecks } from "./routing-observations.js"
import { publicMcpChangeEvidence } from "./routing-community-public.js"
import { prepareRuntimeVrfTicket } from "./vrf-ticket-runtime.js"
import { executeVrfProductionStep, isVrfCompanionResource, VrfProductionAcknowledgedFailure } from "../../services/vrf-production-execution.js"
import { VrfProductionClaimOutcomeUnknown } from "../../services/vrf-production-ticket-store.js"
import { serviceNativeProgress } from "./service-native-progress.js"
import { serviceDiagnosticProgress, serviceDiagnosticReceipt } from "./service-diagnostic-progress.js"
import { admitManualInverseWithinOwner } from "./routing-manual-inverse-store.js"
import { bindRoutingChargeAdmission } from "./routing-reservation-transaction.js"
import { withRoutingReservationAdmission, createRoutingReservationStore, readHistoricalChargesWithinOwner, admitPersistentRoutingForwardWithinOwner, type RoutingReservedForwardAllocation } from "./routing-reservation-store.js"
import { lockRoutingReservationAdmissionSource, readScopedRoutingPathAdmissionDeadline, requiresScopedRoutingOwnerApproval, routingReservationEvidence } from "./routing-reservation-runtime.js"
import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js"
let activeRunners = 0
const runnerQueue: Array<() => void> = []
async function runBounded(work: () => Promise<void>) {
if (activeRunners >= 2) await new Promise<void>(resolve => runnerQueue.push(resolve))
else activeRunners++
try { await work() }
finally { const next = runnerQueue.shift(); if (next) next(); else activeRunners-- }
}
async function runOrdinaryOperation(work: () => Promise<void>): Promise<void> {
for (;;) {
await sharedDatabaseGate.whenOpen()
let local = false, entered = false
try {
return await runBounded(() => sharedDatabaseGate.request(() => {
local = true
return withMcpRunnerDatabaseRequest(() => {
entered = true
return work()
})
}))
} catch (error) {
if (entered || (local
? !(error instanceof McpError && ["PROTOCOL_STORAGE_BUSY", "RATE_LIMITED"].includes(error.code))
: !sharedDatabaseGate.isPaused())) throw error
await waitForRunnerAdmission(250)
}
}
}
async function policyInput(client: Pick<PoolClient, "query">, connection: ConnectionRow, operationId: string, target: ChangeEvidence["target"], evidence?: ChangeEvidence, source?: PollRuntimePolicySource): Promise<McpPolicyPreviewRequest> {
if (operationId === "operations.recover" && evidence?.snapshots.some(snapshot => ["routing_policy:routing_filter", "routing_policy:routing_filter_select"].includes(snapshot.resource)) &&
(!hasPermission(connection.policy.permissions, "mm:filters:read") || !hasPermission(connection.owner_permissions, "mm:filters:read"))) {
throw new McpError("PERMISSION_DENIED", "Для восстановления привязки OSPF требуется право чтения фильтров", 403)
}
const { rows: protections } = source ? { rows: source.protections } : await client.query("SELECT object_ref FROM mcp_protections LIMIT 1001")
if (protections.length > 1000) throw new McpError("STATE_UNAVAILABLE", "Число защит превышает предел проверки; изменение заблокировано", 503)
const { rows: [usage] } = source ? { rows: [source.usage] } : await client.query<{ calls: number; mutations: number; reads: number }>(`
SELECT COALESCE((SELECT used FROM mcp_budget_buckets WHERE connection_id=$1 AND kind='calls' AND bucket=date_trunc('minute',now())),0) AS calls,
COALESCE((SELECT used FROM mcp_budget_buckets WHERE connection_id=$1 AND kind='mutations' AND bucket=date_trunc('hour',now())),0) AS mutations,
(SELECT count(*)::int FROM mcp_read_leases WHERE connection_id=$1 AND expires_at>now()) AS reads`, [connection.id])
return {
operationId, policy: connection.policy, ownerPermissions: connection.owner_permissions, target,
objects: [...(evidence?.objects ?? []), ...(evidence?.requiresHuman && target.scope !== "servers" ? [{ kind: "mcp-operation", id: operationId }] : [])], objectCount: evidence ? changeMutationCost(evidence) + (evidence.recovery ? 1 : 0) : 0,
protections: [
...protections.map(row => mcpProtectedObjectSchema.parse(row.object_ref)),
...(evidence?.requiresHuman && target.scope === "servers" ? target.serverIds.map(serverId => ({ kind: "server", id: String(serverId), serverId, protection: "human-approval-required" as const })) : []),
...(evidence?.requiresHuman && target.scope !== "servers" ? [{ kind: "mcp-operation", id: operationId, protection: "human-approval-required" as const }] : []),
],
usage: { callsInMinute: Number(usage.calls), mutationsInHour: Number(usage.mutations), concurrentReads: Number(usage.reads) }, auditAvailable: true,
}
}
export async function readMcpPolicyPreviewInput(connection: ConnectionRow, input: Pick<McpPolicyPreviewRequest, "operationId" | "target" | "objects" | "objectCount">): Promise<McpPolicyPreviewRequest> {
const current = await policyInput(pool, connection, input.operationId, input.target)
return { ...current, objects: input.objects, objectCount: input.objectCount }
}
async function temporaryScope(client: Pick<PoolClient, "query">, connection: ConnectionRow, credentialId: string) {
const { rows } = await client.query("SELECT payload FROM mcp_grants WHERE connection_id=$1 AND revoked_at IS NULL AND payload->>'credentialId'=$2", [connection.id, credentialId])
if (!rows.length) return undefined
if (rows.length !== 1) throw new McpError("GRANT_INACTIVE", "Неоднозначный временный доступ")
const { credentialId: _credential, ...grant } = rows[0].payload
return mcpTemporaryGrantSchema.parse(grant)
}
async function reserveCall(principal: McpPrincipal, toolName: string, input: Record<string, unknown>, signal?: AbortSignal) {
const action = findAction(toolName), requestId = randomUUID()
const reservation = await requestAccountingWrite(async client => {
await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId])
const connection = await validatePrincipal(principal, client)
const grant = await temporaryScope(client, connection, principal.credentialId)
const context: ActionContext = { principal, connection: intersectGrant(connection, grant), signal }
const evidence = await policyInput(client, connection, action.operation.id, await action.target(input, { ...context, transactionClient: client }))
evidence.temporaryGrant = grant
const result = evaluateMcpPolicy(evidence)
await client.query("INSERT INTO mcp_requests(id,connection_id,actor_id,tool,server_ids,status,input,error_code) VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8)", [requestId, connection.id, connection.owner_id, toolName, evidence.target.scope === "servers" ? evidence.target.serverIds : [], result.decision === "denied" ? "denied" : "running", JSON.stringify(redact(input)), result.decision === "denied" ? result.reasons[0]?.code : null])
if (result.decision !== "denied") {
await client.query("INSERT INTO mcp_budget_buckets(connection_id,kind,bucket,used) VALUES($1,'calls',date_trunc('minute',now()),1) ON CONFLICT(connection_id,kind,bucket) DO UPDATE SET used=mcp_budget_buckets.used+1", [connection.id])
if (action.operation.effect === "read") await client.query("INSERT INTO mcp_read_leases(id,connection_id,expires_at) VALUES($1,$2,now()+interval '2 minutes')", [requestId, connection.id])
await client.query("UPDATE mcp_connections SET last_used_at=now() WHERE id=$1", [connection.id])
}
return { context, result }
})
if (reservation.result.decision === "denied") throw new McpError(reservation.result.reasons[0]?.code ?? "PERMISSION_DENIED", reservation.result.reasons.map(reason => reason.message).join("; "))
return { action, requestId, context: reservation.context }
}
async function completeRequest(id: string, started: number, result: unknown, error?: unknown, assertDeliveryAccess?: (client: PoolClient) => Promise<void>) {
await requestAccountingWrite(async client => {
await client.query("UPDATE mcp_requests SET status=$2,result=$3::jsonb,error_code=$4,duration_ms=$5,finished_at=now() WHERE id=$1", [id, error ? "error" : "completed", JSON.stringify(redact(error ? { message: error instanceof McpError ? error.message : "Внутренняя ошибка операции" } : result)), error instanceof McpError ? error.code : error ? "INTERNAL_ERROR" : null, Math.min(2147483647, Date.now() - started)])
await client.query("DELETE FROM mcp_read_leases WHERE id=$1", [id])
await assertDeliveryAccess?.(client)
})
}
type InternalMcpCall = { persistentRoutingProfile: NonNullable<ActionContext["persistentRoutingProfile"]> }
| { captureRoutingConfirmation: NonNullable<ActionContext["captureRoutingConfirmation"]> }
export async function invokeMcpTool(principal: McpPrincipal, toolName: string, raw: unknown, signal?: AbortSignal, internal?: InternalMcpCall): Promise<unknown> {
if (toolName === "mm_logs_follow") {
const maintenance = new AbortController()
return sharedDatabaseGate.request(
() => withMcpDatabaseRequest(() => invokeMcpToolInner(principal, toolName, raw, signal, internal, maintenance.signal)),
() => maintenance.abort(new McpError("RESET_REQUIRED", "База данных перешла в режим обслуживания; начните новое чтение", 409)),
)
}
return sharedDatabaseGate.request(() => withMcpDatabaseRequest(() => invokeMcpToolInner(principal, toolName, raw, signal, internal)))
}
async function invokeMcpToolInner(principal: McpPrincipal, toolName: string, raw: unknown, signal?: AbortSignal, internal?: InternalMcpCall, logsFollowMaintenance?: AbortSignal): Promise<unknown> {
const action = findAction(toolName), input = action.schema.parse(raw) as Record<string, unknown>, started = Date.now()
const leaseSignal = mcpDatabaseRequestSignal()
leaseSignal?.throwIfAborted()
const actionSignal = leaseSignal && signal ? AbortSignal.any([leaseSignal, signal]) : leaseSignal ?? signal
const call = await reserveCall(principal, toolName, input, actionSignal)
const cancelled = (): McpError | undefined => leaseSignal?.aborted ? leaseSignal.reason : logsFollowMaintenance && logsFollowCancellation(logsFollowMaintenance, signal)
const assertActive = () => { const failure = cancelled(); if (failure) throw failure }
let vrfPreparing = false
let routingConfirmation: Parameters<NonNullable<ActionContext["captureRoutingConfirmation"]>>[0] | undefined
try {
assertActive()
if (logsFollowMaintenance) call.context.logsFollowMaintenance = leaseSignal ? AbortSignal.any([logsFollowMaintenance, leaseSignal]) : logsFollowMaintenance
if (internal && "persistentRoutingProfile" in internal) {
if (action.operation.id !== "routing.community.switch") throw new McpError("INVALID_INPUT", "Контекст профиля не относится к этой операции", 400)
call.context.persistentRoutingProfile = z.object({ id: z.string().uuid(), version: z.number().int().positive().safe() }).strict().parse(internal.persistentRoutingProfile)
}
if (internal && "captureRoutingConfirmation" in internal) {
if (action.operation.id !== "routing.analyze") throw new McpError("INVALID_INPUT", "Измерения контроллера не относятся к этой операции", 400)
call.context.captureRoutingConfirmation = observations => { routingConfirmation = observations }
}
let result: unknown
if (action.read) result = await action.read(input, call.context)
else if (action.prepare) {
vrfPreparing = ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete"].includes(action.operation.id)
const evidence = await action.prepare(input, call.context)
result = await transaction(async client => {
assertActive()
await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId])
const connection = await validatePrincipal(principal, client)
const check = await policyInput(client, connection, action.operation.id, evidence.target, evidence)
check.temporaryGrant = await temporaryScope(client, connection, principal.credentialId)
// The call already reserved one call slot. Preparation does not reserve a mutation.
check.usage.callsInMinute = Math.max(0, check.usage.callsInMinute - 1)
let decision = evaluateMcpPolicy(check)
if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; "))
const id = randomUUID(), expiresAt = preparedPlanExpiry(action.operation.id, action.operation.section, evidence)
if (!expiresAt) throw new McpError("PLAN_EXPIRED", "Срок исходных данных плана истёк или недоступен; подготовьте новый план", 409)
const generation = await accessGeneration()
await client.query("INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at) VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10)", [id, connection.id, action.operation.id, action.operation.version, connection.policy.version, generation, JSON.stringify({ sealed: await encryptSecret(JSON.stringify(input)) }), JSON.stringify({ sealed: await encryptSecret(JSON.stringify(evidence)) }), decision.decision === "approval-required" ? "awaiting-approval" : "draft", expiresAt])
const vrfTicket = await prepareRuntimeVrfTicket({ operationId: action.operation.id, operationVersion: action.operation.version, planId: id, connectionId: connection.id, ownerId: connection.owner_id, evidence, sql: client,
assertAuthority: async () => {
if (signal?.aborted || generation !== await accessGeneration() || expiresAt.getTime() <= Date.now()) throw new McpError("PLAN_EXPIRED", "Подготовка VRF отменена или устарела", 409)
const current = await validatePrincipal(principal, client)
if (current.owner_id !== connection.owner_id || fingerprint(current.policy) !== fingerprint(connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(connection.owner_permissions)) throw new McpError("STATE_CONFLICT", "Доступ изменился во время подготовки VRF", 409)
const fresh = await policyInput(client, current, action.operation.id, evidence.target, evidence)
fresh.temporaryGrant = await temporaryScope(client, current, principal.credentialId)
fresh.usage.callsInMinute = Math.max(0, fresh.usage.callsInMinute - 1)
const reviewed = evaluateMcpPolicy(fresh)
if (reviewed.decision === "denied") throw new McpError(reviewed.reasons[0].code, reviewed.reasons.map(reason => reason.message).join("; "))
if (reviewed.decision !== decision.decision) throw new McpError("STATE_CONFLICT", "Политика подготовки VRF изменилась", 409)
},
assertCurrentSource: async (serverId, proof) => {
const { server } = await routerClient(serverId)
const current = await privateSecretFingerprint(fingerprint({ id: server.id, enabled: server.enabled, host: server.host, port: server.port, useSsl: server.useSsl, verifySsl: server.verifySsl, username: server.username, password: server.password, epoch: server.routingEndpointEpoch }), "mcp-ros-snapshot-source-v3")
if (!server.enabled || current !== proof) throw new McpError("STATE_CONFLICT", "Источник VRF изменился", 409)
},
})
if (vrfTicket) {
// Admission closes physical/VLAN parents and implicit native objects.
// Recheck the existing object policy/budget before publishing the
// final encrypted plan; the whole transaction rolls back on refusal.
const finalConnection = await validatePrincipal(principal, client)
const finalCheck = await policyInput(client, finalConnection, action.operation.id, evidence.target, evidence)
finalCheck.temporaryGrant = await temporaryScope(client, finalConnection, principal.credentialId)
finalCheck.usage.callsInMinute = Math.max(0, finalCheck.usage.callsInMinute - 1)
decision = evaluateMcpPolicy(finalCheck)
if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; "))
await client.query("UPDATE mcp_plans SET evidence=$2::jsonb,state=$3 WHERE id=$1", [id, JSON.stringify({ sealed: await encryptSecret(JSON.stringify(evidence)) }), decision.decision === "approval-required" ? "awaiting-approval" : "draft"])
}
await audit(client, connection.owner_id, "plan.create", connection.id, { planId: id, operationId: action.operation.id, decision })
assertActive()
return { planId: id, expiresAt: expiresAt.toISOString(), state: decision.decision === "approval-required" ? "awaiting-approval" : "draft", decision, changes: redact(publicMcpChangeEvidence(action.operation.id, evidence)), ...(vrfTicket ? { vrfTicket } : {}), executionAvailable: true }
})
vrfPreparing = false
} else throw new McpError("UNSUPPORTED_CAPABILITY", "Нет обработчика операции", 409)
assertActive()
const assertCurrentResultAccess = async (client?: PoolClient) => {
const current = await effectiveMcpConnection(principal, client)
assertActive()
if (fingerprint(current.policy) !== fingerprint(call.context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(call.context.connection.owner_permissions)) throw new McpError("STATE_CONFLICT", "Доступ изменился во время вызова; результат не выдан", 409)
}
await assertCurrentResultAccess()
const output = redact(result)
const checkDelivery = action.operation.section === "logs" && action.operation.effect === "read" ? assertCurrentResultAccess : leaseSignal ? async () => leaseSignal.throwIfAborted() : undefined
await completeRequest(call.requestId, started, output, undefined, checkDelivery)
await checkDelivery?.()
assertActive()
if (routingConfirmation && internal && "captureRoutingConfirmation" in internal) internal.captureRoutingConfirmation(routingConfirmation)
return output
} catch (error) {
if (vrfPreparing) {
try {
const diagnostic = vrfPreparationDiagnostic(call.requestId, action.operation.id, error)
if (diagnostic) console.error(JSON.stringify(diagnostic))
} catch { /* Diagnostics must never replace the original failure. */ }
}
const failure = cancelled() ?? error
await completeRequest(call.requestId, started, null, failure)
throw failure
}
}
export async function loadPlan(id: string, client: Pick<PoolClient, "query"> = pool): Promise<StoredPlan> {
const { rows: [plan] } = await client.query<StoredPlan>("SELECT * FROM mcp_plans WHERE id=$1", [id])
if (!plan) throw new McpError("NOT_FOUND", "План не найден", 404)
return plan
}
export async function publicPlan(plan: StoredPlan) {
const evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence
return { id: plan.id, connectionId: plan.connection_id, operationId: plan.operation_id, state: plan.state,
expiresAt: plan.expires_at.toISOString(), createdAt: plan.created_at.toISOString(), approvedBy: plan.approved_by,
changes: redact(publicMcpChangeEvidence(plan.operation_id, evidence)), input: redact(JSON.parse(await decryptSecret(plan.payload.sealed))) }
}
export async function approvePlan(id: string, actor: AuthUser, approve: boolean) {
return transaction(async client => {
await client.query("SELECT id FROM mcp_plans WHERE id=$1 FOR UPDATE", [id])
const plan = await loadPlan(id, client), connection = await getConnection(plan.connection_id, client)
assertConnectionVisible(connection, actor)
if (!["awaiting-approval", "draft"].includes(plan.state) || plan.expires_at.getTime() <= Date.now() || plan.generation !== await accessGeneration()) throw new McpError("PLAN_EXPIRED", "План устарел или уже запущен", 409)
const action = findActionByOperation(plan.operation_id)
if (!hasPermission(actor.permissions, "mm:mcp:approve") || action.operation.requiredPermissions.some(permission => !hasPermission(actor.permissions, permission))) throw new McpError("PERMISSION_DENIED", "Нет прав на подтверждение действия")
if (approve && actor.id !== connection.owner_id && plan.operation_id === "routing.community.switch" &&
requiresScopedRoutingOwnerApproval(JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence)) {
throw new McpError("PERMISSION_DENIED", "Переключение постоянного профиля подтверждает владелец подключения", 403)
}
await client.query("UPDATE mcp_plans SET state=$2,approved_by=$3,approved_at=now() WHERE id=$1", [id, approve ? "approved" : "cancelled", actor.id])
await audit(client, actor.id, approve ? "plan.approve" : "plan.reject", connection.id, { planId: id })
})
}
async function serviceCurrent(service: string, input: Record<string, unknown>): Promise<unknown> {
const custom = findMcpService(service)
if (custom) return custom.current(input)
if (service === "logs.settings.update") return getLogSettings()
if (service.startsWith("users.")) return input.id ? getUserById(String(input.id)) : null
throw new McpError("UNSUPPORTED_CAPABILITY", "Неизвестный сервис", 409)
}
async function serviceExecute(service: string, input: Record<string, unknown>, context: McpServiceExecutionContext): Promise<unknown> {
const custom = findMcpService(service)
if (custom) {
if (!custom.cancellableDiagnostic) return custom.execute(input, context)
let entered = false
try {
return await withDiagnosticCancellation(context, signal => {
entered = true
return custom.execute(input, context.phase === "forward" ? { ...context, signal } : context)
})
} catch (error) {
if (!entered && context.phase === "forward" && error instanceof McpError && error.code === "READ_CANCELLED" && custom.diagnosticIdentity) {
// Only the watcher knows execute was never entered. The handler still
// proves the exact encrypted plan/operation before supplying identity.
try {
const identity = await custom.diagnosticIdentity(input, context)
const progress = serviceDiagnosticProgress(Object.assign(new McpError("READ_CANCELLED", "Диагностика отменена до начала"), {
diagnosticProgress: { ...identity, operationId: context.operationId, state: "cancelled", attemptedStages: 0, completedStages: 0, automaticReplay: false },
}), true, context.operationId, identity.stageBudget)
if (progress) Object.assign(error, { diagnosticNotStarted: true, diagnosticProgress: progress })
} catch { /* Failure to prove ownership keeps the conservative unknown. */ }
}
throw error
}
}
if (service === "users.create") return createUser(appUserCreateSchema.parse(input.user))
if (service === "users.update") return updateUser(String(input.id), appUserUpdateSchema.parse(input.user))
if (service === "users.delete") return deleteUser(String(input.id))
if (service === "users.binding_add") return addBinding(String(input.id), userBindingCreateSchema.parse(input.binding), { assertCurrent: assertVerifiedConfigurationAccess })
if (service === "users.binding_remove") return removeBinding(String(input.id), String(input.bindingId))
if (service === "logs.settings.update") { const result = await updateLogSettings(logSettingsPatchSchema.parse(input.settings)); await refreshScheduler(); return result }
throw new McpError("UNSUPPORTED_CAPABILITY", "Неизвестный сервис", 409)
}
export async function verifyStep(step: ChangeStep, result: unknown, before?: RosSnapshot, after?: RosSnapshot): Promise<boolean> {
if (step.kind === "service") {
const custom = findMcpService(step.service)
if (custom?.verify) return custom.verify(step.input, result)
if (step.service === "users.delete") return !await getUserById(String(step.input.id))
if (step.service === "users.create" || step.service === "users.update") {
const id = step.service === "users.create" && result && typeof result === "object" ? String((result as { id?: string }).id ?? "") : String(step.input.id)
const current = await getUserById(id)
if (!current) return false
const expected = step.service === "users.create" ? appUserCreateSchema.parse(step.input.user) : appUserUpdateSchema.parse(step.input.user)
return Object.entries(expected).every(([key, value]) => {
if (key === "avatar" && typeof value === "string" && !value.trim()) return true // Existing service derives initials or retains the prior avatar.
const normalized = ["name", "login", "email", "avatar"].includes(key) && typeof value === "string" ? value.trim() : value
return fingerprint((current as unknown as Record<string, unknown>)[key]) === fingerprint(normalized)
})
}
if (step.service === "users.binding_remove") { const current = await getUserById(String(step.input.id)); return !!current && !current.bindings.some(binding => binding.id === step.input.bindingId) }
if (step.service === "users.binding_add") {
const binding = userBindingCreateSchema.parse(step.input.binding), current = await getUserById(String(step.input.id))
const createdId = result && typeof result === "object" ? (result as { id?: unknown }).id : undefined
return typeof createdId === "string" && !!current?.bindings.some(item => item.id === createdId
&& item.serverId === binding.serverId && item.interfaceName === binding.interfaceName.trim()
&& (binding.interfaceType === undefined || item.interfaceType === binding.interfaceType)
&& item.peerPublicKey === normalizeBindingPeer(item.interfaceType, binding.peerPublicKey))
}
if (step.service === "logs.settings.update") {
const current = await getLogSettings() as unknown as Record<string, unknown>
return Object.entries(logSettingsPatchSchema.parse(step.input.settings)).every(([key, value]) => current[key] === value)
}
// Service methods confirm their own DB changes; the receipt remains inspectable.
return result !== undefined || step.service === "users.binding_remove"
}
assertNoCallerCreateId(step.step)
const id = step.step.action === "create" ? acknowledgedCreateId(result) : step.step.id
if (step.step.action === "create" && !id) return false
if (!before) return false
return verifyRosStepSnapshots(step.step, result, before, after ?? await readRosSnapshot(step.step.serverId, step.step.resource))
}
export async function applyPlan(principal: McpPrincipal, planId: string, idempotencyKey: string): Promise<StoredOperation> {
return withMcpDatabaseRequest(() => admitPlan(principal, planId, idempotencyKey))
}
async function admitPlan(principal: McpPrincipal, planId: string, idempotencyKey: string): Promise<StoredOperation> {
const operationAdmissionStart = beginMcpOperationAdmission(principal)
let reservedRouting = false
let originalServiceGrant:OriginalServiceGrant|undefined
let operationAdmission: McpOperationAdmission | undefined
const admitted = await withRoutingReservationAdmission(async (client, reserveRouting) => {
await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId])
const connection = await validatePrincipal(principal, client)
const { rows: [existing] } = await client.query<StoredOperation>("SELECT * FROM mcp_operations WHERE connection_id=$1 AND (idempotency_key=$2 OR plan_id=$3) ORDER BY created_at LIMIT 1", [connection.id, idempotencyKey, planId])
if (existing) {
if (existing.plan_id !== planId) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409)
const original = await loadPlan(planId, client)
const originalEvidence=JSON.parse(await decryptSecret(original.evidence.sealed)) as ChangeEvidence
reservedRouting = !!await routingReservationEvidence(original, originalEvidence)
return existing
}
await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey)
await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))")
const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')")
if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429)
const selected = await lockRoutingReservationAdmissionSource(client, await loadPlan(planId, client), principal)
const { plan, evidence } = selected
reservedRouting = selected.reserved
if (plan.connection_id !== connection.id) throw new McpError("NOT_FOUND", "План не найден", 404)
const action = findActionByOperation(plan.operation_id)
if (plan.policy_version !== connection.policy.version || plan.operation_version !== action.operation.version || plan.generation !== await accessGeneration() || plan.expires_at.getTime() <= Date.now()) throw new McpError("PLAN_EXPIRED", "План или его разрешения устарели", 409)
if (!["draft", "approved"].includes(plan.state)) throw new McpError("APPROVAL_REQUIRED", "План требует подтверждения или недоступен", 409)
for (const change of evidence.steps) {
if (change.kind === "ros" && change.step.resource === "ip_settings" && change.step.fields["rp-filter"] === "strict") {
checkRpfPolicy(change.step.serverId, evidence.snapshots, "strict")
}
}
let scopedPathExpiresAt: number | undefined
if (plan.operation_id === "routing.community.switch" && requiresScopedRoutingOwnerApproval(evidence)) {
if (plan.approved_by !== connection.owner_id) throw new McpError("APPROVAL_REQUIRED", "Переключение постоянного профиля подтверждает владелец подключения", 409)
scopedPathExpiresAt = await readScopedRoutingPathAdmissionDeadline(client, evidence, connection.id)
}
if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) {
throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409)
}
if (evidence.recovery) {
if (Date.now() >= Date.parse(evidence.recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Срок зарезервированного восстановления истёк", 409)
const { rows: [grant] } = await client.query("SELECT config FROM mcp_routing_profiles WHERE id=$1 AND connection_id=$2 AND credential_id=$3 AND enabled AND grant_expires_at>now() AND policy_version=$4 AND generation=$5", [evidence.recovery.authorityProfileId, connection.id, principal.credentialId, connection.policy.version, await accessGeneration()])
const step = evidence.recovery.step
if (!grant?.config?.autoRecovery || step.kind !== "service" || fingerprint(grant.config) !== fingerprint(step.input.profile)) throw new McpError("PERMISSION_DENIED", "Допуск восстановления изменён или отозван")
}
const input = await policyInput(client, connection, action.operation.id, evidence.target, evidence)
input.temporaryGrant = await temporaryScope(client, connection, principal.credentialId)
// tools/call already reserved its slot. A full minute bucket must not reject
// the very call that legitimately consumed the last slot.
input.usage.callsInMinute = Math.max(0, input.usage.callsInMinute - 1)
const decision = evaluateMcpPolicy(input)
if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; "))
if (decision.decision === "approval-required" && plan.state !== "approved") throw new McpError("APPROVAL_REQUIRED", "Требуется подтверждение человеком", 409)
// Whole-operation provenance pins the ORIGINAL admission grant for ROS and
// service plans alike; a later runner must never adopt a replacement grant.
{
const grantWall=Date.now(),grantMonotonic=performance.now()
const grant=await readMcpTemporaryGrantOrigin(client,connection.id,principal.credentialId)
const admissionExpiresAt = Math.min(plan.expires_at.getTime(), evidence.validUntil ? Date.parse(evidence.validUntil) : Infinity,
evidence.recovery ? Date.parse(evidence.recovery.validUntil) : Infinity, Date.parse(principal.credentialExpiresAt),
Date.parse(connection.policy.expiresAt), connection.owner_valid_until.getTime(), grant.expiresAt ?? Infinity, scopedPathExpiresAt ?? Infinity)
originalServiceGrant=Object.freeze({...grant,monotonicEnd:grant.expiresAt===null?Infinity:grantMonotonic+Math.max(0,grant.expiresAt-grantWall),admissionExpiresAt,admissionMonotonicEnd:grantMonotonic+Math.max(0,admissionExpiresAt-grantWall)})
await assertOriginalMcpTemporaryGrant(client,connection.id,principal.credentialId,originalServiceGrant)
}
const mutationCost = reservedRouting ? 2 : changeMutationCost(evidence) + (evidence.recovery ? 1 : 0)
if (input.usage.mutationsInHour + mutationCost > connection.policy.limits.mutationsPerHour) throw new McpError("BUDGET_EXCEEDED", "Недостаточно бюджета для всех шагов плана", 429)
if (!reservedRouting) await client.query("INSERT INTO mcp_budget_buckets(connection_id,kind,bucket,used) VALUES($1,'mutations',date_trunc('hour',now()),$2) ON CONFLICT(connection_id,kind,bucket) DO UPDATE SET used=mcp_budget_buckets.used+EXCLUDED.used", [connection.id, mutationCost])
const inserted = await insertMcpOperationAdmission(client, operationAdmissionStart, planId, idempotencyKey, originalServiceGrant)
const row = inserted.operation, id = row.id
operationAdmission = inserted.admission; originalServiceGrant = inserted.originalGrant
if (plan.operation_id === "routing.community.switch" || plan.operation_id === "operations.recover") {
bindRoutingChargeAdmission(client, connection.id, principal.credentialId, inserted.originalGrant)
}
await client.query("UPDATE mcp_plans SET state='executing' WHERE id=$1", [planId])
if (reservedRouting) await reserveRouting(id)
else if (plan.operation_id === "routing.community.switch") await admitPersistentRoutingForwardWithinOwner(client, id)
else if (plan.operation_id === "operations.recover") await admitManualInverseWithinOwner(client, id, plan, evidence, principal,
(profileId, current, observedAt, deadlineAt) => readHistoricalChargesWithinOwner(client, profileId, current, observedAt, deadlineAt))
await audit(client, connection.owner_id, "operation.start", connection.id, { operationId: id, planId })
return row
}).catch(async error => { await retireMcpOperationAdmission(operationAdmission); throw error })
const operation = admitted.value
// Reserved native dispatch remains closed until the common engine's bounded
// network verifier and complete positive/negative lifecycle are accepted.
// The atomic hold is durable; neither read nor another apply mints its cap.
if (operation.state === "pending" && !reservedRouting && originalServiceGrant) {
const operationKind = (await loadPlan(operation.plan_id)).operation_id
const terminal = operationKind === "database.restore"
setImmediate(() => { void (terminal || operationKind === "database.backup" ? runBounded(() => terminal
? import("./database-restore.js").then(module => module.executeMcpDatabaseRestore(operation.id, principal,originalServiceGrant))
: import("../../services/system-database-backup.js").then(module => module.withSystemDatabaseBackupAdmission(() => executeOperation(operation.id, principal,undefined,undefined,originalServiceGrant,operationAdmission))))
: runOrdinaryOperation(() => executeOperation(operation.id, principal, admitted.reservation?.forwardAllocation, admitted.reservation?.reservationId,originalServiceGrant,operationAdmission))).catch(async () => { if (!terminal) await retireMcpOperationAdmission(operationAdmission) /* no retry; interrupted intents retain explicit UNKNOWN */ }) })
}
return { ...operation, result: stripMcpOperationExecutionPrivateResult(operation.result) }
}
async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise<void> {
return withMcpOperationExecutionOrigin(operationAdmission, async () => {
const lease = await pool.connect()
let terminalProbe: Awaited<ReturnType<typeof executeOperationWithLease>>
let locked = false, released = false
const ownerLifetime = new AbortController()
// This closure is the SINGLE owner of physical destruction/release.
const retire = () => {
if (released) return
released = true
ownerLifetime.abort()
try { lease.release(true) } catch { /* Never reuse an uncertain lock session. */ }
}
const onError = () => retire()
lease.on("error", onError)
try {
try {
locked = (await lease.query<{ locked: boolean }>("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + operationId])).rows[0].locked
} catch (error) {
// The backend may hold the lock despite a rejected/lost acquire ACK.
// An error event is not required to make healthy reuse unsafe.
retire()
throw error
}
if (released || !locked) return
terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal,
() => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant), mcpDatabaseRequestSignal())
} finally {
if (!released && locked) {
try { await lease.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", ["mcp-operation:" + operationId]) }
catch { retire() }
}
if (!released) {
// Keep the absorber on a destroyed client; remove only before a healthy
// release after EVERY borrowed lock facade/query has settled.
lease.removeListener("error", onError)
released = true
lease.release()
}
}
// Capture has its own bounded checkout and may fail without rewriting the
// acknowledged terminal outcome or redispatching the diagnostic.
if (terminalProbe) {
try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ }
}
})
}
async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) {
const { rows: [operation] } = await pool.query<StoredOperation>("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId])
if (!operation) return
const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence
const serverIds = evidence.target.scope === "servers" ? evidence.target.serverIds : evidence.objects.flatMap(object => object.serverId ? [object.serverId] : [])
let attempted = false, confirmed = 0, activeServiceStep = false
let ecmpStoppedBeforeDispatch = false
let vrfDiagnosticFallback: ReturnType<typeof createVrfRuntimeDiagnosticBoundary> | undefined
let reservedForwardPassed = false
let reservedRecoverySummary: Record<string, unknown> | undefined
const diagnosticOnly = evidence.steps.length === 1 && evidence.steps.every(step => step.kind === "service" && findMcpService(step.service)?.cancellableDiagnostic === true)
const productionVrf = evidence.vrfProductionAdmission === true
let diagnosticReceipt: ReturnType<typeof serviceDiagnosticReceipt>
const authorizeGrant = async () => {
assertOperationConfigurationLockScope()
if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась; подготовьте новый план", 409)
const connection = await validatePrincipal(principal)
assertOperationConfigurationLockScope()
if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась во время операции", 409)
return authorizeAfterPrincipal(connection)
}
const authorizeAfterPrincipal = async (connection: ConnectionRow, policyLoan?: RuntimePollPolicyLoan) => {
assertOperationConfigurationLockScope()
if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась; подготовьте новый план", 409)
if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась во время операции", 409)
const checkPolicy = async (client: Pick<PoolClient, "query">) => {
assertOperationConfigurationLockScope()
if(evidence.steps.some(step=>step.kind==="service")){
await assertOriginalMcpTemporaryGrant(client,connection.id,principal.credentialId,originalServiceGrant)
assertOperationConfigurationLockScope()
}
const pollSource = plan.operation_id === "servers.poll"
? await readPollRuntimePolicySource(client, connection.id, principal.credentialId) : undefined
assertOperationConfigurationLockScope()
const policy = await policyInput(client, connection, plan.operation_id, evidence.target, evidence, pollSource)
assertOperationConfigurationLockScope()
policy.temporaryGrant = pollSource ? pollSource.temporaryGrant : await temporaryScope(client, connection, principal.credentialId)
assertOperationConfigurationLockScope()
// Budget was reserved for the complete batch at admission.
policy.usage.callsInMinute = 0; policy.usage.mutationsInHour = 0
const decision = evaluateMcpPolicy(policy)
if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; "))
if (decision.decision === "approval-required" && !plan.approved_by) throw new McpError("APPROVAL_REQUIRED", "Изменение теперь требует подтверждения", 409)
}
if (policyLoan) await executeRuntimePollPolicyTransaction(policyLoan, checkPolicy)
else await transaction(checkPolicy)
assertOperationConfigurationLockScope()
await verifyRoutingObservationChecks(evidence.observationChecks)
assertOperationConfigurationLockScope()
if(evidence.steps.some(step=>step.kind==="service"))await assertOriginalMcpTemporaryGrant(pool,connection.id,principal.credentialId,originalServiceGrant)
assertOperationConfigurationLockScope()
return connection
}
const authorize = async () => {
assertOperationConfigurationLockScope()
const connection = await authorizeGrant()
assertOperationConfigurationLockScope()
await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant)
assertOperationConfigurationLockScope()
return connection
}
let ownerSources: (() => Promise<void>) | undefined
const authorizeCatalog = async () => {
assertOperationConfigurationLockScope()
const connection = await authorizeGrant()
assertOperationConfigurationLockScope()
ownerSources ??= createOspfOwnerSourceGuard(evidence.ospfOwnerGraph, serverIds)
await ownerSources()
assertOperationConfigurationLockScope()
return connection
}
try {
if (productionVrf && (evidence.steps.length !== 1 || evidence.steps[0].kind !== "ros" || evidence.steps[0].step.resource !== "vrf")) throw new McpError("STATE_CONFLICT", "VRF execution требует exact single typed step", 409)
if (await routingReservationEvidence(plan, evidence) && (!routingReservationForward || !routingReservationId)) throw new McpError("STATE_CONFLICT", "No confirmed reserved forward admission", 409)
if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась до запуска", 409)
const checksResolvedNextHop = evidence.healthChecks?.some(check => check.field === "resolved") === true
if (plan.operation_id === "operations.recover" && !checksResolvedNextHop && evidence.steps.some(change => change.kind === "ros" && ["ipv4_route", "ipv6_route"].includes(change.step.resource))) {
const { operationId: sourceId } = JSON.parse(await decryptSecret(plan.payload.sealed)) as { operationId: string }
const { rows: [source] } = await pool.query("SELECT p.operation_id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=$1 AND o.connection_id=$2", [sourceId, plan.connection_id])
if (!source || source.operation_id === "routing.ecmp.update") throw new McpError("PLAN_EXPIRED", "Подготовьте новый обратный план с проверкой next-hop", 409)
}
const applyWithLocks = () => withConfigMutationLock(serverIds, async () => {
await authorize()
const prefix = evidence.steps.some(step => step.kind === "ros") ? createRosPrefix(productionVrf ? evidence.snapshots.filter(snapshot => !isVrfCompanionResource(snapshot.resource)) : evidence.snapshots) : undefined
for (const change of evidence.steps) {
if (change.kind === "ros" && change.step.resource === "ip_settings" && change.step.fields["rp-filter"] === "strict") {
checkRpfPolicy(change.step.serverId, evidence.snapshots, "strict")
}
}
if (prefix) {
// An ordinary single-step plan has no device observation before its pre-intent check.
const needsEntrySnapshot = productionVrf || evidence.steps.length !== 1 ||
evidence.healthChecks?.length || evidence.managementChannels?.length ||
evidence.observationChecks?.length || evidence.ospfOwnerGraph || evidence.recovery
if (needsEntrySnapshot) await prefix.check(authorizeCatalog)
} else for (const snapshot of evidence.snapshots) {
if ((await readRosSnapshot(snapshot.serverId, snapshot.resource)).fingerprint !== snapshot.fingerprint) throw new McpError("STATE_CONFLICT", "Конфигурация изменилась после подготовки", 409)
}
await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog)
for (const step of evidence.steps) {
if (step.kind === "service" && fingerprint(await serviceCurrent(step.service, step.input)) !== fingerprint(evidence.before)) throw new McpError("STATE_CONFLICT", "Данные сервиса изменились", 409)
}
// Deterministic one-server stages. Each successful write is verified before the next.
for (let index = 0; index < evidence.steps.length; index++) {
if (evidence.recovery && Date.now() >= Date.parse(evidence.recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Срок гарантированного восстановления истёк до записи", 409)
for (const channel of evidence.managementChannels ?? []) {
if (!serverIds.includes(channel.serverId)) throw new McpError("SERVER_DENIED", "Management-проверка вне плана")
await verifyManagementChannel(channel.serverId, channel.channelId)
}
if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409)
const connection = await authorize()
const step = evidence.steps[index]
let vrfBeforeDispatch: (() => Promise<void>) | undefined
if (step.kind === "ros") {
await prefix!.check(authorizeCatalog)
const before = productionVrf ? evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId) : prefix!.before(step.step)
if (!before) throw new McpError("STATE_CONFLICT", "Нет исходного VRF source snapshot", 409)
await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,$2,$3)", [operationId, index, await encryptSecret(JSON.stringify(before))])
} else {
await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,$2,$3)", [operationId, index, await encryptSecret(JSON.stringify(evidence.snapshots))])
}
await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,server_id,state,intended_state) VALUES($1,$2,$3,'intent',$4::jsonb)", [operationId, index, step.kind === "ros" ? step.step.serverId : null, JSON.stringify(redact(step))])
attempted = true
activeServiceStep = step.kind === "service"
const serviceExecution=step.kind==="service"?{operationId,planId:plan.id,operationName:plan.operation_id,serviceName:step.service,stepIndex:index,phase:"forward" as const,
operationVersion:plan.operation_version,policyVersion:plan.policy_version,generation:plan.generation,principal:{...principal},
originalTemporaryGrantProof:originalServiceGrant!.proof,originalTemporaryGrantExpiresAt:originalServiceGrant!.expiresAt,
sealedEvidenceProof:createHash("sha256").update(plan.evidence.sealed).digest("hex"),sealedPayloadProof:createHash("sha256").update(plan.payload.sealed).digest("hex"),
originalExpiresAt:Math.min(plan.expires_at.getTime(),evidence.validUntil?Date.parse(evidence.validUntil):Infinity,Date.parse(principal.credentialExpiresAt),Date.parse(connection.policy.expiresAt),connection.owner_valid_until.getTime(),originalServiceGrant!.admissionExpiresAt),
serviceInput:step.input}:undefined
const reservedStep: { outcome?: ReservedRoutingStepOutcome<unknown> } = {}
vrfDiagnosticFallback = productionVrf && step.kind === "ros" && step.step.resource === "vrf" && ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id)
? createVrfRuntimeDiagnosticBoundary(operationId, step.step.action === "update" ? "patch" : step.step.action, diagnostic => { console.error(JSON.stringify(diagnostic)) }) : undefined
const beforeDispatch = step.kind === "ros" ? async () => {
await prefix!.check(authorizeCatalog)
if (checksResolvedNextHop) {
try {
await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog)
} catch (error) {
ecmpStoppedBeforeDispatch = true
throw error
}
}
await authorize()
await authorizeCatalog()
if (productionVrf) {
if (!vrfBeforeDispatch) throw new McpError("STATE_CONFLICT", "VRF command guard отсутствует", 409)
await vrfBeforeDispatch()
}
} : undefined
const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, beforeDispatch },
async () => {
vrfDiagnosticFallback?.executorEntered()
if (step.kind === "ros") {
if (!productionVrf) return executeRosStep(step.step)
const before = evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId)
if (!before?.sourceProof) throw new McpError("STATE_CONFLICT", "Нет VRF source-v3 proof", 409)
if (!["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id)) throw new McpError("STATE_CONFLICT", "VRF action binding изменён", 409)
const receipt = await executeVrfProductionStep({ operationId, planId: plan.id, connectionId: connection.id, ownerId: connection.owner_id,
operationName: plan.operation_id as "ros.vrf.create" | "ros.vrf.update" | "ros.vrf.delete" | "routing.vrf.update", operationVersion: plan.operation_version,
stepIndex: index, step: step.step, sourceProof: before.sourceProof, originalSnapshots: evidence.snapshots, authorize: authorizeCatalog,
assertDependencies: () => prefix!.check(authorizeCatalog), installBeforeDispatch: guard => { vrfBeforeDispatch = guard },
onDiagnostic: diagnostic => { vrfDiagnosticFallback?.observed(diagnostic) } })
vrfDiagnosticFallback?.executorReturned()
return receipt
}
const forward = (retainRoutingNetworkSession?: (session: RoutingReservedNetworkSession) => void) =>
serviceExecute(step.service, step.input, { operationId, phase: "forward", stepIndex: index,
...(routingReservationForward ? { routingReservationForward, retainRoutingNetworkSession } : {}) })
if (!routingReservationId) return step.service === "servers.poll" ? withRuntimePollPrincipalSource(principal, async (current, policyLoan) => {
assertOperationConfigurationLockScope()
await authorizeAfterPrincipal(current, policyLoan)
assertOperationConfigurationLockScope()
await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant)
assertOperationConfigurationLockScope()
}, forward) : forward()
reservedStep.outcome = await executeReservedRoutingStep({ reservationId: routingReservationId, sourceOperationId: operationId, principal, forward })
return reservedStep.outcome.forwardResult
})
vrfDiagnosticFallback = undefined
if (reservedStep.outcome?.kind === "recovered") {
// The generic source remains intent/result=null until the private
// inverse has finished. Record the forward ACK, never B VERIFIED
// after restoring A. No second operation or mutation debit is added.
confirmed++
reservedRecoverySummary = { state: "verified", mode: "same-slot", outcome: "recovered", nativeWrites: 1,
configurationVerified: true, controlProbeVerified: true, forwardingVerified: false,
recoveryCheckpointProof: reservedStep.outcome.recoveryResult.recoveryCheckpointProof }
await pool.query("UPDATE mcp_operation_steps SET state='applied',result=$3::jsonb,finished_at=now() WHERE operation_id=$1 AND step_index=$2", [operationId, index, JSON.stringify(redact(result ?? null))])
throw new McpError("VERIFICATION_FAILED", "Forward не прошёл сетевую проверку; исходная конфигурация восстановлена и проверена", 409)
}
if (reservedStep.outcome?.kind === "forwardPassed") reservedForwardPassed = true
if (diagnosticOnly) {
diagnosticReceipt = serviceDiagnosticReceipt(result, true, operationId, evidence.mutationCost)
if (!diagnosticReceipt) throw new McpError("VERIFICATION_FAILED", "Квитанция этапов диагностики не подтверждена", 409)
}
confirmed++
await pool.query("UPDATE mcp_operation_steps SET state='applied',result=$3::jsonb,finished_at=now() WHERE operation_id=$1 AND step_index=$2", [operationId, index, JSON.stringify(redact(result ?? null))])
if (step.kind === "service") {
const after = await Promise.all(evidence.snapshots.map(snapshot => readRosSnapshot(snapshot.serverId, snapshot.resource)))
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))])
}
if (step.kind === "ros" && productionVrf) {
await prefix!.check(authorizeCatalog)
} else if (step.kind === "ros") {
const after = await readRosSnapshot(step.step.serverId, step.step.resource, authorizeCatalog, prefix!.before(step.step).sourceProof)
await prefix!.advance(step.step, result, after, authorizeCatalog)
await verifyOspfOwnerGraphAtPhase(evidence.ospfOwnerGraph, serverIds, authorizeGrant, "post-ack")
await authorizeCatalog()
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))])
} else if (!await verifyStep(step, result)) throw new McpError("VERIFICATION_FAILED", "Запись подтверждена, но проверка результата не прошла", 409)
await pool.query("UPDATE mcp_operation_steps SET state=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, evidence.verificationScope === "acknowledgement" ? "acknowledged" : "verified"])
}
await verifyHealthChecks(evidence, serverIds, true, authorizeCatalog)
await prefix?.check(authorizeCatalog)
if (evidence.ospfOwnerGraph) {
await authorizeGrant()
await verifyOspfOwnerGraphAtPhase(evidence.ospfOwnerGraph, serverIds, authorizeGrant, confirmed > 0 ? "post-ack" : "admission")
await authorizeCatalog()
}
})
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks())
assertOperationConfigurationLockCheckpoint()
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
const finishedAt = await transaction(async client => {
assertOperationConfigurationLockCheckpoint()
const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1"
+ (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""),
[operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"])
assertOperationConfigurationLockCheckpoint()
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id])
assertOperationConfigurationLockCheckpoint()
await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) })
assertOperationConfigurationLockCheckpoint()
return receiptId ? terminal.rows[0]?.finished_at : undefined
})
assertOperationConfigurationLockCheckpoint()
if (routingReservationId && reservedForwardPassed) {
// Requires the real terminal applied + generic VERIFIED source. Failure
// conservatively retains the held inverse and both original charges.
await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {})
}
assertOperationConfigurationLockCheckpoint()
if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt }
} catch (error) {
vrfDiagnosticFallback?.failure(error)
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE"
const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost)
const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost)
const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" &&
(error as { diagnosticNotStarted?: unknown } | null)?.diagnosticNotStarted === true &&
diagnosticProgress?.state === "cancelled" && diagnosticProgress.attemptedStages === 0 && diagnosticProgress.completedStages === 0
const state = ecmpStoppedBeforeDispatch ? confirmed === 0 ? "rejected" : "partial"
: !attempted || diagnosticNotStarted ? "rejected" : code === "VERIFICATION_FAILED" ? "partial" : "unknown"
const dispatchOutcome = ecmpStoppedBeforeDispatch ? { failedStepDispatched: false } : {}
// Private operator STDERR only; never HTTP/result/audit payload.
try {
const [step] = evidence.steps
const manualRouting = !attempted && plan.operation_id === "routing.community.switch" && evidence.steps.length === 1 &&
step?.kind === "service" && step.service === "routing.community.scoped" && step.input.proofRevision === 2
const diagnostic = manualRouting ? manualRoutingPredispatchDiagnostic(error, operationId) : pollErrorLog(error, operationId, plan.operation_id)
if (diagnostic) console.error(JSON.stringify(diagnostic))
} catch { /* Logging cannot replace operation outcome. */ }
const recovery = reservedRecoverySummary ?? (code === "VERIFICATION_FAILED" && confirmed === evidence.steps.length && evidence.recovery
? await performReservedRecovery(operation, plan, evidence, principal, serverIds)
: undefined)
const persistFailure = mcpDatabaseRequestSignal()?.aborted ? requestAccountingWrite : transaction
await persistFailure(async client => {
await client.query("UPDATE mcp_operations SET state=$2,verification=$3,error_code=$4,result=$5::jsonb,finished_at=now() WHERE id=$1", [operationId, state, code === "VERIFICATION_FAILED" ? "failed" : "inconclusive", code, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, needsReview: state !== "rejected", ...dispatchOutcome, recovery, ...(nativeProgress ? { nativeProgress } : {}), ...(diagnosticProgress ? { diagnosticProgress } : {}), ...(diagnosticOnly ? { completedServiceSteps: confirmed } : {}) })])
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id])
await audit(client, "mcp-runner", "operation.stop", operation.connection_id, { operationId, state, code, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...dispatchOutcome, ...(nativeProgress ? { nativeProgress } : {}), ...(diagnosticProgress ? { diagnosticProgress } : {}) })
})
}
}
async function performReservedRecovery(operation: StoredOperation, plan: StoredPlan, evidence: ChangeEvidence, principal: McpPrincipal, serverIds: number[]) {
const recovery = evidence.recovery!
let attempted = false
try {
return await withApplicationMutationLock(() => withConfigMutationLock(serverIds, async () => {
if (Date.now() >= Date.parse(recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Зарезервированный срок восстановления истёк", 409)
const connection = await validatePrincipal(principal)
if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась; допуск восстановления недействителен", 409)
const { rows: [grant] } = await pool.query("SELECT config FROM mcp_routing_profiles WHERE id=$1 AND connection_id=$2 AND credential_id=$3 AND enabled AND grant_expires_at>now() AND policy_version=$4 AND generation=$5", [recovery.authorityProfileId, connection.id, principal.credentialId, connection.policy.version, await accessGeneration()])
if (!grant?.config?.autoRecovery || recovery.step.kind !== "service" || recovery.step.service !== "routing.community.restore") throw new McpError("PERMISSION_DENIED", "Нет действующего допуска этого восстановления")
const recoveryProfile = recovery.step.input.profile
if (fingerprint(grant.config) !== fingerprint(recoveryProfile)) throw new McpError("STATE_CONFLICT", "Профиль восстановления изменился", 409)
await transaction(async client => {
const input = await policyInput(client, connection, plan.operation_id, evidence.target, evidence)
input.temporaryGrant = await temporaryScope(client, connection, principal.credentialId)
input.usage.callsInMinute = 0; input.usage.mutationsInHour = 0
const decision = evaluateMcpPolicy(input)
if (decision.decision === "denied" || decision.decision === "approval-required" && !plan.approved_by) throw new McpError("PERMISSION_DENIED", "Текущая защита или допуск запрещает восстановление")
})
for (const channel of evidence.managementChannels ?? []) await verifyManagementChannel(channel.serverId, channel.channelId)
if (!evidence.managementChannels?.length) throw new McpError("APPROVAL_REQUIRED", "Восстановление требует независимого канала")
for (const [index, step] of evidence.steps.entries()) {
const { rows: [checkpoint] } = await pool.query("SELECT after_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=$2", [operation.id, index])
if (!checkpoint?.after_sealed) throw new McpError("STATE_CONFLICT", "Нет подтверждённого снимка после исходного шага", 409)
const raw = JSON.parse(await decryptSecret(checkpoint.after_sealed))
const snapshots = step.kind === "ros" ? [raw] : raw
for (const snapshot of snapshots) if ((await readRosSnapshot(snapshot.serverId, snapshot.resource)).fingerprint !== snapshot.fingerprint) throw new McpError("STATE_CONFLICT", "Зависимость изменилась после записи; восстановление запрещено", 409)
}
if (fingerprint(await serviceCurrent(recovery.step.service, recovery.step.input)) !== fingerprint(recovery.expected)) throw new McpError("STATE_CONFLICT", "Конфигурация изменилась извне; восстановление остановлено", 409)
const intent = await pool.query("INSERT INTO mcp_recovery_intents(operation_id,state,authority_profile_id) VALUES($1,'intent',$2) ON CONFLICT DO NOTHING RETURNING operation_id", [operation.id, recovery.authorityProfileId])
if (!intent.rowCount) throw new McpError("STATE_CONFLICT", "Восстановление уже начиналось и не повторяется", 409)
await audit(pool, connection.owner_id, "recovery.intent", connection.id, { operationId: operation.id, authorityProfileId: recovery.authorityProfileId })
attempted = true
const step = recovery.step
const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: true }, () => serviceExecute(step.service, step.input, { operationId: operation.id, phase: "recovery" }))
if (!await verifyStep(step, result)) throw new McpError("VERIFICATION_FAILED", "Проверка восстановления не прошла", 409)
await pool.query("UPDATE mcp_recovery_intents SET state='verified',finished_at=now() WHERE operation_id=$1", [operation.id])
await audit(pool, connection.owner_id, "recovery.verified", connection.id, { operationId: operation.id })
return { state: "verified", authorityProfileId: recovery.authorityProfileId }
}))
} catch (error) {
if (attempted) await pool.query("UPDATE mcp_recovery_intents SET state='unknown',finished_at=now() WHERE operation_id=$1 AND state='intent'", [operation.id]).catch(() => {})
return { state: attempted ? "unknown" : "blocked", errorCode: error instanceof McpError ? error.code : "STATE_UNAVAILABLE", automaticRetry: false }
}
}
export async function recoverMcpOperations() {
// A second backend must not interrupt a runner still holding its session lease.
// A pending operation can still be queued behind maintenance. Its original
// plan expiry bounds that wait across processes without granting a new lease.
const interrupted = `o.created_at<now()-interval '2 minutes'
AND (o.state='running' OR (o.state='pending' AND p.expires_at<=now()))`
const { rows } = await pool.query(`SELECT o.id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id
WHERE ${interrupted} ORDER BY o.created_at,o.id LIMIT 100`)
for (const row of rows) {
await withBackgroundPartitionWrite(async ({ client }) => {
const { rows: [lock] } = await client.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + row.id])
if (!lock.locked) return
// Defer the whole transition when either row is busy.
const { rows: [target] } = await client.query(`SELECT o.plan_id,o.connection_id
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id
WHERE o.id=$1 AND ${interrupted}
FOR UPDATE OF o,p SKIP LOCKED`, [row.id])
if (!target) return
await client.query("UPDATE mcp_operations SET state='unknown',verification='inconclusive',error_code='PROCESS_INTERRUPTED',finished_at=now() WHERE id=$1", [row.id])
await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [target.plan_id])
await audit(client, "mcp-recovery", "operation.interrupted", target.connection_id, { operationId: row.id, retryAllowed: false })
})
}
await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS (
SELECT id FROM mcp_plans
WHERE expires_at<now() AND state IN ('draft','approved','awaiting-approval')
ORDER BY expires_at,id LIMIT 1000 FOR UPDATE SKIP LOCKED
)
UPDATE mcp_plans SET state='expired' FROM expired WHERE mcp_plans.id=expired.id`)).catch(error => {
const diagnostic = partitionWriteFailureDiagnostic(error)
if (diagnostic) {
try { console.error(JSON.stringify({ event: "mcp.worker.plan-expiry-failed", ...diagnostic })) }
catch { /* Preserve the original transaction failure. */ }
}
throw error
})
await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS (
SELECT id FROM mcp_read_leases WHERE expires_at<now()
ORDER BY expires_at,id LIMIT 1000 FOR UPDATE SKIP LOCKED
)
DELETE FROM mcp_read_leases USING expired WHERE mcp_read_leases.id=expired.id`))
await withBackgroundPartitionWrite(({ client }) => client.query(`WITH interrupted AS (
SELECT intent.operation_id FROM mcp_recovery_intents intent
JOIN mcp_operations operation ON operation.id=intent.operation_id
WHERE intent.state='intent' AND operation.state NOT IN ('pending','running')
ORDER BY intent.created_at,intent.operation_id LIMIT 1000 FOR UPDATE OF intent SKIP LOCKED
)
UPDATE mcp_recovery_intents SET state='unknown',finished_at=now() FROM interrupted
WHERE mcp_recovery_intents.operation_id=interrupted.operation_id`))
}