import { partitionWriteFailureDiagnostic, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js" import { pollErrorLog } from "../../services/server-poll-error-provenance.js" import { manualRoutingPredispatchDiagnostic } from "./routing-community-predispatch-diagnostic.js" import { vrfPreparationDiagnostic } from "./vrf-preparation-diagnostic.js" import { createVrfRuntimeDiagnosticBoundary } from "../../services/vrf-execution-diagnostic.js" import { capturePublishedServerProbeTerminalProof } from "../../services/server-probe-operation-retirement.js" import { SERVER_PROBE_OPERATION } from "../../services/server-probe-authority.js" import { preparedPlanExpiry } from "./plan-expiry.js" import { acknowledgedCreateId, assertNoCallerCreateId } from "./native-create-identity.js" import { randomUUID, createHash } from "node:crypto" import { performance } from "node:perf_hooks" import { readMcpTemporaryGrantOrigin,assertOriginalMcpTemporaryGrant,type OriginalServiceGrant } from "./temporary-grant-origin.js" import type { PoolClient } from "pg" import { mcpProtectedObjectSchema, mcpTemporaryGrantSchema, type McpPolicyPreviewRequest } from "@mmapp/contracts/mcp" import { appUserCreateSchema, appUserUpdateSchema, userBindingCreateSchema } from "@mmapp/contracts/users" import { logSettingsPatchSchema } from "@mmapp/contracts" import { pool } from "../../db/index.js" import { hasPermission, type AuthUser } from "../../lib/permissions.js" import { createUser, updateUser, deleteUser, addBinding, removeBinding, getUserById } from "../users/service/users-service.js" import { normalizeBindingPeer } from "../users/peer-bind.js" import { getLogSettings, updateLogSettings } from "../../services/router-logs-collector.js" import { refreshScheduler } from "../../services/scheduler.js" import { withApplicationMutationLock, withConfigMutationLock } from "../../services/config-mutation-lock.js" import { withOperationConfigurationLockClient, assertOperationConfigurationLockScope, assertOperationConfigurationLockCheckpoint } from "../../services/operation-config-lock-lease.js" import { sharedDatabaseGate } from "../../services/shared-database-gate.js" import { mcpDatabaseRequestSignal, withMcpDatabaseRequest, withMcpRunnerDatabaseRequest } from "./database-request-lease.js" import { setTimeout as waitForRunnerAdmission } from "node:timers/promises" import { withRuntimePollPrincipalSource, executeRuntimePollPolicyTransaction, type RuntimePollPolicyLoan } from "./runtime-poll-principal-boundary.js" import { readPollRuntimePolicySource, type PollRuntimePolicySource } from "./poll-runtime-policy-source.js" import { withVerifiedConfigurationChange, assertVerifiedConfigurationAccess } from "../../services/config-mutation-policy.js" import { beginMcpOperationAdmission, insertMcpOperationAdmission, retireMcpOperationAdmission, withMcpOperationExecutionOrigin, stripMcpOperationExecutionPrivateResult, type McpOperationAdmission } from "./operation-execution-origin.js" import { createOspfOwnerSourceGuard, verifyOspfOwnerGraph, verifyOspfOwnerGraphAtPhase } from "./ospf-owner-graph.js" import { evaluateMcpPolicy } from "./policy.js" import { findAction, findActionByOperation } from "./actions.js" import { refuseExpiredOperationReplay } from "./operation-replay-tombstones.js" import { accessGeneration, decryptSecret, encryptSecret, fingerprint, privateSecretFingerprint, McpError, redact } from "./security.js" import { audit, getConnection, transaction, requestAccountingWrite, validatePrincipal, effectiveMcpConnection, assertConnectionVisible, type McpPrincipal, type ConnectionRow, type StoredPlan, type StoredOperation } from "./store.js" import { executeRosStep, readRosSnapshot, routerClient, type RosSnapshot } from "./resources.js" import { verifyHealthChecks } from "./routing-health-checks.js" import { verifyRosStepSnapshots } from "./ros-step-verification.js" import { createRosPrefix } from "./ros-prefix.js" import { checkRpfPolicy } from "./routing-controls.js" import type { ActionContext, ChangeEvidence, ChangeStep } from "./action-types.js" import { logsFollowCancellation } from "./logs-follow-cancellation.js" import { findMcpService, type McpServiceExecutionContext, type RoutingReservedNetworkSession } from "./service-registry.js" import { withDiagnosticCancellation } from "./diagnostic-cancellation.js" import { z } from "zod" import { verifyManagementChannel } from "./management.js" import { intersectGrant } from "./store.js" import { changeMutationCost } from "./change-cost.js" import { verifyRoutingObservationChecks } from "./routing-observations.js" import { publicMcpChangeEvidence } from "./routing-community-public.js" import { prepareRuntimeVrfTicket } from "./vrf-ticket-runtime.js" import { executeVrfProductionStep, isVrfCompanionResource, VrfProductionAcknowledgedFailure } from "../../services/vrf-production-execution.js" import { VrfProductionClaimOutcomeUnknown } from "../../services/vrf-production-ticket-store.js" import { serviceNativeProgress } from "./service-native-progress.js" import { serviceDiagnosticProgress, serviceDiagnosticReceipt } from "./service-diagnostic-progress.js" import { admitManualInverseWithinOwner } from "./routing-manual-inverse-store.js" import { bindRoutingChargeAdmission } from "./routing-reservation-transaction.js" import { withRoutingReservationAdmission, createRoutingReservationStore, readHistoricalChargesWithinOwner, admitPersistentRoutingForwardWithinOwner, type RoutingReservedForwardAllocation } from "./routing-reservation-store.js" import { lockRoutingReservationAdmissionSource, readScopedRoutingPathAdmissionDeadline, requiresScopedRoutingOwnerApproval, routingReservationEvidence } from "./routing-reservation-runtime.js" import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js" let activeRunners = 0 const runnerQueue: Array<() => void> = [] async function runBounded(work: () => Promise) { if (activeRunners >= 2) await new Promise(resolve => runnerQueue.push(resolve)) else activeRunners++ try { await work() } finally { const next = runnerQueue.shift(); if (next) next(); else activeRunners-- } } async function runOrdinaryOperation(work: () => Promise): Promise { for (;;) { await sharedDatabaseGate.whenOpen() let local = false, entered = false try { return await runBounded(() => sharedDatabaseGate.request(() => { local = true return withMcpRunnerDatabaseRequest(() => { entered = true return work() }) })) } catch (error) { if (entered || (local ? !(error instanceof McpError && ["PROTOCOL_STORAGE_BUSY", "RATE_LIMITED"].includes(error.code)) : !sharedDatabaseGate.isPaused())) throw error await waitForRunnerAdmission(250) } } } async function policyInput(client: Pick, connection: ConnectionRow, operationId: string, target: ChangeEvidence["target"], evidence?: ChangeEvidence, source?: PollRuntimePolicySource): Promise { if (operationId === "operations.recover" && evidence?.snapshots.some(snapshot => ["routing_policy:routing_filter", "routing_policy:routing_filter_select"].includes(snapshot.resource)) && (!hasPermission(connection.policy.permissions, "mm:filters:read") || !hasPermission(connection.owner_permissions, "mm:filters:read"))) { throw new McpError("PERMISSION_DENIED", "Для восстановления привязки OSPF требуется право чтения фильтров", 403) } const { rows: protections } = source ? { rows: source.protections } : await client.query("SELECT object_ref FROM mcp_protections LIMIT 1001") if (protections.length > 1000) throw new McpError("STATE_UNAVAILABLE", "Число защит превышает предел проверки; изменение заблокировано", 503) const { rows: [usage] } = source ? { rows: [source.usage] } : await client.query<{ calls: number; mutations: number; reads: number }>(` SELECT COALESCE((SELECT used FROM mcp_budget_buckets WHERE connection_id=$1 AND kind='calls' AND bucket=date_trunc('minute',now())),0) AS calls, COALESCE((SELECT used FROM mcp_budget_buckets WHERE connection_id=$1 AND kind='mutations' AND bucket=date_trunc('hour',now())),0) AS mutations, (SELECT count(*)::int FROM mcp_read_leases WHERE connection_id=$1 AND expires_at>now()) AS reads`, [connection.id]) return { operationId, policy: connection.policy, ownerPermissions: connection.owner_permissions, target, objects: [...(evidence?.objects ?? []), ...(evidence?.requiresHuman && target.scope !== "servers" ? [{ kind: "mcp-operation", id: operationId }] : [])], objectCount: evidence ? changeMutationCost(evidence) + (evidence.recovery ? 1 : 0) : 0, protections: [ ...protections.map(row => mcpProtectedObjectSchema.parse(row.object_ref)), ...(evidence?.requiresHuman && target.scope === "servers" ? target.serverIds.map(serverId => ({ kind: "server", id: String(serverId), serverId, protection: "human-approval-required" as const })) : []), ...(evidence?.requiresHuman && target.scope !== "servers" ? [{ kind: "mcp-operation", id: operationId, protection: "human-approval-required" as const }] : []), ], usage: { callsInMinute: Number(usage.calls), mutationsInHour: Number(usage.mutations), concurrentReads: Number(usage.reads) }, auditAvailable: true, } } export async function readMcpPolicyPreviewInput(connection: ConnectionRow, input: Pick): Promise { const current = await policyInput(pool, connection, input.operationId, input.target) return { ...current, objects: input.objects, objectCount: input.objectCount } } async function temporaryScope(client: Pick, connection: ConnectionRow, credentialId: string) { const { rows } = await client.query("SELECT payload FROM mcp_grants WHERE connection_id=$1 AND revoked_at IS NULL AND payload->>'credentialId'=$2", [connection.id, credentialId]) if (!rows.length) return undefined if (rows.length !== 1) throw new McpError("GRANT_INACTIVE", "Неоднозначный временный доступ") const { credentialId: _credential, ...grant } = rows[0].payload return mcpTemporaryGrantSchema.parse(grant) } async function reserveCall(principal: McpPrincipal, toolName: string, input: Record, signal?: AbortSignal) { const action = findAction(toolName), requestId = randomUUID() const reservation = await requestAccountingWrite(async client => { await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId]) const connection = await validatePrincipal(principal, client) const grant = await temporaryScope(client, connection, principal.credentialId) const context: ActionContext = { principal, connection: intersectGrant(connection, grant), signal } const evidence = await policyInput(client, connection, action.operation.id, await action.target(input, { ...context, transactionClient: client })) evidence.temporaryGrant = grant const result = evaluateMcpPolicy(evidence) await client.query("INSERT INTO mcp_requests(id,connection_id,actor_id,tool,server_ids,status,input,error_code) VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8)", [requestId, connection.id, connection.owner_id, toolName, evidence.target.scope === "servers" ? evidence.target.serverIds : [], result.decision === "denied" ? "denied" : "running", JSON.stringify(redact(input)), result.decision === "denied" ? result.reasons[0]?.code : null]) if (result.decision !== "denied") { await client.query("INSERT INTO mcp_budget_buckets(connection_id,kind,bucket,used) VALUES($1,'calls',date_trunc('minute',now()),1) ON CONFLICT(connection_id,kind,bucket) DO UPDATE SET used=mcp_budget_buckets.used+1", [connection.id]) if (action.operation.effect === "read") await client.query("INSERT INTO mcp_read_leases(id,connection_id,expires_at) VALUES($1,$2,now()+interval '2 minutes')", [requestId, connection.id]) await client.query("UPDATE mcp_connections SET last_used_at=now() WHERE id=$1", [connection.id]) } return { context, result } }) if (reservation.result.decision === "denied") throw new McpError(reservation.result.reasons[0]?.code ?? "PERMISSION_DENIED", reservation.result.reasons.map(reason => reason.message).join("; ")) return { action, requestId, context: reservation.context } } async function completeRequest(id: string, started: number, result: unknown, error?: unknown, assertDeliveryAccess?: (client: PoolClient) => Promise) { await requestAccountingWrite(async client => { await client.query("UPDATE mcp_requests SET status=$2,result=$3::jsonb,error_code=$4,duration_ms=$5,finished_at=now() WHERE id=$1", [id, error ? "error" : "completed", JSON.stringify(redact(error ? { message: error instanceof McpError ? error.message : "Внутренняя ошибка операции" } : result)), error instanceof McpError ? error.code : error ? "INTERNAL_ERROR" : null, Math.min(2147483647, Date.now() - started)]) await client.query("DELETE FROM mcp_read_leases WHERE id=$1", [id]) await assertDeliveryAccess?.(client) }) } type InternalMcpCall = { persistentRoutingProfile: NonNullable } | { captureRoutingConfirmation: NonNullable } export async function invokeMcpTool(principal: McpPrincipal, toolName: string, raw: unknown, signal?: AbortSignal, internal?: InternalMcpCall): Promise { if (toolName === "mm_logs_follow") { const maintenance = new AbortController() return sharedDatabaseGate.request( () => withMcpDatabaseRequest(() => invokeMcpToolInner(principal, toolName, raw, signal, internal, maintenance.signal)), () => maintenance.abort(new McpError("RESET_REQUIRED", "База данных перешла в режим обслуживания; начните новое чтение", 409)), ) } return sharedDatabaseGate.request(() => withMcpDatabaseRequest(() => invokeMcpToolInner(principal, toolName, raw, signal, internal))) } async function invokeMcpToolInner(principal: McpPrincipal, toolName: string, raw: unknown, signal?: AbortSignal, internal?: InternalMcpCall, logsFollowMaintenance?: AbortSignal): Promise { const action = findAction(toolName), input = action.schema.parse(raw) as Record, started = Date.now() const leaseSignal = mcpDatabaseRequestSignal() leaseSignal?.throwIfAborted() const actionSignal = leaseSignal && signal ? AbortSignal.any([leaseSignal, signal]) : leaseSignal ?? signal const call = await reserveCall(principal, toolName, input, actionSignal) const cancelled = (): McpError | undefined => leaseSignal?.aborted ? leaseSignal.reason : logsFollowMaintenance && logsFollowCancellation(logsFollowMaintenance, signal) const assertActive = () => { const failure = cancelled(); if (failure) throw failure } let vrfPreparing = false let routingConfirmation: Parameters>[0] | undefined try { assertActive() if (logsFollowMaintenance) call.context.logsFollowMaintenance = leaseSignal ? AbortSignal.any([logsFollowMaintenance, leaseSignal]) : logsFollowMaintenance if (internal && "persistentRoutingProfile" in internal) { if (action.operation.id !== "routing.community.switch") throw new McpError("INVALID_INPUT", "Контекст профиля не относится к этой операции", 400) call.context.persistentRoutingProfile = z.object({ id: z.string().uuid(), version: z.number().int().positive().safe() }).strict().parse(internal.persistentRoutingProfile) } if (internal && "captureRoutingConfirmation" in internal) { if (action.operation.id !== "routing.analyze") throw new McpError("INVALID_INPUT", "Измерения контроллера не относятся к этой операции", 400) call.context.captureRoutingConfirmation = observations => { routingConfirmation = observations } } let result: unknown if (action.read) result = await action.read(input, call.context) else if (action.prepare) { vrfPreparing = ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete"].includes(action.operation.id) const evidence = await action.prepare(input, call.context) result = await transaction(async client => { assertActive() await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId]) const connection = await validatePrincipal(principal, client) const check = await policyInput(client, connection, action.operation.id, evidence.target, evidence) check.temporaryGrant = await temporaryScope(client, connection, principal.credentialId) // The call already reserved one call slot. Preparation does not reserve a mutation. check.usage.callsInMinute = Math.max(0, check.usage.callsInMinute - 1) let decision = evaluateMcpPolicy(check) if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; ")) const id = randomUUID(), expiresAt = preparedPlanExpiry(action.operation.id, action.operation.section, evidence) if (!expiresAt) throw new McpError("PLAN_EXPIRED", "Срок исходных данных плана истёк или недоступен; подготовьте новый план", 409) const generation = await accessGeneration() await client.query("INSERT INTO mcp_plans(id,connection_id,operation_id,operation_version,policy_version,generation,payload,evidence,state,expires_at) VALUES($1,$2,$3,$4,$5,$6,$7::jsonb,$8::jsonb,$9,$10)", [id, connection.id, action.operation.id, action.operation.version, connection.policy.version, generation, JSON.stringify({ sealed: await encryptSecret(JSON.stringify(input)) }), JSON.stringify({ sealed: await encryptSecret(JSON.stringify(evidence)) }), decision.decision === "approval-required" ? "awaiting-approval" : "draft", expiresAt]) const vrfTicket = await prepareRuntimeVrfTicket({ operationId: action.operation.id, operationVersion: action.operation.version, planId: id, connectionId: connection.id, ownerId: connection.owner_id, evidence, sql: client, assertAuthority: async () => { if (signal?.aborted || generation !== await accessGeneration() || expiresAt.getTime() <= Date.now()) throw new McpError("PLAN_EXPIRED", "Подготовка VRF отменена или устарела", 409) const current = await validatePrincipal(principal, client) if (current.owner_id !== connection.owner_id || fingerprint(current.policy) !== fingerprint(connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(connection.owner_permissions)) throw new McpError("STATE_CONFLICT", "Доступ изменился во время подготовки VRF", 409) const fresh = await policyInput(client, current, action.operation.id, evidence.target, evidence) fresh.temporaryGrant = await temporaryScope(client, current, principal.credentialId) fresh.usage.callsInMinute = Math.max(0, fresh.usage.callsInMinute - 1) const reviewed = evaluateMcpPolicy(fresh) if (reviewed.decision === "denied") throw new McpError(reviewed.reasons[0].code, reviewed.reasons.map(reason => reason.message).join("; ")) if (reviewed.decision !== decision.decision) throw new McpError("STATE_CONFLICT", "Политика подготовки VRF изменилась", 409) }, assertCurrentSource: async (serverId, proof) => { const { server } = await routerClient(serverId) const current = await privateSecretFingerprint(fingerprint({ id: server.id, enabled: server.enabled, host: server.host, port: server.port, useSsl: server.useSsl, verifySsl: server.verifySsl, username: server.username, password: server.password, epoch: server.routingEndpointEpoch }), "mcp-ros-snapshot-source-v3") if (!server.enabled || current !== proof) throw new McpError("STATE_CONFLICT", "Источник VRF изменился", 409) }, }) if (vrfTicket) { // Admission closes physical/VLAN parents and implicit native objects. // Recheck the existing object policy/budget before publishing the // final encrypted plan; the whole transaction rolls back on refusal. const finalConnection = await validatePrincipal(principal, client) const finalCheck = await policyInput(client, finalConnection, action.operation.id, evidence.target, evidence) finalCheck.temporaryGrant = await temporaryScope(client, finalConnection, principal.credentialId) finalCheck.usage.callsInMinute = Math.max(0, finalCheck.usage.callsInMinute - 1) decision = evaluateMcpPolicy(finalCheck) if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; ")) await client.query("UPDATE mcp_plans SET evidence=$2::jsonb,state=$3 WHERE id=$1", [id, JSON.stringify({ sealed: await encryptSecret(JSON.stringify(evidence)) }), decision.decision === "approval-required" ? "awaiting-approval" : "draft"]) } await audit(client, connection.owner_id, "plan.create", connection.id, { planId: id, operationId: action.operation.id, decision }) assertActive() return { planId: id, expiresAt: expiresAt.toISOString(), state: decision.decision === "approval-required" ? "awaiting-approval" : "draft", decision, changes: redact(publicMcpChangeEvidence(action.operation.id, evidence)), ...(vrfTicket ? { vrfTicket } : {}), executionAvailable: true } }) vrfPreparing = false } else throw new McpError("UNSUPPORTED_CAPABILITY", "Нет обработчика операции", 409) assertActive() const assertCurrentResultAccess = async (client?: PoolClient) => { const current = await effectiveMcpConnection(principal, client) assertActive() if (fingerprint(current.policy) !== fingerprint(call.context.connection.policy) || fingerprint(current.owner_permissions) !== fingerprint(call.context.connection.owner_permissions)) throw new McpError("STATE_CONFLICT", "Доступ изменился во время вызова; результат не выдан", 409) } await assertCurrentResultAccess() const output = redact(result) const checkDelivery = action.operation.section === "logs" && action.operation.effect === "read" ? assertCurrentResultAccess : leaseSignal ? async () => leaseSignal.throwIfAborted() : undefined await completeRequest(call.requestId, started, output, undefined, checkDelivery) await checkDelivery?.() assertActive() if (routingConfirmation && internal && "captureRoutingConfirmation" in internal) internal.captureRoutingConfirmation(routingConfirmation) return output } catch (error) { if (vrfPreparing) { try { const diagnostic = vrfPreparationDiagnostic(call.requestId, action.operation.id, error) if (diagnostic) console.error(JSON.stringify(diagnostic)) } catch { /* Diagnostics must never replace the original failure. */ } } const failure = cancelled() ?? error await completeRequest(call.requestId, started, null, failure) throw failure } } export async function loadPlan(id: string, client: Pick = pool): Promise { const { rows: [plan] } = await client.query("SELECT * FROM mcp_plans WHERE id=$1", [id]) if (!plan) throw new McpError("NOT_FOUND", "План не найден", 404) return plan } export async function publicPlan(plan: StoredPlan) { const evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence return { id: plan.id, connectionId: plan.connection_id, operationId: plan.operation_id, state: plan.state, expiresAt: plan.expires_at.toISOString(), createdAt: plan.created_at.toISOString(), approvedBy: plan.approved_by, changes: redact(publicMcpChangeEvidence(plan.operation_id, evidence)), input: redact(JSON.parse(await decryptSecret(plan.payload.sealed))) } } export async function approvePlan(id: string, actor: AuthUser, approve: boolean) { return transaction(async client => { await client.query("SELECT id FROM mcp_plans WHERE id=$1 FOR UPDATE", [id]) const plan = await loadPlan(id, client), connection = await getConnection(plan.connection_id, client) assertConnectionVisible(connection, actor) if (!["awaiting-approval", "draft"].includes(plan.state) || plan.expires_at.getTime() <= Date.now() || plan.generation !== await accessGeneration()) throw new McpError("PLAN_EXPIRED", "План устарел или уже запущен", 409) const action = findActionByOperation(plan.operation_id) if (!hasPermission(actor.permissions, "mm:mcp:approve") || action.operation.requiredPermissions.some(permission => !hasPermission(actor.permissions, permission))) throw new McpError("PERMISSION_DENIED", "Нет прав на подтверждение действия") if (approve && actor.id !== connection.owner_id && plan.operation_id === "routing.community.switch" && requiresScopedRoutingOwnerApproval(JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence)) { throw new McpError("PERMISSION_DENIED", "Переключение постоянного профиля подтверждает владелец подключения", 403) } await client.query("UPDATE mcp_plans SET state=$2,approved_by=$3,approved_at=now() WHERE id=$1", [id, approve ? "approved" : "cancelled", actor.id]) await audit(client, actor.id, approve ? "plan.approve" : "plan.reject", connection.id, { planId: id }) }) } async function serviceCurrent(service: string, input: Record): Promise { const custom = findMcpService(service) if (custom) return custom.current(input) if (service === "logs.settings.update") return getLogSettings() if (service.startsWith("users.")) return input.id ? getUserById(String(input.id)) : null throw new McpError("UNSUPPORTED_CAPABILITY", "Неизвестный сервис", 409) } async function serviceExecute(service: string, input: Record, context: McpServiceExecutionContext): Promise { const custom = findMcpService(service) if (custom) { if (!custom.cancellableDiagnostic) return custom.execute(input, context) let entered = false try { return await withDiagnosticCancellation(context, signal => { entered = true return custom.execute(input, context.phase === "forward" ? { ...context, signal } : context) }) } catch (error) { if (!entered && context.phase === "forward" && error instanceof McpError && error.code === "READ_CANCELLED" && custom.diagnosticIdentity) { // Only the watcher knows execute was never entered. The handler still // proves the exact encrypted plan/operation before supplying identity. try { const identity = await custom.diagnosticIdentity(input, context) const progress = serviceDiagnosticProgress(Object.assign(new McpError("READ_CANCELLED", "Диагностика отменена до начала"), { diagnosticProgress: { ...identity, operationId: context.operationId, state: "cancelled", attemptedStages: 0, completedStages: 0, automaticReplay: false }, }), true, context.operationId, identity.stageBudget) if (progress) Object.assign(error, { diagnosticNotStarted: true, diagnosticProgress: progress }) } catch { /* Failure to prove ownership keeps the conservative unknown. */ } } throw error } } if (service === "users.create") return createUser(appUserCreateSchema.parse(input.user)) if (service === "users.update") return updateUser(String(input.id), appUserUpdateSchema.parse(input.user)) if (service === "users.delete") return deleteUser(String(input.id)) if (service === "users.binding_add") return addBinding(String(input.id), userBindingCreateSchema.parse(input.binding), { assertCurrent: assertVerifiedConfigurationAccess }) if (service === "users.binding_remove") return removeBinding(String(input.id), String(input.bindingId)) if (service === "logs.settings.update") { const result = await updateLogSettings(logSettingsPatchSchema.parse(input.settings)); await refreshScheduler(); return result } throw new McpError("UNSUPPORTED_CAPABILITY", "Неизвестный сервис", 409) } export async function verifyStep(step: ChangeStep, result: unknown, before?: RosSnapshot, after?: RosSnapshot): Promise { if (step.kind === "service") { const custom = findMcpService(step.service) if (custom?.verify) return custom.verify(step.input, result) if (step.service === "users.delete") return !await getUserById(String(step.input.id)) if (step.service === "users.create" || step.service === "users.update") { const id = step.service === "users.create" && result && typeof result === "object" ? String((result as { id?: string }).id ?? "") : String(step.input.id) const current = await getUserById(id) if (!current) return false const expected = step.service === "users.create" ? appUserCreateSchema.parse(step.input.user) : appUserUpdateSchema.parse(step.input.user) return Object.entries(expected).every(([key, value]) => { if (key === "avatar" && typeof value === "string" && !value.trim()) return true // Existing service derives initials or retains the prior avatar. const normalized = ["name", "login", "email", "avatar"].includes(key) && typeof value === "string" ? value.trim() : value return fingerprint((current as unknown as Record)[key]) === fingerprint(normalized) }) } if (step.service === "users.binding_remove") { const current = await getUserById(String(step.input.id)); return !!current && !current.bindings.some(binding => binding.id === step.input.bindingId) } if (step.service === "users.binding_add") { const binding = userBindingCreateSchema.parse(step.input.binding), current = await getUserById(String(step.input.id)) const createdId = result && typeof result === "object" ? (result as { id?: unknown }).id : undefined return typeof createdId === "string" && !!current?.bindings.some(item => item.id === createdId && item.serverId === binding.serverId && item.interfaceName === binding.interfaceName.trim() && (binding.interfaceType === undefined || item.interfaceType === binding.interfaceType) && item.peerPublicKey === normalizeBindingPeer(item.interfaceType, binding.peerPublicKey)) } if (step.service === "logs.settings.update") { const current = await getLogSettings() as unknown as Record return Object.entries(logSettingsPatchSchema.parse(step.input.settings)).every(([key, value]) => current[key] === value) } // Service methods confirm their own DB changes; the receipt remains inspectable. return result !== undefined || step.service === "users.binding_remove" } assertNoCallerCreateId(step.step) const id = step.step.action === "create" ? acknowledgedCreateId(result) : step.step.id if (step.step.action === "create" && !id) return false if (!before) return false return verifyRosStepSnapshots(step.step, result, before, after ?? await readRosSnapshot(step.step.serverId, step.step.resource)) } export async function applyPlan(principal: McpPrincipal, planId: string, idempotencyKey: string): Promise { return withMcpDatabaseRequest(() => admitPlan(principal, planId, idempotencyKey)) } async function admitPlan(principal: McpPrincipal, planId: string, idempotencyKey: string): Promise { const operationAdmissionStart = beginMcpOperationAdmission(principal) let reservedRouting = false let originalServiceGrant:OriginalServiceGrant|undefined let operationAdmission: McpOperationAdmission | undefined const admitted = await withRoutingReservationAdmission(async (client, reserveRouting) => { await client.query("SELECT id FROM mcp_connections WHERE id=$1 FOR UPDATE", [principal.connectionId]) const connection = await validatePrincipal(principal, client) const { rows: [existing] } = await client.query("SELECT * FROM mcp_operations WHERE connection_id=$1 AND (idempotency_key=$2 OR plan_id=$3) ORDER BY created_at LIMIT 1", [connection.id, idempotencyKey, planId]) if (existing) { if (existing.plan_id !== planId) throw new McpError("STATE_CONFLICT", "Ключ уже используется другим планом", 409) const original = await loadPlan(planId, client) const originalEvidence=JSON.parse(await decryptSecret(original.evidence.sealed)) as ChangeEvidence reservedRouting = !!await routingReservationEvidence(original, originalEvidence) return existing } await refuseExpiredOperationReplay(client, principal, connection.id, planId, idempotencyKey) await client.query("SELECT pg_advisory_xact_lock(hashtextextended('mcp-runner-admission',0))") const pending = await client.query<{ count: number }>("SELECT count(*)::int AS count FROM mcp_operations WHERE state IN ('pending','running')") if (pending.rows[0].count >= 100) throw new McpError("BUDGET_EXCEEDED", "Очередь исполнителя заполнена; повторите подготовленное применение позже", 429) const selected = await lockRoutingReservationAdmissionSource(client, await loadPlan(planId, client), principal) const { plan, evidence } = selected reservedRouting = selected.reserved if (plan.connection_id !== connection.id) throw new McpError("NOT_FOUND", "План не найден", 404) const action = findActionByOperation(plan.operation_id) if (plan.policy_version !== connection.policy.version || plan.operation_version !== action.operation.version || plan.generation !== await accessGeneration() || plan.expires_at.getTime() <= Date.now()) throw new McpError("PLAN_EXPIRED", "План или его разрешения устарели", 409) if (!["draft", "approved"].includes(plan.state)) throw new McpError("APPROVAL_REQUIRED", "План требует подтверждения или недоступен", 409) for (const change of evidence.steps) { if (change.kind === "ros" && change.step.resource === "ip_settings" && change.step.fields["rp-filter"] === "strict") { checkRpfPolicy(change.step.serverId, evidence.snapshots, "strict") } } let scopedPathExpiresAt: number | undefined if (plan.operation_id === "routing.community.switch" && requiresScopedRoutingOwnerApproval(evidence)) { if (plan.approved_by !== connection.owner_id) throw new McpError("APPROVAL_REQUIRED", "Переключение постоянного профиля подтверждает владелец подключения", 409) scopedPathExpiresAt = await readScopedRoutingPathAdmissionDeadline(client, evidence, connection.id) } if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) { throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409) } if (evidence.recovery) { if (Date.now() >= Date.parse(evidence.recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Срок зарезервированного восстановления истёк", 409) const { rows: [grant] } = await client.query("SELECT config FROM mcp_routing_profiles WHERE id=$1 AND connection_id=$2 AND credential_id=$3 AND enabled AND grant_expires_at>now() AND policy_version=$4 AND generation=$5", [evidence.recovery.authorityProfileId, connection.id, principal.credentialId, connection.policy.version, await accessGeneration()]) const step = evidence.recovery.step if (!grant?.config?.autoRecovery || step.kind !== "service" || fingerprint(grant.config) !== fingerprint(step.input.profile)) throw new McpError("PERMISSION_DENIED", "Допуск восстановления изменён или отозван") } const input = await policyInput(client, connection, action.operation.id, evidence.target, evidence) input.temporaryGrant = await temporaryScope(client, connection, principal.credentialId) // tools/call already reserved its slot. A full minute bucket must not reject // the very call that legitimately consumed the last slot. input.usage.callsInMinute = Math.max(0, input.usage.callsInMinute - 1) const decision = evaluateMcpPolicy(input) if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; ")) if (decision.decision === "approval-required" && plan.state !== "approved") throw new McpError("APPROVAL_REQUIRED", "Требуется подтверждение человеком", 409) // Whole-operation provenance pins the ORIGINAL admission grant for ROS and // service plans alike; a later runner must never adopt a replacement grant. { const grantWall=Date.now(),grantMonotonic=performance.now() const grant=await readMcpTemporaryGrantOrigin(client,connection.id,principal.credentialId) const admissionExpiresAt = Math.min(plan.expires_at.getTime(), evidence.validUntil ? Date.parse(evidence.validUntil) : Infinity, evidence.recovery ? Date.parse(evidence.recovery.validUntil) : Infinity, Date.parse(principal.credentialExpiresAt), Date.parse(connection.policy.expiresAt), connection.owner_valid_until.getTime(), grant.expiresAt ?? Infinity, scopedPathExpiresAt ?? Infinity) originalServiceGrant=Object.freeze({...grant,monotonicEnd:grant.expiresAt===null?Infinity:grantMonotonic+Math.max(0,grant.expiresAt-grantWall),admissionExpiresAt,admissionMonotonicEnd:grantMonotonic+Math.max(0,admissionExpiresAt-grantWall)}) await assertOriginalMcpTemporaryGrant(client,connection.id,principal.credentialId,originalServiceGrant) } const mutationCost = reservedRouting ? 2 : changeMutationCost(evidence) + (evidence.recovery ? 1 : 0) if (input.usage.mutationsInHour + mutationCost > connection.policy.limits.mutationsPerHour) throw new McpError("BUDGET_EXCEEDED", "Недостаточно бюджета для всех шагов плана", 429) if (!reservedRouting) await client.query("INSERT INTO mcp_budget_buckets(connection_id,kind,bucket,used) VALUES($1,'mutations',date_trunc('hour',now()),$2) ON CONFLICT(connection_id,kind,bucket) DO UPDATE SET used=mcp_budget_buckets.used+EXCLUDED.used", [connection.id, mutationCost]) const inserted = await insertMcpOperationAdmission(client, operationAdmissionStart, planId, idempotencyKey, originalServiceGrant) const row = inserted.operation, id = row.id operationAdmission = inserted.admission; originalServiceGrant = inserted.originalGrant if (plan.operation_id === "routing.community.switch" || plan.operation_id === "operations.recover") { bindRoutingChargeAdmission(client, connection.id, principal.credentialId, inserted.originalGrant) } await client.query("UPDATE mcp_plans SET state='executing' WHERE id=$1", [planId]) if (reservedRouting) await reserveRouting(id) else if (plan.operation_id === "routing.community.switch") await admitPersistentRoutingForwardWithinOwner(client, id) else if (plan.operation_id === "operations.recover") await admitManualInverseWithinOwner(client, id, plan, evidence, principal, (profileId, current, observedAt, deadlineAt) => readHistoricalChargesWithinOwner(client, profileId, current, observedAt, deadlineAt)) await audit(client, connection.owner_id, "operation.start", connection.id, { operationId: id, planId }) return row }).catch(async error => { await retireMcpOperationAdmission(operationAdmission); throw error }) const operation = admitted.value // Reserved native dispatch remains closed until the common engine's bounded // network verifier and complete positive/negative lifecycle are accepted. // The atomic hold is durable; neither read nor another apply mints its cap. if (operation.state === "pending" && !reservedRouting && originalServiceGrant) { const operationKind = (await loadPlan(operation.plan_id)).operation_id const terminal = operationKind === "database.restore" setImmediate(() => { void (terminal || operationKind === "database.backup" ? runBounded(() => terminal ? import("./database-restore.js").then(module => module.executeMcpDatabaseRestore(operation.id, principal,originalServiceGrant)) : import("../../services/system-database-backup.js").then(module => module.withSystemDatabaseBackupAdmission(() => executeOperation(operation.id, principal,undefined,undefined,originalServiceGrant,operationAdmission)))) : runOrdinaryOperation(() => executeOperation(operation.id, principal, admitted.reservation?.forwardAllocation, admitted.reservation?.reservationId,originalServiceGrant,operationAdmission))).catch(async () => { if (!terminal) await retireMcpOperationAdmission(operationAdmission) /* no retry; interrupted intents retain explicit UNKNOWN */ }) }) } return { ...operation, result: stripMcpOperationExecutionPrivateResult(operation.result) } } async function executeOperation(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant, operationAdmission?: McpOperationAdmission): Promise { return withMcpOperationExecutionOrigin(operationAdmission, async () => { const lease = await pool.connect() let terminalProbe: Awaited> let locked = false, released = false const ownerLifetime = new AbortController() // This closure is the SINGLE owner of physical destruction/release. const retire = () => { if (released) return released = true ownerLifetime.abort() try { lease.release(true) } catch { /* Never reuse an uncertain lock session. */ } } const onError = () => retire() lease.on("error", onError) try { try { locked = (await lease.query<{ locked: boolean }>("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + operationId])).rows[0].locked } catch (error) { // The backend may hold the lock despite a rejected/lost acquire ACK. // An error event is not required to make healthy reuse unsafe. retire() throw error } if (released || !locked) return terminalProbe = await withOperationConfigurationLockClient(lease, retire, ownerLifetime.signal, () => executeOperationWithLease(operationId, principal, routingReservationForward, routingReservationId, originalServiceGrant), mcpDatabaseRequestSignal()) } finally { if (!released && locked) { try { await lease.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", ["mcp-operation:" + operationId]) } catch { retire() } } if (!released) { // Keep the absorber on a destroyed client; remove only before a healthy // release after EVERY borrowed lock facade/query has settled. lease.removeListener("error", onError) released = true lease.release() } } // Capture has its own bounded checkout and may fail without rewriting the // acknowledged terminal outcome or redispatching the diagnostic. if (terminalProbe) { try { await capturePublishedServerProbeTerminalProof(terminalProbe) } catch { /* Existing retention remains authoritative. */ } } }) } async function executeOperationWithLease(operationId: string, principal: McpPrincipal, routingReservationForward?: RoutingReservedForwardAllocation, routingReservationId?: string,originalServiceGrant?:OriginalServiceGrant) { const { rows: [operation] } = await pool.query("SELECT * FROM mcp_operations WHERE id=$1 AND state='running'", [operationId]) if (!operation) return const plan = await loadPlan(operation.plan_id), evidence = JSON.parse(await decryptSecret(plan.evidence.sealed)) as ChangeEvidence const serverIds = evidence.target.scope === "servers" ? evidence.target.serverIds : evidence.objects.flatMap(object => object.serverId ? [object.serverId] : []) let attempted = false, confirmed = 0, activeServiceStep = false let ecmpStoppedBeforeDispatch = false let vrfDiagnosticFallback: ReturnType | undefined let reservedForwardPassed = false let reservedRecoverySummary: Record | undefined const diagnosticOnly = evidence.steps.length === 1 && evidence.steps.every(step => step.kind === "service" && findMcpService(step.service)?.cancellableDiagnostic === true) const productionVrf = evidence.vrfProductionAdmission === true let diagnosticReceipt: ReturnType const authorizeGrant = async () => { assertOperationConfigurationLockScope() if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась; подготовьте новый план", 409) const connection = await validatePrincipal(principal) assertOperationConfigurationLockScope() if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась во время операции", 409) return authorizeAfterPrincipal(connection) } const authorizeAfterPrincipal = async (connection: ConnectionRow, policyLoan?: RuntimePollPolicyLoan) => { assertOperationConfigurationLockScope() if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась; подготовьте новый план", 409) if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась во время операции", 409) const checkPolicy = async (client: Pick) => { assertOperationConfigurationLockScope() if(evidence.steps.some(step=>step.kind==="service")){ await assertOriginalMcpTemporaryGrant(client,connection.id,principal.credentialId,originalServiceGrant) assertOperationConfigurationLockScope() } const pollSource = plan.operation_id === "servers.poll" ? await readPollRuntimePolicySource(client, connection.id, principal.credentialId) : undefined assertOperationConfigurationLockScope() const policy = await policyInput(client, connection, plan.operation_id, evidence.target, evidence, pollSource) assertOperationConfigurationLockScope() policy.temporaryGrant = pollSource ? pollSource.temporaryGrant : await temporaryScope(client, connection, principal.credentialId) assertOperationConfigurationLockScope() // Budget was reserved for the complete batch at admission. policy.usage.callsInMinute = 0; policy.usage.mutationsInHour = 0 const decision = evaluateMcpPolicy(policy) if (decision.decision === "denied") throw new McpError(decision.reasons[0].code, decision.reasons.map(reason => reason.message).join("; ")) if (decision.decision === "approval-required" && !plan.approved_by) throw new McpError("APPROVAL_REQUIRED", "Изменение теперь требует подтверждения", 409) } if (policyLoan) await executeRuntimePollPolicyTransaction(policyLoan, checkPolicy) else await transaction(checkPolicy) assertOperationConfigurationLockScope() await verifyRoutingObservationChecks(evidence.observationChecks) assertOperationConfigurationLockScope() if(evidence.steps.some(step=>step.kind==="service"))await assertOriginalMcpTemporaryGrant(pool,connection.id,principal.credentialId,originalServiceGrant) assertOperationConfigurationLockScope() return connection } const authorize = async () => { assertOperationConfigurationLockScope() const connection = await authorizeGrant() assertOperationConfigurationLockScope() await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant) assertOperationConfigurationLockScope() return connection } let ownerSources: (() => Promise) | undefined const authorizeCatalog = async () => { assertOperationConfigurationLockScope() const connection = await authorizeGrant() assertOperationConfigurationLockScope() ownerSources ??= createOspfOwnerSourceGuard(evidence.ospfOwnerGraph, serverIds) await ownerSources() assertOperationConfigurationLockScope() return connection } try { if (productionVrf && (evidence.steps.length !== 1 || evidence.steps[0].kind !== "ros" || evidence.steps[0].step.resource !== "vrf")) throw new McpError("STATE_CONFLICT", "VRF execution требует exact single typed step", 409) if (await routingReservationEvidence(plan, evidence) && (!routingReservationForward || !routingReservationId)) throw new McpError("STATE_CONFLICT", "No confirmed reserved forward admission", 409) if (plan.operation_version !== findActionByOperation(plan.operation_id).operation.version) throw new McpError("PLAN_EXPIRED", "Версия операции изменилась до запуска", 409) const checksResolvedNextHop = evidence.healthChecks?.some(check => check.field === "resolved") === true if (plan.operation_id === "operations.recover" && !checksResolvedNextHop && evidence.steps.some(change => change.kind === "ros" && ["ipv4_route", "ipv6_route"].includes(change.step.resource))) { const { operationId: sourceId } = JSON.parse(await decryptSecret(plan.payload.sealed)) as { operationId: string } const { rows: [source] } = await pool.query("SELECT p.operation_id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=$1 AND o.connection_id=$2", [sourceId, plan.connection_id]) if (!source || source.operation_id === "routing.ecmp.update") throw new McpError("PLAN_EXPIRED", "Подготовьте новый обратный план с проверкой next-hop", 409) } const applyWithLocks = () => withConfigMutationLock(serverIds, async () => { await authorize() const prefix = evidence.steps.some(step => step.kind === "ros") ? createRosPrefix(productionVrf ? evidence.snapshots.filter(snapshot => !isVrfCompanionResource(snapshot.resource)) : evidence.snapshots) : undefined for (const change of evidence.steps) { if (change.kind === "ros" && change.step.resource === "ip_settings" && change.step.fields["rp-filter"] === "strict") { checkRpfPolicy(change.step.serverId, evidence.snapshots, "strict") } } if (prefix) { // An ordinary single-step plan has no device observation before its pre-intent check. const needsEntrySnapshot = productionVrf || evidence.steps.length !== 1 || evidence.healthChecks?.length || evidence.managementChannels?.length || evidence.observationChecks?.length || evidence.ospfOwnerGraph || evidence.recovery if (needsEntrySnapshot) await prefix.check(authorizeCatalog) } else for (const snapshot of evidence.snapshots) { if ((await readRosSnapshot(snapshot.serverId, snapshot.resource)).fingerprint !== snapshot.fingerprint) throw new McpError("STATE_CONFLICT", "Конфигурация изменилась после подготовки", 409) } await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog) for (const step of evidence.steps) { if (step.kind === "service" && fingerprint(await serviceCurrent(step.service, step.input)) !== fingerprint(evidence.before)) throw new McpError("STATE_CONFLICT", "Данные сервиса изменились", 409) } // Deterministic one-server stages. Each successful write is verified before the next. for (let index = 0; index < evidence.steps.length; index++) { if (evidence.recovery && Date.now() >= Date.parse(evidence.recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Срок гарантированного восстановления истёк до записи", 409) for (const channel of evidence.managementChannels ?? []) { if (!serverIds.includes(channel.serverId)) throw new McpError("SERVER_DENIED", "Management-проверка вне плана") await verifyManagementChannel(channel.serverId, channel.channelId) } if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409) const connection = await authorize() const step = evidence.steps[index] let vrfBeforeDispatch: (() => Promise) | undefined if (step.kind === "ros") { await prefix!.check(authorizeCatalog) const before = productionVrf ? evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId) : prefix!.before(step.step) if (!before) throw new McpError("STATE_CONFLICT", "Нет исходного VRF source snapshot", 409) await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,$2,$3)", [operationId, index, await encryptSecret(JSON.stringify(before))]) } else { await pool.query("INSERT INTO mcp_operation_checkpoints(operation_id,step_index,before_sealed) VALUES($1,$2,$3)", [operationId, index, await encryptSecret(JSON.stringify(evidence.snapshots))]) } await pool.query("INSERT INTO mcp_operation_steps(operation_id,step_index,server_id,state,intended_state) VALUES($1,$2,$3,'intent',$4::jsonb)", [operationId, index, step.kind === "ros" ? step.step.serverId : null, JSON.stringify(redact(step))]) attempted = true activeServiceStep = step.kind === "service" const serviceExecution=step.kind==="service"?{operationId,planId:plan.id,operationName:plan.operation_id,serviceName:step.service,stepIndex:index,phase:"forward" as const, operationVersion:plan.operation_version,policyVersion:plan.policy_version,generation:plan.generation,principal:{...principal}, originalTemporaryGrantProof:originalServiceGrant!.proof,originalTemporaryGrantExpiresAt:originalServiceGrant!.expiresAt, sealedEvidenceProof:createHash("sha256").update(plan.evidence.sealed).digest("hex"),sealedPayloadProof:createHash("sha256").update(plan.payload.sealed).digest("hex"), originalExpiresAt:Math.min(plan.expires_at.getTime(),evidence.validUntil?Date.parse(evidence.validUntil):Infinity,Date.parse(principal.credentialExpiresAt),Date.parse(connection.policy.expiresAt),connection.owner_valid_until.getTime(),originalServiceGrant!.admissionExpiresAt), serviceInput:step.input}:undefined const reservedStep: { outcome?: ReservedRoutingStepOutcome } = {} vrfDiagnosticFallback = productionVrf && step.kind === "ros" && step.step.resource === "vrf" && ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id) ? createVrfRuntimeDiagnosticBoundary(operationId, step.step.action === "update" ? "patch" : step.step.action, diagnostic => { console.error(JSON.stringify(diagnostic)) }) : undefined const beforeDispatch = step.kind === "ros" ? async () => { await prefix!.check(authorizeCatalog) if (checksResolvedNextHop) { try { await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog) } catch (error) { ecmpStoppedBeforeDispatch = true throw error } } await authorize() await authorizeCatalog() if (productionVrf) { if (!vrfBeforeDispatch) throw new McpError("STATE_CONFLICT", "VRF command guard отсутствует", 409) await vrfBeforeDispatch() } } : undefined const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, beforeDispatch }, async () => { vrfDiagnosticFallback?.executorEntered() if (step.kind === "ros") { if (!productionVrf) return executeRosStep(step.step) const before = evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId) if (!before?.sourceProof) throw new McpError("STATE_CONFLICT", "Нет VRF source-v3 proof", 409) if (!["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id)) throw new McpError("STATE_CONFLICT", "VRF action binding изменён", 409) const receipt = await executeVrfProductionStep({ operationId, planId: plan.id, connectionId: connection.id, ownerId: connection.owner_id, operationName: plan.operation_id as "ros.vrf.create" | "ros.vrf.update" | "ros.vrf.delete" | "routing.vrf.update", operationVersion: plan.operation_version, stepIndex: index, step: step.step, sourceProof: before.sourceProof, originalSnapshots: evidence.snapshots, authorize: authorizeCatalog, assertDependencies: () => prefix!.check(authorizeCatalog), installBeforeDispatch: guard => { vrfBeforeDispatch = guard }, onDiagnostic: diagnostic => { vrfDiagnosticFallback?.observed(diagnostic) } }) vrfDiagnosticFallback?.executorReturned() return receipt } const forward = (retainRoutingNetworkSession?: (session: RoutingReservedNetworkSession) => void) => serviceExecute(step.service, step.input, { operationId, phase: "forward", stepIndex: index, ...(routingReservationForward ? { routingReservationForward, retainRoutingNetworkSession } : {}) }) if (!routingReservationId) return step.service === "servers.poll" ? withRuntimePollPrincipalSource(principal, async (current, policyLoan) => { assertOperationConfigurationLockScope() await authorizeAfterPrincipal(current, policyLoan) assertOperationConfigurationLockScope() await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant) assertOperationConfigurationLockScope() }, forward) : forward() reservedStep.outcome = await executeReservedRoutingStep({ reservationId: routingReservationId, sourceOperationId: operationId, principal, forward }) return reservedStep.outcome.forwardResult }) vrfDiagnosticFallback = undefined if (reservedStep.outcome?.kind === "recovered") { // The generic source remains intent/result=null until the private // inverse has finished. Record the forward ACK, never B VERIFIED // after restoring A. No second operation or mutation debit is added. confirmed++ reservedRecoverySummary = { state: "verified", mode: "same-slot", outcome: "recovered", nativeWrites: 1, configurationVerified: true, controlProbeVerified: true, forwardingVerified: false, recoveryCheckpointProof: reservedStep.outcome.recoveryResult.recoveryCheckpointProof } await pool.query("UPDATE mcp_operation_steps SET state='applied',result=$3::jsonb,finished_at=now() WHERE operation_id=$1 AND step_index=$2", [operationId, index, JSON.stringify(redact(result ?? null))]) throw new McpError("VERIFICATION_FAILED", "Forward не прошёл сетевую проверку; исходная конфигурация восстановлена и проверена", 409) } if (reservedStep.outcome?.kind === "forwardPassed") reservedForwardPassed = true if (diagnosticOnly) { diagnosticReceipt = serviceDiagnosticReceipt(result, true, operationId, evidence.mutationCost) if (!diagnosticReceipt) throw new McpError("VERIFICATION_FAILED", "Квитанция этапов диагностики не подтверждена", 409) } confirmed++ await pool.query("UPDATE mcp_operation_steps SET state='applied',result=$3::jsonb,finished_at=now() WHERE operation_id=$1 AND step_index=$2", [operationId, index, JSON.stringify(redact(result ?? null))]) if (step.kind === "service") { const after = await Promise.all(evidence.snapshots.map(snapshot => readRosSnapshot(snapshot.serverId, snapshot.resource))) await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))]) } if (step.kind === "ros" && productionVrf) { await prefix!.check(authorizeCatalog) } else if (step.kind === "ros") { const after = await readRosSnapshot(step.step.serverId, step.step.resource, authorizeCatalog, prefix!.before(step.step).sourceProof) await prefix!.advance(step.step, result, after, authorizeCatalog) await verifyOspfOwnerGraphAtPhase(evidence.ospfOwnerGraph, serverIds, authorizeGrant, "post-ack") await authorizeCatalog() await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))]) } else if (!await verifyStep(step, result)) throw new McpError("VERIFICATION_FAILED", "Запись подтверждена, но проверка результата не прошла", 409) await pool.query("UPDATE mcp_operation_steps SET state=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, evidence.verificationScope === "acknowledgement" ? "acknowledged" : "verified"]) } await verifyHealthChecks(evidence, serverIds, true, authorizeCatalog) await prefix?.check(authorizeCatalog) if (evidence.ospfOwnerGraph) { await authorizeGrant() await verifyOspfOwnerGraphAtPhase(evidence.ospfOwnerGraph, serverIds, authorizeGrant, confirmed > 0 ? "post-ack" : "admission") await authorizeCatalog() } }) await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks()) assertOperationConfigurationLockCheckpoint() const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined const finishedAt = await transaction(async client => { assertOperationConfigurationLockCheckpoint() const terminal = await client.query<{ finished_at: string }>("UPDATE mcp_operations SET state='applied',verification=$3,result=$2::jsonb,finished_at=now() WHERE id=$1" + (receiptId ? ` RETURNING to_char(finished_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS finished_at` : ""), [operationId, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, verificationScope: evidence.verificationScope ?? "configuration", ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }), evidence.verificationScope === "acknowledgement" ? "inconclusive" : "passed"]) assertOperationConfigurationLockCheckpoint() await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id]) assertOperationConfigurationLockCheckpoint() await audit(client, "mcp-runner", "operation.complete", operation.connection_id, { operationId, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...(diagnosticOnly ? { completedServiceSteps: confirmed, diagnosticReceipt } : {}) }) assertOperationConfigurationLockCheckpoint() return receiptId ? terminal.rows[0]?.finished_at : undefined }) assertOperationConfigurationLockCheckpoint() if (routingReservationId && reservedForwardPassed) { // Requires the real terminal applied + generic VERIFIED source. Failure // conservatively retains the held inverse and both original charges. await createRoutingReservationStore().proveUnusedInverse(routingReservationId).catch(() => {}) } assertOperationConfigurationLockCheckpoint() if (receiptId && finishedAt) return { receiptId, operationId, connectionId:operation.connection_id, finishedAt } } catch (error) { vrfDiagnosticFallback?.failure(error) if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1) if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites) const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE" const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost) const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost) const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" && (error as { diagnosticNotStarted?: unknown } | null)?.diagnosticNotStarted === true && diagnosticProgress?.state === "cancelled" && diagnosticProgress.attemptedStages === 0 && diagnosticProgress.completedStages === 0 const state = ecmpStoppedBeforeDispatch ? confirmed === 0 ? "rejected" : "partial" : !attempted || diagnosticNotStarted ? "rejected" : code === "VERIFICATION_FAILED" ? "partial" : "unknown" const dispatchOutcome = ecmpStoppedBeforeDispatch ? { failedStepDispatched: false } : {} // Private operator STDERR only; never HTTP/result/audit payload. try { const [step] = evidence.steps const manualRouting = !attempted && plan.operation_id === "routing.community.switch" && evidence.steps.length === 1 && step?.kind === "service" && step.service === "routing.community.scoped" && step.input.proofRevision === 2 const diagnostic = manualRouting ? manualRoutingPredispatchDiagnostic(error, operationId) : pollErrorLog(error, operationId, plan.operation_id) if (diagnostic) console.error(JSON.stringify(diagnostic)) } catch { /* Logging cannot replace operation outcome. */ } const recovery = reservedRecoverySummary ?? (code === "VERIFICATION_FAILED" && confirmed === evidence.steps.length && evidence.recovery ? await performReservedRecovery(operation, plan, evidence, principal, serverIds) : undefined) const persistFailure = mcpDatabaseRequestSignal()?.aborted ? requestAccountingWrite : transaction await persistFailure(async client => { await client.query("UPDATE mcp_operations SET state=$2,verification=$3,error_code=$4,result=$5::jsonb,finished_at=now() WHERE id=$1", [operationId, state, code === "VERIFICATION_FAILED" ? "failed" : "inconclusive", code, JSON.stringify({ confirmedWrites: diagnosticOnly ? 0 : confirmed, needsReview: state !== "rejected", ...dispatchOutcome, recovery, ...(nativeProgress ? { nativeProgress } : {}), ...(diagnosticProgress ? { diagnosticProgress } : {}), ...(diagnosticOnly ? { completedServiceSteps: confirmed } : {}) })]) await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [plan.id]) await audit(client, "mcp-runner", "operation.stop", operation.connection_id, { operationId, state, code, confirmedWrites: diagnosticOnly ? 0 : confirmed, ...dispatchOutcome, ...(nativeProgress ? { nativeProgress } : {}), ...(diagnosticProgress ? { diagnosticProgress } : {}) }) }) } } async function performReservedRecovery(operation: StoredOperation, plan: StoredPlan, evidence: ChangeEvidence, principal: McpPrincipal, serverIds: number[]) { const recovery = evidence.recovery! let attempted = false try { return await withApplicationMutationLock(() => withConfigMutationLock(serverIds, async () => { if (Date.now() >= Date.parse(recovery.validUntil)) throw new McpError("PLAN_EXPIRED", "Зарезервированный срок восстановления истёк", 409) const connection = await validatePrincipal(principal) if (connection.policy.version !== plan.policy_version) throw new McpError("STATE_CONFLICT", "Политика изменилась; допуск восстановления недействителен", 409) const { rows: [grant] } = await pool.query("SELECT config FROM mcp_routing_profiles WHERE id=$1 AND connection_id=$2 AND credential_id=$3 AND enabled AND grant_expires_at>now() AND policy_version=$4 AND generation=$5", [recovery.authorityProfileId, connection.id, principal.credentialId, connection.policy.version, await accessGeneration()]) if (!grant?.config?.autoRecovery || recovery.step.kind !== "service" || recovery.step.service !== "routing.community.restore") throw new McpError("PERMISSION_DENIED", "Нет действующего допуска этого восстановления") const recoveryProfile = recovery.step.input.profile if (fingerprint(grant.config) !== fingerprint(recoveryProfile)) throw new McpError("STATE_CONFLICT", "Профиль восстановления изменился", 409) await transaction(async client => { const input = await policyInput(client, connection, plan.operation_id, evidence.target, evidence) input.temporaryGrant = await temporaryScope(client, connection, principal.credentialId) input.usage.callsInMinute = 0; input.usage.mutationsInHour = 0 const decision = evaluateMcpPolicy(input) if (decision.decision === "denied" || decision.decision === "approval-required" && !plan.approved_by) throw new McpError("PERMISSION_DENIED", "Текущая защита или допуск запрещает восстановление") }) for (const channel of evidence.managementChannels ?? []) await verifyManagementChannel(channel.serverId, channel.channelId) if (!evidence.managementChannels?.length) throw new McpError("APPROVAL_REQUIRED", "Восстановление требует независимого канала") for (const [index, step] of evidence.steps.entries()) { const { rows: [checkpoint] } = await pool.query("SELECT after_sealed FROM mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=$2", [operation.id, index]) if (!checkpoint?.after_sealed) throw new McpError("STATE_CONFLICT", "Нет подтверждённого снимка после исходного шага", 409) const raw = JSON.parse(await decryptSecret(checkpoint.after_sealed)) const snapshots = step.kind === "ros" ? [raw] : raw for (const snapshot of snapshots) if ((await readRosSnapshot(snapshot.serverId, snapshot.resource)).fingerprint !== snapshot.fingerprint) throw new McpError("STATE_CONFLICT", "Зависимость изменилась после записи; восстановление запрещено", 409) } if (fingerprint(await serviceCurrent(recovery.step.service, recovery.step.input)) !== fingerprint(recovery.expected)) throw new McpError("STATE_CONFLICT", "Конфигурация изменилась извне; восстановление остановлено", 409) const intent = await pool.query("INSERT INTO mcp_recovery_intents(operation_id,state,authority_profile_id) VALUES($1,'intent',$2) ON CONFLICT DO NOTHING RETURNING operation_id", [operation.id, recovery.authorityProfileId]) if (!intent.rowCount) throw new McpError("STATE_CONFLICT", "Восстановление уже начиналось и не повторяется", 409) await audit(pool, connection.owner_id, "recovery.intent", connection.id, { operationId: operation.id, authorityProfileId: recovery.authorityProfileId }) attempted = true const step = recovery.step const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: true }, () => serviceExecute(step.service, step.input, { operationId: operation.id, phase: "recovery" })) if (!await verifyStep(step, result)) throw new McpError("VERIFICATION_FAILED", "Проверка восстановления не прошла", 409) await pool.query("UPDATE mcp_recovery_intents SET state='verified',finished_at=now() WHERE operation_id=$1", [operation.id]) await audit(pool, connection.owner_id, "recovery.verified", connection.id, { operationId: operation.id }) return { state: "verified", authorityProfileId: recovery.authorityProfileId } })) } catch (error) { if (attempted) await pool.query("UPDATE mcp_recovery_intents SET state='unknown',finished_at=now() WHERE operation_id=$1 AND state='intent'", [operation.id]).catch(() => {}) return { state: attempted ? "unknown" : "blocked", errorCode: error instanceof McpError ? error.code : "STATE_UNAVAILABLE", automaticRetry: false } } } export async function recoverMcpOperations() { // A second backend must not interrupt a runner still holding its session lease. // A pending operation can still be queued behind maintenance. Its original // plan expiry bounds that wait across processes without granting a new lease. const interrupted = `o.created_at { const { rows: [lock] } = await client.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + row.id]) if (!lock.locked) return // Defer the whole transition when either row is busy. const { rows: [target] } = await client.query(`SELECT o.plan_id,o.connection_id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id WHERE o.id=$1 AND ${interrupted} FOR UPDATE OF o,p SKIP LOCKED`, [row.id]) if (!target) return await client.query("UPDATE mcp_operations SET state='unknown',verification='inconclusive',error_code='PROCESS_INTERRUPTED',finished_at=now() WHERE id=$1", [row.id]) await client.query("UPDATE mcp_plans SET state='completed' WHERE id=$1", [target.plan_id]) await audit(client, "mcp-recovery", "operation.interrupted", target.connection_id, { operationId: row.id, retryAllowed: false }) }) } await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS ( SELECT id FROM mcp_plans WHERE expires_at { const diagnostic = partitionWriteFailureDiagnostic(error) if (diagnostic) { try { console.error(JSON.stringify({ event: "mcp.worker.plan-expiry-failed", ...diagnostic })) } catch { /* Preserve the original transaction failure. */ } } throw error }) await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS ( SELECT id FROM mcp_read_leases WHERE expires_at client.query(`WITH interrupted AS ( SELECT intent.operation_id FROM mcp_recovery_intents intent JOIN mcp_operations operation ON operation.id=intent.operation_id WHERE intent.state='intent' AND operation.state NOT IN ('pending','running') ORDER BY intent.created_at,intent.operation_id LIMIT 1000 FOR UPDATE OF intent SKIP LOCKED ) UPDATE mcp_recovery_intents SET state='unknown',finished_at=now() FROM interrupted WHERE mcp_recovery_intents.operation_id=interrupted.operation_id`)) }