diff --git a/backend/src/modules/mcp/runtime.ts b/backend/src/modules/mcp/runtime.ts index 40239e0a..d2c4bebc 100644 --- a/backend/src/modules/mcp/runtime.ts +++ b/backend/src/modules/mcp/runtime.ts @@ -63,6 +63,8 @@ import { lockRoutingReservationAdmissionSource, readScopedRoutingPathAdmissionDe import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js" let activeRunners = 0 +// Collectors hold the global configuration lock shared for a few seconds per router read. +export const MCP_APPLY_LOCK_WAIT_MS = 30_000 const runnerQueue: Array<() => void> = [] async function runBounded(work: () => Promise) { if (activeRunners >= 2) await new Promise(resolve => runnerQueue.push(resolve)) @@ -751,7 +753,8 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin await authorizeCatalog() } }) - await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks()) + // Collectors hold the global lock shared while they read routers; wait for them instead of losing the approved plan. + await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks, { waitMs: MCP_APPLY_LOCK_WAIT_MS }) : applyWithLocks()) assertOperationConfigurationLockCheckpoint() const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined const finishedAt = await transaction(async client => { @@ -779,6 +782,9 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1) if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites) const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE" + // The generic fallback hid the cause of every unexpected failure; keep the class and a redacted message in the log. + if (code === "DEVICE_UNAVAILABLE" && !(error instanceof McpError)) console.error(JSON.stringify({ event: "mcp.operation.unexpected-error", operationId, operation: plan.operation_id, + errorClass: error instanceof Error ? error.constructor.name : typeof error, message: String(redact(error instanceof Error ? error.message : String(error))).slice(0, 300) })) const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost) const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost) const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" && diff --git a/backend/src/services/config-mutation-lock.ts b/backend/src/services/config-mutation-lock.ts index 48e61675..ee6b58d3 100644 --- a/backend/src/services/config-mutation-lock.ts +++ b/backend/src/services/config-mutation-lock.ts @@ -1,4 +1,5 @@ import { AsyncLocalStorage } from "node:async_hooks" +import { setTimeout as delay } from "node:timers/promises" import type { PoolClient } from "pg" import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js" import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js" @@ -13,11 +14,16 @@ const held = new AsyncLocalStorage() const applicationLease = new AsyncLocalStorage() const collectorLease = new AsyncLocalStorage() const applicationLockKey = "hashtextextended('mm-config:application',0)" +/** Another mutation holds the global configuration lock, or collectors hold it shared while they read routers. + * Same message and 409 as before; the code lets MCP report it instead of a generic device failure. */ +export class ConfigurationBusyError extends McpError { + constructor() { super("CONFIGURATION_BUSY", "Глобальная конфигурация изменяется другой операцией", 409) } +} export async function acquireApplicationMutationTransactionLock(client: Pick): Promise { assertOperationConfigurationLockScope() const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`) - if (row?.acquired !== true) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 }) + if (row?.acquired !== true) throw new ConfigurationBusyError() } export function configurationMutationSignal(): AbortSignal | undefined { @@ -45,7 +51,9 @@ export function captureConfigurationLockHandoff(): Promise | undefined { export function assertNoConfigurationLockContext(): void { if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context") } -export async function withApplicationMutationLock(work: () => Promise): Promise { +/** `waitMs` retries the non-blocking lock until the deadline: an approved MCP operation must not be lost to a + * collector that holds the lock shared for a few seconds (stand IPsec 2026-11-21: 2 of 24 applies refused). */ +export async function withApplicationMutationLock(work: () => Promise, options: { waitMs?: number } = {}): Promise { assertOperationConfigurationLockScope() if (applicationLease.getStore()?.active) return work() if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 }) @@ -54,8 +62,14 @@ export async function withApplicationMutationLock(work: () => Promise): Pr const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed } let acquired = false, unsafe = false try { - acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired - if (!acquired) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 }) + const deadline = Date.now() + Math.max(0, options.waitMs ?? 0) + for (;;) { + acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired + if (acquired || Date.now() >= deadline) break + operationConfigurationSignal()?.throwIfAborted() + await delay(250) + } + if (!acquired) throw new ConfigurationBusyError() return await applicationLease.run(context, work) } finally { context.active = false diff --git a/backend/src/services/operation-config-lock-lease.sql-helper.ts b/backend/src/services/operation-config-lock-lease.sql-helper.ts index 869556b1..21ee696c 100644 --- a/backend/src/services/operation-config-lock-lease.sql-helper.ts +++ b/backend/src/services/operation-config-lock-lease.sql-helper.ts @@ -169,6 +169,23 @@ export async function acceptOperationLockLifetimeSql() { }))) assert(latest().errorEvents >= 1); await assertDestroyed() checks.push("independent admin terminates only the owned backend; real socket error aborts the owner, rejects inherited authorization and frees every advisory lock exactly once") + // A collector holds the global configuration lock shared (as withApplicationCollectorLock does while it reads routers). + { + const { ConfigurationBusyError } = await import("./config-mutation-lock.js") + const collector = "SELECT pg_advisory_lock_shared(hashtextextended('mm-config:application',0))", collectorDone = "SELECT pg_advisory_unlock_shared(hashtextextended('mm-config:application',0))" + await admin.query(collector) + let entered = 0 + await assert.rejects(withApplicationMutationLock(async () => { entered++ }), (error: unknown) => error instanceof ConfigurationBusyError && error.code === "CONFIGURATION_BUSY" && error.statusCode === 409) + const short = Date.now() + await assert.rejects(withApplicationMutationLock(async () => { entered++ }, { waitMs: 600 }), ConfigurationBusyError) + assert(Date.now() - short >= 550, "bounded wait gave up early") + const started = Date.now(), waiting = withApplicationMutationLock(async () => { entered++; return Date.now() - started }, { waitMs: 10000 }) + await delay(700); assert.equal(entered, 0, "work entered while the collector held the lock") + await admin.query(collectorDone) + assert((await waiting) >= 650, "waiting mutation did not wait for the collector") + assert.equal(entered, 1) + checks.push("collector shared hold: no wait refuses CONFIGURATION_BUSY/409 at once, bounded wait refuses at its deadline, a waiting mutation enters only after the collector releases") + } return { checks, cases: checks.length, poolMax: 4, nativeRequests: 0, applicationRowsWritten: 0, atomicOwnerBarrier: false } } finally { for (const resolve of releaseGates) resolve()