fix(mcp): применение не теряет одобренный план из-за опроса сборщиков

Операция с сервисным шагом брала глобальную блокировку конфигурации неблокирующим try-lock. Пока сборщик трафика или uptime держал её в разделяемом режиме, рантайм получал обычный Error и записывал rejected/DEVICE_UNAVAILABLE: одобренный человеком план терялся (стенд IPsec 2026-11-21: 2 из 24 применений). Теперь рантайм ждёт блокировку до 30 с; занятость отдаётся кодом CONFIGURATION_BUSY/409 с прежним текстом; неожиданные исключения рантайма пишутся в журнал с классом и замаскированным сообщением. SQL-тест: разделяемое удержание, отказ без ожидания, отказ по сроку, вход после освобождения.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
DenozordecandClaude Opus 5.5 committed 2026-11-21 16:05:24 +07:00
1 parent 2be3a207f6
commit a7d85fd0f1
3 files changed
+42 -5

No files matched your search

+7 -1
View File
@@ -63,6 +63,8 @@ import { lockRoutingReservationAdmissionSource, readScopedRoutingPathAdmissionDe
import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js" import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js"
let activeRunners = 0 let activeRunners = 0
// Collectors hold the global configuration lock shared for a few seconds per router read.
export const MCP_APPLY_LOCK_WAIT_MS = 30_000
const runnerQueue: Array<() => void> = [] const runnerQueue: Array<() => void> = []
async function runBounded(work: () => Promise<void>) { async function runBounded(work: () => Promise<void>) {
if (activeRunners >= 2) await new Promise<void>(resolve => runnerQueue.push(resolve)) if (activeRunners >= 2) await new Promise<void>(resolve => runnerQueue.push(resolve))
@@ -751,7 +753,8 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
await authorizeCatalog() await authorizeCatalog()
} }
}) })
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks()) // Collectors hold the global lock shared while they read routers; wait for them instead of losing the approved plan.
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks, { waitMs: MCP_APPLY_LOCK_WAIT_MS }) : applyWithLocks())
assertOperationConfigurationLockCheckpoint() assertOperationConfigurationLockCheckpoint()
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
const finishedAt = await transaction(async client => { const finishedAt = await transaction(async client => {
@@ -779,6 +782,9 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1) if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites) if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE" const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE"
// The generic fallback hid the cause of every unexpected failure; keep the class and a redacted message in the log.
if (code === "DEVICE_UNAVAILABLE" && !(error instanceof McpError)) console.error(JSON.stringify({ event: "mcp.operation.unexpected-error", operationId, operation: plan.operation_id,
errorClass: error instanceof Error ? error.constructor.name : typeof error, message: String(redact(error instanceof Error ? error.message : String(error))).slice(0, 300) }))
const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost) const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost)
const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost) const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost)
const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" && const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" &&
+18 -4
View File
@@ -1,4 +1,5 @@
import { AsyncLocalStorage } from "node:async_hooks" import { AsyncLocalStorage } from "node:async_hooks"
import { setTimeout as delay } from "node:timers/promises"
import type { PoolClient } from "pg" import type { PoolClient } from "pg"
import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js" import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js"
import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js" import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js"
@@ -13,11 +14,16 @@ const held = new AsyncLocalStorage<ServerLockCompletion>()
const applicationLease = new AsyncLocalStorage<LockCompletion>() const applicationLease = new AsyncLocalStorage<LockCompletion>()
const collectorLease = new AsyncLocalStorage<LockCompletion>() const collectorLease = new AsyncLocalStorage<LockCompletion>()
const applicationLockKey = "hashtextextended('mm-config:application',0)" const applicationLockKey = "hashtextextended('mm-config:application',0)"
/** Another mutation holds the global configuration lock, or collectors hold it shared while they read routers.
* Same message and 409 as before; the code lets MCP report it instead of a generic device failure. */
export class ConfigurationBusyError extends McpError {
constructor() { super("CONFIGURATION_BUSY", "Глобальная конфигурация изменяется другой операцией", 409) }
}
export async function acquireApplicationMutationTransactionLock(client: Pick<PoolClient, "query">): Promise<void> { export async function acquireApplicationMutationTransactionLock(client: Pick<PoolClient, "query">): Promise<void> {
assertOperationConfigurationLockScope() assertOperationConfigurationLockScope()
const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`) const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`)
if (row?.acquired !== true) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 }) if (row?.acquired !== true) throw new ConfigurationBusyError()
} }
export function configurationMutationSignal(): AbortSignal | undefined { export function configurationMutationSignal(): AbortSignal | undefined {
@@ -45,7 +51,9 @@ export function captureConfigurationLockHandoff(): Promise<void> | undefined {
export function assertNoConfigurationLockContext(): void { export function assertNoConfigurationLockContext(): void {
if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context") if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context")
} }
export async function withApplicationMutationLock<T>(work: () => Promise<T>): Promise<T> { /** `waitMs` retries the non-blocking lock until the deadline: an approved MCP operation must not be lost to a
* collector that holds the lock shared for a few seconds (stand IPsec 2026-11-21: 2 of 24 applies refused). */
export async function withApplicationMutationLock<T>(work: () => Promise<T>, options: { waitMs?: number } = {}): Promise<T> {
assertOperationConfigurationLockScope() assertOperationConfigurationLockScope()
if (applicationLease.getStore()?.active) return work() if (applicationLease.getStore()?.active) return work()
if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 }) if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 })
@@ -54,8 +62,14 @@ export async function withApplicationMutationLock<T>(work: () => Promise<T>): Pr
const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed } const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed }
let acquired = false, unsafe = false let acquired = false, unsafe = false
try { try {
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired const deadline = Date.now() + Math.max(0, options.waitMs ?? 0)
if (!acquired) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 }) for (;;) {
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired
if (acquired || Date.now() >= deadline) break
operationConfigurationSignal()?.throwIfAborted()
await delay(250)
}
if (!acquired) throw new ConfigurationBusyError()
return await applicationLease.run(context, work) return await applicationLease.run(context, work)
} finally { } finally {
context.active = false context.active = false
@@ -169,6 +169,23 @@ export async function acceptOperationLockLifetimeSql() {
}))) })))
assert(latest().errorEvents >= 1); await assertDestroyed() assert(latest().errorEvents >= 1); await assertDestroyed()
checks.push("independent admin terminates only the owned backend; real socket error aborts the owner, rejects inherited authorization and frees every advisory lock exactly once") checks.push("independent admin terminates only the owned backend; real socket error aborts the owner, rejects inherited authorization and frees every advisory lock exactly once")
// A collector holds the global configuration lock shared (as withApplicationCollectorLock does while it reads routers).
{
const { ConfigurationBusyError } = await import("./config-mutation-lock.js")
const collector = "SELECT pg_advisory_lock_shared(hashtextextended('mm-config:application',0))", collectorDone = "SELECT pg_advisory_unlock_shared(hashtextextended('mm-config:application',0))"
await admin.query(collector)
let entered = 0
await assert.rejects(withApplicationMutationLock(async () => { entered++ }), (error: unknown) => error instanceof ConfigurationBusyError && error.code === "CONFIGURATION_BUSY" && error.statusCode === 409)
const short = Date.now()
await assert.rejects(withApplicationMutationLock(async () => { entered++ }, { waitMs: 600 }), ConfigurationBusyError)
assert(Date.now() - short >= 550, "bounded wait gave up early")
const started = Date.now(), waiting = withApplicationMutationLock(async () => { entered++; return Date.now() - started }, { waitMs: 10000 })
await delay(700); assert.equal(entered, 0, "work entered while the collector held the lock")
await admin.query(collectorDone)
assert((await waiting) >= 650, "waiting mutation did not wait for the collector")
assert.equal(entered, 1)
checks.push("collector shared hold: no wait refuses CONFIGURATION_BUSY/409 at once, bounded wait refuses at its deadline, a waiting mutation enters only after the collector releases")
}
return { checks, cases: checks.length, poolMax: 4, nativeRequests: 0, applicationRowsWritten: 0, atomicOwnerBarrier: false } return { checks, cases: checks.length, poolMax: 4, nativeRequests: 0, applicationRowsWritten: 0, atomicOwnerBarrier: false }
} finally { } finally {
for (const resolve of releaseGates) resolve() for (const resolve of releaseGates) resolve()