fix(mcp): применение не теряет одобренный план из-за опроса сборщиков
Операция с сервисным шагом брала глобальную блокировку конфигурации неблокирующим try-lock. Пока сборщик трафика или uptime держал её в разделяемом режиме, рантайм получал обычный Error и записывал rejected/DEVICE_UNAVAILABLE: одобренный человеком план терялся (стенд IPsec 2026-11-21: 2 из 24 применений). Теперь рантайм ждёт блокировку до 30 с; занятость отдаётся кодом CONFIGURATION_BUSY/409 с прежним текстом; неожиданные исключения рантайма пишутся в журнал с классом и замаскированным сообщением. SQL-тест: разделяемое удержание, отказ без ожидания, отказ по сроку, вход после освобождения. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
2be3a207f6
commit
a7d85fd0f1
3 files changed
+42
-5
No files matched your search
@@ -63,6 +63,8 @@ import { lockRoutingReservationAdmissionSource, readScopedRoutingPathAdmissionDe
|
||||
import { executeReservedRoutingStep, hasConfirmedReservedRoutingForward, type ReservedRoutingStepOutcome } from "./routing-reserved-step.js"
|
||||
|
||||
let activeRunners = 0
|
||||
// Collectors hold the global configuration lock shared for a few seconds per router read.
|
||||
export const MCP_APPLY_LOCK_WAIT_MS = 30_000
|
||||
const runnerQueue: Array<() => void> = []
|
||||
async function runBounded(work: () => Promise<void>) {
|
||||
if (activeRunners >= 2) await new Promise<void>(resolve => runnerQueue.push(resolve))
|
||||
@@ -751,7 +753,8 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
await authorizeCatalog()
|
||||
}
|
||||
})
|
||||
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks) : applyWithLocks())
|
||||
// Collectors hold the global lock shared while they read routers; wait for them instead of losing the approved plan.
|
||||
await (evidence.steps.some(step => step.kind === "service") ? withApplicationMutationLock(applyWithLocks, { waitMs: MCP_APPLY_LOCK_WAIT_MS }) : applyWithLocks())
|
||||
assertOperationConfigurationLockCheckpoint()
|
||||
const receiptId = plan.operation_id === SERVER_PROBE_OPERATION ? diagnosticReceipt?.receiptId : undefined
|
||||
const finishedAt = await transaction(async client => {
|
||||
@@ -779,6 +782,9 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
if (routingReservationId && hasConfirmedReservedRoutingForward(error)) confirmed = Math.max(confirmed, 1)
|
||||
if (error instanceof VrfProductionAcknowledgedFailure) confirmed = Math.max(confirmed, error.acknowledgedWrites)
|
||||
const code = error instanceof VrfProductionClaimOutcomeUnknown ? "VRF_CLAIM_OUTCOME_UNKNOWN" : error instanceof McpError ? error.code : "DEVICE_UNAVAILABLE"
|
||||
// The generic fallback hid the cause of every unexpected failure; keep the class and a redacted message in the log.
|
||||
if (code === "DEVICE_UNAVAILABLE" && !(error instanceof McpError)) console.error(JSON.stringify({ event: "mcp.operation.unexpected-error", operationId, operation: plan.operation_id,
|
||||
errorClass: error instanceof Error ? error.constructor.name : typeof error, message: String(redact(error instanceof Error ? error.message : String(error))).slice(0, 300) }))
|
||||
const nativeProgress = serviceNativeProgress(error, activeServiceStep, evidence.mutationCost)
|
||||
const diagnosticProgress = serviceDiagnosticProgress(error, diagnosticOnly && activeServiceStep, operationId, evidence.mutationCost)
|
||||
const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" &&
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { AsyncLocalStorage } from "node:async_hooks"
|
||||
import { setTimeout as delay } from "node:timers/promises"
|
||||
import type { PoolClient } from "pg"
|
||||
import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js"
|
||||
import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js"
|
||||
@@ -13,11 +14,16 @@ const held = new AsyncLocalStorage<ServerLockCompletion>()
|
||||
const applicationLease = new AsyncLocalStorage<LockCompletion>()
|
||||
const collectorLease = new AsyncLocalStorage<LockCompletion>()
|
||||
const applicationLockKey = "hashtextextended('mm-config:application',0)"
|
||||
/** Another mutation holds the global configuration lock, or collectors hold it shared while they read routers.
|
||||
* Same message and 409 as before; the code lets MCP report it instead of a generic device failure. */
|
||||
export class ConfigurationBusyError extends McpError {
|
||||
constructor() { super("CONFIGURATION_BUSY", "Глобальная конфигурация изменяется другой операцией", 409) }
|
||||
}
|
||||
|
||||
export async function acquireApplicationMutationTransactionLock(client: Pick<PoolClient, "query">): Promise<void> {
|
||||
assertOperationConfigurationLockScope()
|
||||
const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`)
|
||||
if (row?.acquired !== true) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 })
|
||||
if (row?.acquired !== true) throw new ConfigurationBusyError()
|
||||
}
|
||||
|
||||
export function configurationMutationSignal(): AbortSignal | undefined {
|
||||
@@ -45,7 +51,9 @@ export function captureConfigurationLockHandoff(): Promise<void> | undefined {
|
||||
export function assertNoConfigurationLockContext(): void {
|
||||
if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context")
|
||||
}
|
||||
export async function withApplicationMutationLock<T>(work: () => Promise<T>): Promise<T> {
|
||||
/** `waitMs` retries the non-blocking lock until the deadline: an approved MCP operation must not be lost to a
|
||||
* collector that holds the lock shared for a few seconds (stand IPsec 2026-11-21: 2 of 24 applies refused). */
|
||||
export async function withApplicationMutationLock<T>(work: () => Promise<T>, options: { waitMs?: number } = {}): Promise<T> {
|
||||
assertOperationConfigurationLockScope()
|
||||
if (applicationLease.getStore()?.active) return work()
|
||||
if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 })
|
||||
@@ -54,8 +62,14 @@ export async function withApplicationMutationLock<T>(work: () => Promise<T>): Pr
|
||||
const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed }
|
||||
let acquired = false, unsafe = false
|
||||
try {
|
||||
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired
|
||||
if (!acquired) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 })
|
||||
const deadline = Date.now() + Math.max(0, options.waitMs ?? 0)
|
||||
for (;;) {
|
||||
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired
|
||||
if (acquired || Date.now() >= deadline) break
|
||||
operationConfigurationSignal()?.throwIfAborted()
|
||||
await delay(250)
|
||||
}
|
||||
if (!acquired) throw new ConfigurationBusyError()
|
||||
return await applicationLease.run(context, work)
|
||||
} finally {
|
||||
context.active = false
|
||||
|
||||
@@ -169,6 +169,23 @@ export async function acceptOperationLockLifetimeSql() {
|
||||
})))
|
||||
assert(latest().errorEvents >= 1); await assertDestroyed()
|
||||
checks.push("independent admin terminates only the owned backend; real socket error aborts the owner, rejects inherited authorization and frees every advisory lock exactly once")
|
||||
// A collector holds the global configuration lock shared (as withApplicationCollectorLock does while it reads routers).
|
||||
{
|
||||
const { ConfigurationBusyError } = await import("./config-mutation-lock.js")
|
||||
const collector = "SELECT pg_advisory_lock_shared(hashtextextended('mm-config:application',0))", collectorDone = "SELECT pg_advisory_unlock_shared(hashtextextended('mm-config:application',0))"
|
||||
await admin.query(collector)
|
||||
let entered = 0
|
||||
await assert.rejects(withApplicationMutationLock(async () => { entered++ }), (error: unknown) => error instanceof ConfigurationBusyError && error.code === "CONFIGURATION_BUSY" && error.statusCode === 409)
|
||||
const short = Date.now()
|
||||
await assert.rejects(withApplicationMutationLock(async () => { entered++ }, { waitMs: 600 }), ConfigurationBusyError)
|
||||
assert(Date.now() - short >= 550, "bounded wait gave up early")
|
||||
const started = Date.now(), waiting = withApplicationMutationLock(async () => { entered++; return Date.now() - started }, { waitMs: 10000 })
|
||||
await delay(700); assert.equal(entered, 0, "work entered while the collector held the lock")
|
||||
await admin.query(collectorDone)
|
||||
assert((await waiting) >= 650, "waiting mutation did not wait for the collector")
|
||||
assert.equal(entered, 1)
|
||||
checks.push("collector shared hold: no wait refuses CONFIGURATION_BUSY/409 at once, bounded wait refuses at its deadline, a waiting mutation enters only after the collector releases")
|
||||
}
|
||||
return { checks, cases: checks.length, poolMax: 4, nativeRequests: 0, applicationRowsWritten: 0, atomicOwnerBarrier: false }
|
||||
} finally {
|
||||
for (const resolve of releaseGates) resolve()
|
||||
|
||||
Reference in new issue
Block a user