Files
MikrotikManager/backend/src/services/config-mutation-lock.ts
T
DenozordecandClaude Opus 5.5 a7d85fd0f1 fix(mcp): применение не теряет одобренный план из-за опроса сборщиков
Операция с сервисным шагом брала глобальную блокировку конфигурации неблокирующим try-lock. Пока сборщик трафика или uptime держал её в разделяемом режиме, рантайм получал обычный Error и записывал rejected/DEVICE_UNAVAILABLE: одобренный человеком план терялся (стенд IPsec 2026-11-21: 2 из 24 применений). Теперь рантайм ждёт блокировку до 30 с; занятость отдаётся кодом CONFIGURATION_BUSY/409 с прежним текстом; неожиданные исключения рантайма пишутся в журнал с классом и замаскированным сообщением. SQL-тест: разделяемое удержание, отказ без ожидания, отказ по сроку, вход после освобождения.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-11-21 16:05:24 +07:00

189 lines
11 KiB
TypeScript

import { AsyncLocalStorage } from "node:async_hooks"
import { setTimeout as delay } from "node:timers/promises"
import type { PoolClient } from "pg"
import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js"
import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js"
import { assertDatabaseRestoreQuarantineOpen, databaseRestoreQuarantineEnabled } from "../db/database-restore-quarantine.js"
import { McpError } from "../modules/mcp/security.js"
import { withActiveDatabaseRequest } from "../modules/mcp/database-request-lease.js"
import { assertOperationConfigurationLockScope, borrowOperationConfigurationLockClient, captureOperationConfigurationLockHandoff, operationConfigurationSignal, operationDatabaseRequestSignal } from "./operation-config-lock-lease.js"
type LockCompletion = { active: boolean; readonly completed: Promise<void> }
type ServerLockCompletion = LockCompletion & { ids: ReadonlySet<number>; readonly parent: ServerLockCompletion | undefined; children: Set<Promise<unknown>>; signal: AbortSignal }
const held = new AsyncLocalStorage<ServerLockCompletion>()
const applicationLease = new AsyncLocalStorage<LockCompletion>()
const collectorLease = new AsyncLocalStorage<LockCompletion>()
const applicationLockKey = "hashtextextended('mm-config:application',0)"
/** Another mutation holds the global configuration lock, or collectors hold it shared while they read routers.
* Same message and 409 as before; the code lets MCP report it instead of a generic device failure. */
export class ConfigurationBusyError extends McpError {
constructor() { super("CONFIGURATION_BUSY", "Глобальная конфигурация изменяется другой операцией", 409) }
}
export async function acquireApplicationMutationTransactionLock(client: Pick<PoolClient, "query">): Promise<void> {
assertOperationConfigurationLockScope()
const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`)
if (row?.acquired !== true) throw new ConfigurationBusyError()
}
export function configurationMutationSignal(): AbortSignal | undefined {
return held.getStore()?.signal
}
/** Read-only scheduler handoff; never exposes a client or admits arbitrary work. */
export function captureConfigurationLockHandoff(): Promise<void> | undefined {
const completions: Promise<void>[] = []
const application = applicationLease.getStore()
if (application) {
if (!application.active) throw new Error("Configuration lock scope has completed")
completions.push(application.completed)
}
const collector = collectorLease.getStore()
if (collector) {
if (!collector.active) throw new Error("Collector lock scope has completed")
completions.push(collector.completed)
}
for (let server = held.getStore(); server; server = server.parent) {
if (!server.active) throw new Error("Configuration lock scope has completed")
completions.push(server.completed)
}
return completions.length ? Promise.all(completions).then(() => {}) : undefined
}
export function assertNoConfigurationLockContext(): void {
if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context")
}
/** `waitMs` retries the non-blocking lock until the deadline: an approved MCP operation must not be lost to a
* collector that holds the lock shared for a few seconds (stand IPsec 2026-11-21: 2 of 24 applies refused). */
export async function withApplicationMutationLock<T>(work: () => Promise<T>, options: { waitMs?: number } = {}): Promise<T> {
assertOperationConfigurationLockScope()
if (applicationLease.getStore()?.active) return work()
if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 })
let complete!: () => void
const completed = new Promise<void>(resolve => { complete = resolve })
const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed }
let acquired = false, unsafe = false
try {
const deadline = Date.now() + Math.max(0, options.waitMs ?? 0)
for (;;) {
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired
if (acquired || Date.now() >= deadline) break
operationConfigurationSignal()?.throwIfAborted()
await delay(250)
}
if (!acquired) throw new ConfigurationBusyError()
return await applicationLease.run(context, work)
} finally {
context.active = false
if (acquired) try { await client.query(`SELECT pg_advisory_unlock(${applicationLockKey})`) } catch { unsafe = true }
try { client.release(unsafe) } finally { complete() }
}
}
/** Collectors coexist with each other but exclude configuration mutation for
* the complete native/sample/status operation. This lock grants no authority.
* Verified operations retain their original exact exclusive client protocol. */
export async function withApplicationCollectorLock<T>(work: () => Promise<T>): Promise<T> {
assertOperationConfigurationLockScope()
if (applicationLease.getStore()?.active || captureOperationConfigurationLockHandoff()) return withApplicationMutationLock(work)
if (collectorLease.getStore()?.active) return work()
let complete!: () => void
const completed = new Promise<void>(resolve => { complete = resolve })
const client = await pool.connect(), context = { active: true, completed }
let acquired = false, unsafe = false
try {
acquired = (await client.query(`SELECT pg_try_advisory_lock_shared(${applicationLockKey}) AS acquired`)).rows[0].acquired
if (!acquired) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 })
return await collectorLease.run(context, work)
} finally {
context.active = false
if (acquired) try { await client.query(`SELECT pg_advisory_unlock_shared(${applicationLockKey})`) } catch { unsafe = true }
try { client.release(unsafe) } finally { complete() }
}
}
/** Shared by UI, MCP and background tasks; sorted acquisition avoids cross-router deadlocks. */
export function withConfigMutationLock<T>(serverIds: number[], work: () => Promise<T>): Promise<T> {
const scope = held.getStore()
const parent = scope?.active ? scope : undefined
if (!parent) {
const admitted = withActiveDatabaseRequest(signal => runConfigMutationLock(serverIds, work, undefined, signal))
if (admitted) return admitted
}
const child = runConfigMutationLock(serverIds, work, parent)
if (parent) {
parent.children.add(child)
const settled = () => { parent.children.delete(child) }
void child.then(settled, settled)
}
return child
}
async function runConfigMutationLock<T>(serverIds: number[], work: () => Promise<T>, parent: ServerLockCompletion | undefined, requestSignal?: AbortSignal): Promise<T> {
assertOperationConfigurationLockScope()
const inherited = parent?.ids ?? new Set<number>()
const requested = [...new Set(serverIds)].sort((a, b) => a - b).filter(id => !inherited.has(id))
parent?.signal.throwIfAborted()
if (!requested.length) return work()
// Never extend an existing multi-router lease out of order.
if (inherited.size && requested[0] < Math.max(...inherited)) throw new Error("Некорректный порядок блокировок серверов")
const borrowed = borrowOperationConfigurationLockClient()
const owned = borrowed ? undefined : await pool.connect()
const client = borrowed ?? owned!
const acquired: number[] = []
let unsafe = false, databaseRequest = false
let complete!: () => void
const completed = new Promise<void>(resolve => { complete = resolve })
const lifetime = new AbortController()
const lostLease = new McpError("PROTOCOL_STORAGE_BUSY", "Соединение с блокировкой конфигурации потеряно", 503)
const inheritedSignal = parent?.signal ?? requestSignal ?? operationDatabaseRequestSignal()
const signal = inheritedSignal ? AbortSignal.any([inheritedSignal, lifetime.signal]) : lifetime.signal
const context: ServerLockCompletion = { ids: new Set([...inherited, ...requested]), active: true, completed, parent, children: new Set(), signal }
const disconnected = () => {
unsafe = true
lifetime.abort(lostLease)
}
const operationSignal = borrowed ? operationConfigurationSignal() : undefined
operationSignal?.addEventListener("abort", disconnected, { once: true })
owned?.on?.("error", disconnected)
owned?.on?.("end", disconnected)
try {
if (operationSignal?.aborted) disconnected()
signal.throwIfAborted()
// An ordinary MCP runner already holds its database request lease.
if (!borrowed && !parent && !requestSignal) {
unsafe = true
const { rows: [row] } = await client.query<{ acquired: boolean }>("SELECT pg_catalog.pg_try_advisory_lock_shared($1::bigint) AS acquired", [SYSTEM_DATABASE_REQUEST_LOCK])
signal.throwIfAborted()
if (typeof row?.acquired !== "boolean") throw new McpError("AUDIT_UNAVAILABLE", "Допуск записи RouterOS не подтверждён", 503)
unsafe = false
if (!row.acquired) throw new McpError("PROTOCOL_STORAGE_BUSY", "Выполняется операция с базой приложения", 503)
databaseRequest = true
if (databaseRestoreQuarantineEnabled()) await assertDatabaseRestoreQuarantineOpen(client)
}
for (const id of requested) {
const { rows: [row] } = await client.query<{ acquired: boolean }>("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS acquired", ["mm-config:" + id])
if (!row.acquired) throw Object.assign(new Error("Конфигурация сервера изменяется другой операцией"), { statusCode: 409 })
acquired.push(id)
signal.throwIfAborted()
}
assertRuntimeDatabaseActivationCurrent()
return await held.run(context, work)
} finally {
// Started nested work owns its parent's lease until it settles.
while (context.children.size) await Promise.allSettled(context.children)
context.active = false
for (const id of acquired.reverse()) {
if (lifetime.signal.aborted) break
try { await client.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", ["mm-config:" + id]) } catch { unsafe = true }
}
if (databaseRequest && !unsafe) {
try {
const { rows: [row] } = await client.query<{ released: boolean }>("SELECT pg_catalog.pg_advisory_unlock_shared($1::bigint) AS released", [SYSTEM_DATABASE_REQUEST_LOCK])
unsafe = row?.released !== true
} catch { unsafe = true }
}
owned?.off?.("error", disconnected)
owned?.off?.("end", disconnected)
operationSignal?.removeEventListener("abort", disconnected)
lifetime.abort(lostLease)
try { client.release(unsafe) } finally { complete() }
}
}