Операция с сервисным шагом брала глобальную блокировку конфигурации неблокирующим try-lock. Пока сборщик трафика или uptime держал её в разделяемом режиме, рантайм получал обычный Error и записывал rejected/DEVICE_UNAVAILABLE: одобренный человеком план терялся (стенд IPsec 2026-11-21: 2 из 24 применений). Теперь рантайм ждёт блокировку до 30 с; занятость отдаётся кодом CONFIGURATION_BUSY/409 с прежним текстом; неожиданные исключения рантайма пишутся в журнал с классом и замаскированным сообщением. SQL-тест: разделяемое удержание, отказ без ожидания, отказ по сроку, вход после освобождения. Co-Authored-By: Claude Opus 5.5 <[email protected]>
189 lines
11 KiB
TypeScript
189 lines
11 KiB
TypeScript
import { AsyncLocalStorage } from "node:async_hooks"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import type { PoolClient } from "pg"
|
|
import { assertRuntimeDatabaseActivationCurrent, pool } from "../db/index.js"
|
|
import { SYSTEM_DATABASE_REQUEST_LOCK } from "../db/operation-lock.js"
|
|
import { assertDatabaseRestoreQuarantineOpen, databaseRestoreQuarantineEnabled } from "../db/database-restore-quarantine.js"
|
|
import { McpError } from "../modules/mcp/security.js"
|
|
import { withActiveDatabaseRequest } from "../modules/mcp/database-request-lease.js"
|
|
import { assertOperationConfigurationLockScope, borrowOperationConfigurationLockClient, captureOperationConfigurationLockHandoff, operationConfigurationSignal, operationDatabaseRequestSignal } from "./operation-config-lock-lease.js"
|
|
|
|
type LockCompletion = { active: boolean; readonly completed: Promise<void> }
|
|
type ServerLockCompletion = LockCompletion & { ids: ReadonlySet<number>; readonly parent: ServerLockCompletion | undefined; children: Set<Promise<unknown>>; signal: AbortSignal }
|
|
const held = new AsyncLocalStorage<ServerLockCompletion>()
|
|
const applicationLease = new AsyncLocalStorage<LockCompletion>()
|
|
const collectorLease = new AsyncLocalStorage<LockCompletion>()
|
|
const applicationLockKey = "hashtextextended('mm-config:application',0)"
|
|
/** Another mutation holds the global configuration lock, or collectors hold it shared while they read routers.
|
|
* Same message and 409 as before; the code lets MCP report it instead of a generic device failure. */
|
|
export class ConfigurationBusyError extends McpError {
|
|
constructor() { super("CONFIGURATION_BUSY", "Глобальная конфигурация изменяется другой операцией", 409) }
|
|
}
|
|
|
|
export async function acquireApplicationMutationTransactionLock(client: Pick<PoolClient, "query">): Promise<void> {
|
|
assertOperationConfigurationLockScope()
|
|
const { rows: [row] } = await client.query<{ acquired: boolean }>(`SELECT pg_try_advisory_xact_lock(${applicationLockKey}) AS acquired`)
|
|
if (row?.acquired !== true) throw new ConfigurationBusyError()
|
|
}
|
|
|
|
export function configurationMutationSignal(): AbortSignal | undefined {
|
|
return held.getStore()?.signal
|
|
}
|
|
/** Read-only scheduler handoff; never exposes a client or admits arbitrary work. */
|
|
export function captureConfigurationLockHandoff(): Promise<void> | undefined {
|
|
const completions: Promise<void>[] = []
|
|
const application = applicationLease.getStore()
|
|
if (application) {
|
|
if (!application.active) throw new Error("Configuration lock scope has completed")
|
|
completions.push(application.completed)
|
|
}
|
|
const collector = collectorLease.getStore()
|
|
if (collector) {
|
|
if (!collector.active) throw new Error("Collector lock scope has completed")
|
|
completions.push(collector.completed)
|
|
}
|
|
for (let server = held.getStore(); server; server = server.parent) {
|
|
if (!server.active) throw new Error("Configuration lock scope has completed")
|
|
completions.push(server.completed)
|
|
}
|
|
return completions.length ? Promise.all(completions).then(() => {}) : undefined
|
|
}
|
|
export function assertNoConfigurationLockContext(): void {
|
|
if (held.getStore() || applicationLease.getStore() || collectorLease.getStore()) throw new Error("Scheduler bootstrap inside configuration lock context")
|
|
}
|
|
/** `waitMs` retries the non-blocking lock until the deadline: an approved MCP operation must not be lost to a
|
|
* collector that holds the lock shared for a few seconds (stand IPsec 2026-11-21: 2 of 24 applies refused). */
|
|
export async function withApplicationMutationLock<T>(work: () => Promise<T>, options: { waitMs?: number } = {}): Promise<T> {
|
|
assertOperationConfigurationLockScope()
|
|
if (applicationLease.getStore()?.active) return work()
|
|
if (collectorLease.getStore()?.active) throw Object.assign(new Error("Изменение конфигурации внутри общего сбора запрещено"), { statusCode: 409 })
|
|
let complete!: () => void
|
|
const completed = new Promise<void>(resolve => { complete = resolve })
|
|
const client = borrowOperationConfigurationLockClient() ?? await pool.connect(), context = { active: true, completed }
|
|
let acquired = false, unsafe = false
|
|
try {
|
|
const deadline = Date.now() + Math.max(0, options.waitMs ?? 0)
|
|
for (;;) {
|
|
acquired = (await client.query(`SELECT pg_try_advisory_lock(${applicationLockKey}) AS acquired`)).rows[0].acquired
|
|
if (acquired || Date.now() >= deadline) break
|
|
operationConfigurationSignal()?.throwIfAborted()
|
|
await delay(250)
|
|
}
|
|
if (!acquired) throw new ConfigurationBusyError()
|
|
return await applicationLease.run(context, work)
|
|
} finally {
|
|
context.active = false
|
|
if (acquired) try { await client.query(`SELECT pg_advisory_unlock(${applicationLockKey})`) } catch { unsafe = true }
|
|
try { client.release(unsafe) } finally { complete() }
|
|
}
|
|
}
|
|
/** Collectors coexist with each other but exclude configuration mutation for
|
|
* the complete native/sample/status operation. This lock grants no authority.
|
|
* Verified operations retain their original exact exclusive client protocol. */
|
|
export async function withApplicationCollectorLock<T>(work: () => Promise<T>): Promise<T> {
|
|
assertOperationConfigurationLockScope()
|
|
if (applicationLease.getStore()?.active || captureOperationConfigurationLockHandoff()) return withApplicationMutationLock(work)
|
|
if (collectorLease.getStore()?.active) return work()
|
|
let complete!: () => void
|
|
const completed = new Promise<void>(resolve => { complete = resolve })
|
|
const client = await pool.connect(), context = { active: true, completed }
|
|
let acquired = false, unsafe = false
|
|
try {
|
|
acquired = (await client.query(`SELECT pg_try_advisory_lock_shared(${applicationLockKey}) AS acquired`)).rows[0].acquired
|
|
if (!acquired) throw Object.assign(new Error("Глобальная конфигурация изменяется другой операцией"), { statusCode: 409 })
|
|
return await collectorLease.run(context, work)
|
|
} finally {
|
|
context.active = false
|
|
if (acquired) try { await client.query(`SELECT pg_advisory_unlock_shared(${applicationLockKey})`) } catch { unsafe = true }
|
|
try { client.release(unsafe) } finally { complete() }
|
|
}
|
|
}
|
|
/** Shared by UI, MCP and background tasks; sorted acquisition avoids cross-router deadlocks. */
|
|
export function withConfigMutationLock<T>(serverIds: number[], work: () => Promise<T>): Promise<T> {
|
|
const scope = held.getStore()
|
|
const parent = scope?.active ? scope : undefined
|
|
if (!parent) {
|
|
const admitted = withActiveDatabaseRequest(signal => runConfigMutationLock(serverIds, work, undefined, signal))
|
|
if (admitted) return admitted
|
|
}
|
|
const child = runConfigMutationLock(serverIds, work, parent)
|
|
if (parent) {
|
|
parent.children.add(child)
|
|
const settled = () => { parent.children.delete(child) }
|
|
void child.then(settled, settled)
|
|
}
|
|
return child
|
|
}
|
|
|
|
async function runConfigMutationLock<T>(serverIds: number[], work: () => Promise<T>, parent: ServerLockCompletion | undefined, requestSignal?: AbortSignal): Promise<T> {
|
|
assertOperationConfigurationLockScope()
|
|
const inherited = parent?.ids ?? new Set<number>()
|
|
const requested = [...new Set(serverIds)].sort((a, b) => a - b).filter(id => !inherited.has(id))
|
|
parent?.signal.throwIfAborted()
|
|
if (!requested.length) return work()
|
|
// Never extend an existing multi-router lease out of order.
|
|
if (inherited.size && requested[0] < Math.max(...inherited)) throw new Error("Некорректный порядок блокировок серверов")
|
|
const borrowed = borrowOperationConfigurationLockClient()
|
|
const owned = borrowed ? undefined : await pool.connect()
|
|
const client = borrowed ?? owned!
|
|
const acquired: number[] = []
|
|
let unsafe = false, databaseRequest = false
|
|
let complete!: () => void
|
|
const completed = new Promise<void>(resolve => { complete = resolve })
|
|
const lifetime = new AbortController()
|
|
const lostLease = new McpError("PROTOCOL_STORAGE_BUSY", "Соединение с блокировкой конфигурации потеряно", 503)
|
|
const inheritedSignal = parent?.signal ?? requestSignal ?? operationDatabaseRequestSignal()
|
|
const signal = inheritedSignal ? AbortSignal.any([inheritedSignal, lifetime.signal]) : lifetime.signal
|
|
const context: ServerLockCompletion = { ids: new Set([...inherited, ...requested]), active: true, completed, parent, children: new Set(), signal }
|
|
const disconnected = () => {
|
|
unsafe = true
|
|
lifetime.abort(lostLease)
|
|
}
|
|
const operationSignal = borrowed ? operationConfigurationSignal() : undefined
|
|
operationSignal?.addEventListener("abort", disconnected, { once: true })
|
|
owned?.on?.("error", disconnected)
|
|
owned?.on?.("end", disconnected)
|
|
try {
|
|
if (operationSignal?.aborted) disconnected()
|
|
signal.throwIfAborted()
|
|
// An ordinary MCP runner already holds its database request lease.
|
|
if (!borrowed && !parent && !requestSignal) {
|
|
unsafe = true
|
|
const { rows: [row] } = await client.query<{ acquired: boolean }>("SELECT pg_catalog.pg_try_advisory_lock_shared($1::bigint) AS acquired", [SYSTEM_DATABASE_REQUEST_LOCK])
|
|
signal.throwIfAborted()
|
|
if (typeof row?.acquired !== "boolean") throw new McpError("AUDIT_UNAVAILABLE", "Допуск записи RouterOS не подтверждён", 503)
|
|
unsafe = false
|
|
if (!row.acquired) throw new McpError("PROTOCOL_STORAGE_BUSY", "Выполняется операция с базой приложения", 503)
|
|
databaseRequest = true
|
|
if (databaseRestoreQuarantineEnabled()) await assertDatabaseRestoreQuarantineOpen(client)
|
|
}
|
|
for (const id of requested) {
|
|
const { rows: [row] } = await client.query<{ acquired: boolean }>("SELECT pg_try_advisory_lock(hashtextextended($1,0)) AS acquired", ["mm-config:" + id])
|
|
if (!row.acquired) throw Object.assign(new Error("Конфигурация сервера изменяется другой операцией"), { statusCode: 409 })
|
|
acquired.push(id)
|
|
signal.throwIfAborted()
|
|
}
|
|
assertRuntimeDatabaseActivationCurrent()
|
|
return await held.run(context, work)
|
|
} finally {
|
|
// Started nested work owns its parent's lease until it settles.
|
|
while (context.children.size) await Promise.allSettled(context.children)
|
|
context.active = false
|
|
for (const id of acquired.reverse()) {
|
|
if (lifetime.signal.aborted) break
|
|
try { await client.query("SELECT pg_advisory_unlock(hashtextextended($1,0))", ["mm-config:" + id]) } catch { unsafe = true }
|
|
}
|
|
if (databaseRequest && !unsafe) {
|
|
try {
|
|
const { rows: [row] } = await client.query<{ released: boolean }>("SELECT pg_catalog.pg_advisory_unlock_shared($1::bigint) AS released", [SYSTEM_DATABASE_REQUEST_LOCK])
|
|
unsafe = row?.released !== true
|
|
} catch { unsafe = true }
|
|
}
|
|
owned?.off?.("error", disconnected)
|
|
owned?.off?.("end", disconnected)
|
|
operationSignal?.removeEventListener("abort", disconnected)
|
|
lifetime.abort(lostLease)
|
|
try { client.release(unsafe) } finally { complete() }
|
|
}
|
|
}
|