fix(mcp): проверять права раз на этап, читать каталоги параллельно, отказ VRF только повторный
Решение владельца 2026-11-19 (ускорение и упрощение проверок при той же итоговой защите): - права шага RouterOS доказываются свежей проверкой на границах этапа (начало шага, прямо перед отправкой, сразу после ACK, перед каждой записью); чтения внутри этапа повторно используют доказательство моложе 2 с. Сервисные шаги проверяют права на каждый вызов, как раньше: их проверка общая для чтений и записей; - независимые каталоги при проверках и в снимке VRF читаются параллельно (пул REST ограничивает 2 соединениями на роутер), порядок снимка прежний; после ACK изменённый каталог читается один раз; - конфликт каталога в проверке перед отправкой помечает операцию «не отправлено»; - VRF: помеченный отказ классификатора окончательный, только если тот же отказ повторился на свежем чтении (в пределах 8 повторов). Интеграционный набор PostgreSQL: 164 проверки пройдены; общие 92 проверки 401 -> 354 с. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
1 parent
4e4c871761
commit
6584644112
11 files changed
+162
-51
No files matched your search
@@ -0,0 +1,21 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { coalesceAuthorization } from "./authorization-stage.js"
|
||||
|
||||
let clock = 0, checks = 0, deny = false
|
||||
const stage = coalesceAuthorization(async () => { checks++; await Promise.resolve(); if (deny) throw Error("revoked"); return checks }, 2000, () => clock)
|
||||
|
||||
// Concurrent reads inside one stage share a single check.
|
||||
assert.deepEqual(await Promise.all([stage.coalesced(), stage.coalesced(), stage.coalesced()]), [1, 1, 1])
|
||||
assert.equal(checks, 1)
|
||||
clock = 1999; assert.equal(await stage.coalesced(), 1); assert.equal(checks, 1)
|
||||
// The window is measured from the start of the check; at its end a read re-proves rights.
|
||||
clock = 2000; assert.equal(await stage.coalesced(), 2); assert.equal(checks, 2)
|
||||
// A stage boundary always re-proves rights, even inside the window.
|
||||
clock = 2001; assert.equal(await stage.fresh(), 3); assert.equal(checks, 3)
|
||||
// Revocation seen by any check is never masked by an earlier success.
|
||||
deny = true
|
||||
await assert.rejects(stage.fresh(), /revoked/)
|
||||
await assert.rejects(stage.coalesced(), /revoked/)
|
||||
assert.equal(checks, 5)
|
||||
deny = false; assert.equal(await stage.coalesced(), 6)
|
||||
console.log("MCP authorization stage: shared in-flight check, 2 s window from check start, fresh at boundaries, failures never cached")
|
||||
@@ -0,0 +1,25 @@
|
||||
/** Rights are proven at each stage boundary with `fresh()` (step start, right before
|
||||
* dispatch, right after ACK). Reads inside a stage call `coalesced()`, which reuses
|
||||
* a proof that started less than MCP_AUTHORIZATION_STAGE_MS ago and joins a check
|
||||
* already in flight. A failed check is never reused. */
|
||||
export const MCP_AUTHORIZATION_STAGE_MS = 2000
|
||||
|
||||
export function coalesceAuthorization<T>(check: () => Promise<T>, windowMs = MCP_AUTHORIZATION_STAGE_MS, now = () => performance.now()) {
|
||||
let last: { at: number; value: T } | undefined, pending: Promise<T> | undefined
|
||||
const fresh = async (): Promise<T> => {
|
||||
const started = now()
|
||||
try {
|
||||
const value = await check()
|
||||
if (!last || last.at <= started) last = { at: started, value }
|
||||
return value
|
||||
} catch (error) {
|
||||
last = undefined
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const coalesced = (): Promise<T> => {
|
||||
if (last && now() - last.at < windowMs) return Promise.resolve(last.value)
|
||||
return pending ??= fresh().finally(() => { pending = undefined })
|
||||
}
|
||||
return { fresh, coalesced }
|
||||
}
|
||||
@@ -130,7 +130,8 @@ export async function acceptNativeConservation(actor: AuthUser) {
|
||||
reset(); const plan = await prepare(action), before = writes, beforePolls = polls, result = await apply(plan.planId)
|
||||
assert.equal(result.state, "applied", `normal ${action}`); assert.equal(result.verification, "passed"); assert.equal(writes, before + 1)
|
||||
const sealed = await evidence(plan.planId)
|
||||
assert.equal(polls - beforePolls, new Set(sealed.snapshots.map(snapshot => snapshot.resource)).size * 4 + 1, `normal ${action}: one sweep per execution boundary and changed-catalog readback`)
|
||||
// Pre-intent, pre-dispatch, post-ACK (the changed catalog is read once there) and pre-terminal sweeps.
|
||||
assert.equal(polls - beforePolls, new Set(sealed.snapshots.map(snapshot => snapshot.resource)).size * 4, `normal ${action}: one sweep per execution boundary, changed catalog read once after ACK`)
|
||||
const observed = await invokeMcpTool(principal, "mm_operation_reconcile", { operationId: result.operation.id }) as { automaticRetry: boolean }
|
||||
assert.equal(observed.automaticRetry, false); assert.equal(writes, before + 1)
|
||||
if (action !== "delete") {
|
||||
|
||||
@@ -49,4 +49,19 @@ actual.set(old.resource, old); actual.set(dependency.resource, dependency)
|
||||
await unadvanced.check()
|
||||
await assert.rejects(unadvanced.check(async () => { throw new McpError("PERMISSION_DENIED", "revoked") }), error => error instanceof McpError && error.code === "PERMISSION_DENIED")
|
||||
assert.equal(unadvanced.before(step).rows[0].comment, "old")
|
||||
console.log("ROS prefix: version/source/fingerprint and duplicate-baseline guards, verified advancement, inter-step drift/source changes, failed conservation/dependency do not advance, and live revocation")
|
||||
|
||||
// Catalogs are read concurrently; advance trusts the caller's fresh target read and re-reads only the others.
|
||||
const reads: string[] = []
|
||||
let inFlight = 0, peak = 0
|
||||
const counted = async (_serverId: number, resource: string, beforeRead: () => Promise<unknown> = async () => {}) => {
|
||||
reads.push(resource); inFlight++; peak = Math.max(peak, inFlight)
|
||||
await beforeRead(); await new Promise(resolve => setTimeout(resolve, 5)); inFlight--
|
||||
return structuredClone(actual.get(resource)!)
|
||||
}
|
||||
const concurrent = createRosPrefix([old, dependency], counted)
|
||||
await concurrent.check()
|
||||
assert.deepEqual(reads.sort(), ["ip_filter", "routing_table"]); assert.equal(peak, 2)
|
||||
reads.length = 0; actual.set(old.resource, changed)
|
||||
await concurrent.advance(step, {}, changed)
|
||||
assert.deepEqual(reads, ["routing_table"], "advance does not re-read the target it was just given")
|
||||
console.log("ROS prefix: version/source/fingerprint and duplicate-baseline guards, verified advancement, inter-step drift/source changes, failed conservation/dependency do not advance, live revocation, concurrent reads, no target re-read on advance")
|
||||
@@ -23,11 +23,12 @@ export function createRosPrefix(snapshots: RosSnapshot[], read: typeof readRosSn
|
||||
if (previous && !matches(previous, snapshot)) throw new McpError("STATE_CONFLICT", "Неоднозначный исходный сетевой снимок", 409)
|
||||
expected.set(key(snapshot), structuredClone(snapshot))
|
||||
}
|
||||
const checkMap = async (map: Map<string, RosSnapshot>, beforeRead: () => Promise<unknown>, code: string) => {
|
||||
for (const snapshot of map.values()) {
|
||||
// Independent catalogs are read concurrently (the shared REST pool bounds it to 2 per router).
|
||||
const checkMap = async (map: Map<string, RosSnapshot>, beforeRead: () => Promise<unknown>, code: string, alreadyRead?: string) => {
|
||||
await Promise.all([...map.values()].filter(snapshot => key(snapshot) !== alreadyRead).map(async snapshot => {
|
||||
const current = await read(snapshot.serverId, snapshot.resource, beforeRead, snapshot.sourceProof)
|
||||
if (!matches(snapshot, current)) throw new McpError(code, "Сетевой источник или ожидаемая конфигурация изменились; повтор записи запрещён", 409)
|
||||
}
|
||||
}))
|
||||
await beforeRead()
|
||||
}
|
||||
return {
|
||||
@@ -44,7 +45,8 @@ export function createRosPrefix(snapshots: RosSnapshot[], read: typeof readRosSn
|
||||
if (!before || !valid(after) || before.sourceProof !== after.sourceProof || !verifyRosStepSnapshots(step, result, before, after)) throw new McpError("VERIFICATION_FAILED", "Изменение сетевого каталога не совпало с подтверждённым шагом", 409)
|
||||
const next = new Map(expected)
|
||||
next.set(key(after), structuredClone(after))
|
||||
await checkMap(next, beforeRead, "VERIFICATION_FAILED")
|
||||
// `after` is the caller's fresh post-ACK read of the target; re-read only the other catalogs.
|
||||
await checkMap(next, beforeRead, "VERIFICATION_FAILED", key(after))
|
||||
expected = next
|
||||
},
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { coalesceAuthorization } from "./authorization-stage.js"
|
||||
import { partitionWriteFailureDiagnostic, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js"
|
||||
import { pollErrorLog } from "../../services/server-poll-error-provenance.js"
|
||||
import { manualRoutingPredispatchDiagnostic } from "./routing-community-predispatch-diagnostic.js"
|
||||
@@ -528,6 +529,8 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
const serverIds = evidence.target.scope === "servers" ? evidence.target.serverIds : evidence.objects.flatMap(object => object.serverId ? [object.serverId] : [])
|
||||
let attempted = false, confirmed = 0, activeServiceStep = false
|
||||
let ecmpStoppedBeforeDispatch = false
|
||||
// A sealed catalog changed before the native write was sent: nothing reached the router.
|
||||
let stoppedBeforeDispatch = false
|
||||
let vrfDiagnosticFallback: ReturnType<typeof createVrfRuntimeDiagnosticBoundary> | undefined
|
||||
let reservedForwardPassed = false
|
||||
let reservedRecoverySummary: Record<string, unknown> | undefined
|
||||
@@ -574,16 +577,16 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
assertOperationConfigurationLockScope()
|
||||
return connection
|
||||
}
|
||||
const authorize = async () => {
|
||||
const authorizeStage = coalesceAuthorization(async () => {
|
||||
assertOperationConfigurationLockScope()
|
||||
const connection = await authorizeGrant()
|
||||
assertOperationConfigurationLockScope()
|
||||
await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant)
|
||||
assertOperationConfigurationLockScope()
|
||||
return connection
|
||||
}
|
||||
})
|
||||
let ownerSources: (() => Promise<void>) | undefined
|
||||
const authorizeCatalog = async () => {
|
||||
const catalogStage = coalesceAuthorization(async () => {
|
||||
assertOperationConfigurationLockScope()
|
||||
const connection = await authorizeGrant()
|
||||
assertOperationConfigurationLockScope()
|
||||
@@ -591,7 +594,10 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
await ownerSources()
|
||||
assertOperationConfigurationLockScope()
|
||||
return connection
|
||||
}
|
||||
})
|
||||
// Reads inside a stage reuse a recent proof; stage boundaries below call fresh().
|
||||
const authorize = async () => { assertOperationConfigurationLockScope(); return authorizeStage.coalesced() }
|
||||
const authorizeCatalog = async () => { assertOperationConfigurationLockScope(); return catalogStage.coalesced() }
|
||||
try {
|
||||
if (productionVrf && (evidence.steps.length !== 1 || evidence.steps[0].kind !== "ros" || evidence.steps[0].step.resource !== "vrf")) throw new McpError("STATE_CONFLICT", "VRF execution требует exact single typed step", 409)
|
||||
if (await routingReservationEvidence(plan, evidence) && (!routingReservationForward || !routingReservationId)) throw new McpError("STATE_CONFLICT", "No confirmed reserved forward admission", 409)
|
||||
@@ -631,10 +637,11 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
await verifyManagementChannel(channel.serverId, channel.channelId)
|
||||
}
|
||||
if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409)
|
||||
const connection = await authorize()
|
||||
const connection = await authorizeStage.fresh()
|
||||
const step = evidence.steps[index]
|
||||
let vrfBeforeDispatch: (() => Promise<void>) | undefined
|
||||
let vrfBeforeDispatch: (() => Promise<void>) | undefined, stepDispatches = 0
|
||||
if (step.kind === "ros") {
|
||||
// Refuse before any intent row exists (drift detection treats a newer step row as superseding).
|
||||
await prefix!.check(authorizeCatalog)
|
||||
const before = productionVrf ? evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId) : prefix!.before(step.step)
|
||||
if (!before) throw new McpError("STATE_CONFLICT", "Нет исходного VRF source snapshot", 409)
|
||||
@@ -655,7 +662,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
vrfDiagnosticFallback = productionVrf && step.kind === "ros" && step.step.resource === "vrf" && ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id)
|
||||
? createVrfRuntimeDiagnosticBoundary(operationId, step.step.action === "update" ? "patch" : step.step.action, diagnostic => { console.error(JSON.stringify(diagnostic)) }) : undefined
|
||||
const beforeDispatch = step.kind === "ros" ? async () => {
|
||||
await prefix!.check(authorizeCatalog)
|
||||
try { await prefix!.check(authorizeCatalog) } catch (error) { if (stepDispatches === 0) stoppedBeforeDispatch = true; throw error }
|
||||
if (checksResolvedNextHop) {
|
||||
try {
|
||||
await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog)
|
||||
@@ -664,14 +671,15 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
throw error
|
||||
}
|
||||
}
|
||||
await authorize()
|
||||
await authorizeCatalog()
|
||||
await authorizeStage.fresh()
|
||||
await catalogStage.fresh()
|
||||
if (productionVrf) {
|
||||
if (!vrfBeforeDispatch) throw new McpError("STATE_CONFLICT", "VRF command guard отсутствует", 409)
|
||||
await vrfBeforeDispatch()
|
||||
}
|
||||
stepDispatches++
|
||||
} : undefined
|
||||
const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, beforeDispatch },
|
||||
const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, authorizeWrite: async () => { await (step.kind === "ros" ? catalogStage.fresh() : authorizeStage.fresh()) }, beforeDispatch },
|
||||
async () => {
|
||||
vrfDiagnosticFallback?.executorEntered()
|
||||
if (step.kind === "ros") {
|
||||
@@ -723,6 +731,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
const after = await Promise.all(evidence.snapshots.map(snapshot => readRosSnapshot(snapshot.serverId, snapshot.resource)))
|
||||
await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))])
|
||||
}
|
||||
if (step.kind === "ros") await catalogStage.fresh() // post-ACK stage boundary
|
||||
if (step.kind === "ros" && productionVrf) {
|
||||
await prefix!.check(authorizeCatalog)
|
||||
} else if (step.kind === "ros") {
|
||||
@@ -775,9 +784,10 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin
|
||||
const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" &&
|
||||
(error as { diagnosticNotStarted?: unknown } | null)?.diagnosticNotStarted === true &&
|
||||
diagnosticProgress?.state === "cancelled" && diagnosticProgress.attemptedStages === 0 && diagnosticProgress.completedStages === 0
|
||||
const state = ecmpStoppedBeforeDispatch ? confirmed === 0 ? "rejected" : "partial"
|
||||
const notDispatched = ecmpStoppedBeforeDispatch || stoppedBeforeDispatch
|
||||
const state = notDispatched ? confirmed === 0 ? "rejected" : "partial"
|
||||
: !attempted || diagnosticNotStarted ? "rejected" : code === "VERIFICATION_FAILED" ? "partial" : "unknown"
|
||||
const dispatchOutcome = ecmpStoppedBeforeDispatch ? { failedStepDispatched: false } : {}
|
||||
const dispatchOutcome = notDispatched ? { failedStepDispatched: false } : {}
|
||||
// Private operator STDERR only; never HTTP/result/audit payload.
|
||||
try {
|
||||
const [step] = evidence.steps
|
||||
|
||||
@@ -20,7 +20,9 @@ export interface VerifiedMcpServiceMetadata {
|
||||
/** Snapshot of the exact server-decrypted sealed step, never public plan JSON. */
|
||||
serviceInput:Record<string,unknown>
|
||||
}
|
||||
interface VerifiedChange { active: boolean; actorId: string; connectionId: string; approved: boolean; serverIds: number[]; objects: McpObjectRef[]; serverWide: boolean; authorize?: () => Promise<void>; beforeDispatch?: () => Promise<void>; serviceExecution?:VerifiedMcpServiceMetadata }
|
||||
// `authorize` guards reads inside a stage and may reuse a recent proof; `authorizeWrite`, when given, is a
|
||||
// fresh check before every native or application write (each write is a stage boundary).
|
||||
interface VerifiedChange { active: boolean; actorId: string; connectionId: string; approved: boolean; serverIds: number[]; objects: McpObjectRef[]; serverWide: boolean; authorize?: () => Promise<void>; authorizeWrite?: () => Promise<void>; beforeDispatch?: () => Promise<void>; serviceExecution?:VerifiedMcpServiceMetadata }
|
||||
const verified = new AsyncLocalStorage<VerifiedChange>()
|
||||
declare const serviceExecutionBrand:unique symbol
|
||||
export type VerifiedMcpServiceExecution=Readonly<{[serviceExecutionBrand]:true}>
|
||||
@@ -67,14 +69,14 @@ function checkService(entry:ServiceExecutionScope){
|
||||
export async function captureVerifiedMcpServiceExecution():Promise<VerifiedMcpServiceExecution>{
|
||||
const context=verified.getStore(),entry=context&&serviceScopes.get(context)
|
||||
if(!entry)return deniedService();checkService(entry)
|
||||
await entry.context.authorize!();checkService(entry)
|
||||
await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry)
|
||||
const token=Object.freeze(Object.create(null)) as VerifiedMcpServiceExecution
|
||||
serviceTokens.set(token,entry);return token
|
||||
}
|
||||
export async function inspectVerifiedMcpServiceExecution(token:unknown){
|
||||
const entry=token&&typeof token==="object"?serviceTokens.get(token):undefined
|
||||
if(!entry)return deniedService();checkService(entry)
|
||||
await entry.context.authorize!();checkService(entry)
|
||||
await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry)
|
||||
return freezeService({...entry.metadata,ownerId:entry.scope.actorId,connectionId:entry.scope.connectionId,
|
||||
approved:entry.scope.approved,serverWide:entry.scope.serverWide,serverIds:[...entry.scope.serverIds],objects:structuredClone(entry.scope.objects)})
|
||||
}
|
||||
@@ -83,7 +85,7 @@ export async function inspectVerifiedMcpServiceExecution(token:unknown){
|
||||
export async function inspectVerifiedMcpServiceExecutionLifetime(token:unknown):Promise<Readonly<{expiresAt:number;monotonicDeadline:number;signal:AbortSignal}>>{
|
||||
const entry=token&&typeof token==="object"?serviceTokens.get(token):undefined
|
||||
if(!entry)return deniedService();checkService(entry)
|
||||
await entry.context.authorize!();checkService(entry)
|
||||
await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry)
|
||||
return Object.freeze({expiresAt:entry.metadata.originalExpiresAt,monotonicDeadline:entry.monotonicEnd,signal:entry.controller.signal})
|
||||
}
|
||||
/** Scheduler handoff may leave an active original scope, never a retired one. */
|
||||
@@ -118,7 +120,7 @@ export async function withVerifiedConfigurationChange<T>(input: Omit<VerifiedCha
|
||||
export async function assertVerifiedConfigurationAccess(): Promise<void> {
|
||||
const context = verified.getStore()
|
||||
if (!context?.active || !context.authorize) throw new McpError("PERMISSION_DENIED", "Нет действующего контекста операции измерения", 403)
|
||||
await context.authorize()
|
||||
await (context.authorizeWrite ?? context.authorize)()
|
||||
}
|
||||
|
||||
/** A full database replacement affects every protected domain, including non-router objects. */
|
||||
@@ -146,7 +148,7 @@ export async function executeGuardedConfigurationWrite<T>(serverId: number, meth
|
||||
const inherited = verified.getStore(), context = inherited?.active ? inherited : undefined
|
||||
// A service step can contain many native writes. Revalidate its live access
|
||||
// for each dispatch, rather than only once at the beginning of that step.
|
||||
await context?.authorize?.()
|
||||
await (context?.authorizeWrite ?? context?.authorize)?.()
|
||||
const actorId = context?.actorId ?? actor.getStore() ?? "system"
|
||||
const { rows } = await pool.query("SELECT object_ref FROM mcp_protections WHERE object_ref->>'serverId'=$1 OR (object_ref->>'kind'='server' AND object_ref->>'id'=$1)", [String(serverId)])
|
||||
for (const row of rows) {
|
||||
@@ -192,7 +194,7 @@ export async function executeGuardedApplicationWrite<T>(object: { kind: string;
|
||||
const serverIds = dependencies === "all" ? (await pool.query("SELECT id FROM servers")).rows.map(row => Number(row.id)) : typeof dependencies === "function" ? await dependencies() : dependencies
|
||||
if (serverIds.some(id => !Number.isSafeInteger(id) || id <= 0)) throw new McpError("INVALID_INPUT", "Некорректная зависимость сервера", 400)
|
||||
const inherited = verified.getStore(), context = inherited?.active ? inherited : undefined, actorId = context?.actorId ?? actor.getStore() ?? "system"
|
||||
await context?.authorize?.()
|
||||
await (context?.authorizeWrite ?? context?.authorize)?.()
|
||||
if (context && serverIds.some(id => !context.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Сервис затрагивает зависимость вне проверенного плана")
|
||||
const objects = affectedObjects ? await affectedObjects() : []
|
||||
const { rows } = await pool.query("SELECT object_ref FROM mcp_protections WHERE object_ref->>'serverId'=ANY($1::text[]) OR (object_ref->>'kind'='server' AND object_ref->>'id'=ANY($1::text[])) OR (object_ref->>'kind'=$2 AND object_ref->>'id'=$3) OR EXISTS(SELECT 1 FROM jsonb_array_elements($4::jsonb) AS target WHERE target->>'kind'=object_ref->>'kind' AND target->>'id'=object_ref->>'id')", [serverIds.map(String), object.kind, object.id, JSON.stringify(objects)])
|
||||
|
||||
@@ -234,7 +234,8 @@ for (const [label, current, mutate, guard] of midFlap) {
|
||||
// (d) mutation check: without re-reads the mid-flap read is refused.
|
||||
await assert.rejects(settleVrfTransition(flapping, classify, async () => structuredClone(current.after), noPause, 0), /VRF production transition refused/, label); cases++
|
||||
}
|
||||
// (c) genuine configuration changes are refused at once, with no re-read.
|
||||
// (c) a genuine configuration change persists on the router: the same refusal on one fresh read is final.
|
||||
// A refusal that does not repeat on the fresh read was a half-applied read and the fresh state is classified.
|
||||
for (const [label, current, mutate] of [
|
||||
["static address changed", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv4_address.find(row => greNames(s).includes(String(row.interface)))!.address = "10.99.99.1/30" }],
|
||||
["foreign route row", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv6_route.push({ ".id": "*ffab", opaque: "foreign" }) }],
|
||||
@@ -243,8 +244,11 @@ for (const [label, current, mutate] of [
|
||||
] as const) {
|
||||
const changed = structuredClone(current.after); mutate(changed)
|
||||
let rereads = 0
|
||||
await assert.rejects(settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(current.after) }, noPause), /VRF production transition refused/, label)
|
||||
assert.equal(rereads, 0, label); cases++
|
||||
await assert.rejects(settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(changed) }, noPause), /VRF production transition refused/, label)
|
||||
assert.equal(rereads, 1, label); cases++
|
||||
rereads = 0
|
||||
const healed = await settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(current.after) }, noPause)
|
||||
assert.equal(healed.status, "explained-native-transition", label); assert.equal(rereads, 1, label); cases++
|
||||
}
|
||||
// Owner decision 2026-11-18 (stand experiment, docs/mcp-stand-run-a-plan.md 1dda615b): a membership PATCH that moves an
|
||||
// empty concrete bridge out of the VRF may lose its automatic LL; the loss is revival debt settled once by a reappearance.
|
||||
@@ -295,11 +299,11 @@ for (const [label, current, mutate] of [
|
||||
// (d) still absent on the rejoin: the debt is carried, neither adopted nor refused.
|
||||
const carried = classifyVrfProductionTransition(input(left, withoutLl(rejoined, leaving), rejoin, leftPrevious))
|
||||
assert.deepEqual(durable(carried.checkpoint).leftBridgeLl.map((item: { member: string }) => item.member), [leaving]); cases++
|
||||
// (e) refused at once, no settling re-read: non-empty bridge, changed bridge row, missing static address, extra missing row.
|
||||
// (e) refused after one fresh read repeats the refusal: non-empty bridge, changed bridge row, missing static address, extra missing row.
|
||||
const refusedNow = async (before: VrfProductionSnapshot, after: VrfProductionSnapshot, label: string, reason: RegExp) => {
|
||||
let count = 0
|
||||
await assert.rejects(settleVrfTransition(after, next => classifyVrfProductionTransition(input(before, next, transfer, chainPrevious)), async () => { count++; return after }, async () => {}), reason, label)
|
||||
assert.equal(count, 0, label); cases++
|
||||
assert.equal(count, 1, label); cases++
|
||||
}
|
||||
const bridgeId = String(reenabled.catalogs.bridge.find(row => row.name === leaving)![".id"])
|
||||
const ported = (s: VrfProductionSnapshot) => { const copy = structuredClone(s); copy.catalogs.bridge_vlan.push({ ".id": "*ff70", bridge: leaving, "vlan-ids": "10", tagged: leaving, untagged: "", disabled: "false" }); return copy }
|
||||
@@ -331,15 +335,15 @@ for (const [label, current, mutate] of [
|
||||
count = 0
|
||||
const exhausted = await settleVrfTransition(lingering, classifyLeave, async () => { count++; return structuredClone(lingering) }, async () => {}).then(() => undefined, (error: unknown) => error)
|
||||
assert.equal(readVrfTransitionRefusal(exhausted)?.guard, "PATCH_BRIDGE_LL_CONNECTED_ROUTE_CLOSURE"); assert.equal(count, VRF_SETTLING_REREADS); cases++
|
||||
// The same lingering route with a non-empty bridge or a changed bridge row: refused at once (earlier checks).
|
||||
// The same lingering route with a non-empty bridge or a changed bridge row: refused once the fresh read repeats it (earlier checks).
|
||||
const portedLinger = ported(lingering)
|
||||
count = 0
|
||||
await assert.rejects(settleVrfTransition(portedLinger, after => classifyVrfProductionTransition(input(ported(reenabled), after, transfer, chainPrevious)), async () => { count++; return portedLinger }, async () => {}), /PATCH bridge LL empty configuration/)
|
||||
assert.equal(count, 0); cases++
|
||||
assert.equal(count, 1); cases++
|
||||
const changedLinger = structuredClone(lingering); changedLinger.catalogs.bridge.find(row => row.name === leaving)!.mtu = "1400"
|
||||
count = 0
|
||||
await assert.rejects(settleVrfTransition(changedLinger, classifyLeave, async () => { count++; return changedLinger }, async () => {}), /opaque configuration drift/)
|
||||
assert.equal(count, 0); cases++
|
||||
assert.equal(count, 1); cases++
|
||||
}
|
||||
// Owner decision 2026-11-18 (run v24 step 31): the rejoin post-read holds the owed bridge's fe80 route but not yet its
|
||||
// LL address (non-atomic read). That shape alone is re-read (settling); nothing is accepted without the address.
|
||||
@@ -356,11 +360,11 @@ for (const [label, current, mutate] of [
|
||||
count = 0
|
||||
const exhausted = await settleVrfTransition(routeOnly, classifyRejoin(leftPrevious), async () => { count++; return structuredClone(routeOnly) }, async () => {}).then(() => undefined, (error: unknown) => error)
|
||||
assert.equal(readVrfTransitionRefusal(exhausted)?.guard, "FOREIGN_CATALOG_CONFIGURATION_ORDER_CHANGED"); assert.equal(count, VRF_SETTLING_REREADS); cases++
|
||||
// Immediate refusal (no re-read): no debt, an extra foreign row, a route of a bridge without debt, the route of a GRE.
|
||||
// Refused when one fresh read repeats it: no debt, an extra foreign row, a route of a bridge without debt, the route of a GRE.
|
||||
const immediate = async (first: VrfProductionSnapshot, classify: (after: VrfProductionSnapshot) => unknown, label: string) => {
|
||||
let rereads = 0
|
||||
await assert.rejects(settleVrfTransition(first, classify as (after: VrfProductionSnapshot) => never, async () => { rereads++; return structuredClone(rejoined) }, async () => {}), /foreign ipv6_route configuration\/order changed/, label)
|
||||
assert.equal(rereads, 0, label); cases++
|
||||
await assert.rejects(settleVrfTransition(first, classify as (after: VrfProductionSnapshot) => never, async () => { rereads++; return structuredClone(first) }, async () => {}), /foreign ipv6_route configuration\/order changed/, label)
|
||||
assert.equal(rereads, 1, label); cases++
|
||||
}
|
||||
await immediate(routeOnly, classifyRejoin(undefined), "no debt")
|
||||
const extraForeign = structuredClone(routeOnly); extraForeign.catalogs.ipv6_route.push({ ".id": "*ffc1", "dst-address": "2001:db8::/64", gateway: "foreign", dynamic: "false", "routing-table": "main" })
|
||||
@@ -376,7 +380,7 @@ for (const [label, current, mutate] of [
|
||||
const greAfter = structuredClone(gre.after); greAfter.catalogs.ipv6_route.push({ ...structuredClone(greRoute), ".id": "*ffc3" })
|
||||
let rereads = 0
|
||||
await assert.rejects(settleVrfTransition(greAfter, after => classifyVrfProductionTransition({ ...gre, after }), async () => { rereads++; return greAfter }, async () => {}), /VRF production transition refused/)
|
||||
assert.equal(rereads, 0); cases++
|
||||
assert.equal(rereads, 1); cases++
|
||||
}
|
||||
}
|
||||
// Owner decision 2026-11-18 (run v21 step 31): the reconcile of a verified checkpoint may see the owed bridge LL come
|
||||
|
||||
@@ -63,7 +63,9 @@ for (const change of ["revoke", "disable", "source"] as const) {
|
||||
const m = model()
|
||||
m.afterGet(() => { if (change === "revoke") m.revoke(); if (change === "disable") m.disable(); if (change === "source") m.changeSource() })
|
||||
await assert.rejects(readVrfProductionSnapshot(m.input))
|
||||
assert.equal(m.calls.length, 1)
|
||||
// The fixed catalogs are fetched as one concurrent batch: reads already in flight finish, the snapshot
|
||||
// is refused, and no closure read starts after the change was observed.
|
||||
assert.ok(m.calls.length >= 1 && m.calls.length <= VRF_PRODUCTION_REQUIRED_CATALOGS.length, String(m.calls.length))
|
||||
}
|
||||
const resolving = model(); resolving.afterResolve(() => resolving.revoke())
|
||||
await assert.rejects(readVrfProductionSnapshot(resolving.input)); assert.equal(resolving.calls.length, 0)
|
||||
@@ -155,7 +157,10 @@ assert.deepEqual(await readVrfProductionSnapshot({ ...observed.input, observeRea
|
||||
assert.deepEqual(observed.calls, happy.calls)
|
||||
assert.deepEqual(observed.counters(), happy.counters())
|
||||
assert.equal(observations.length, observed.calls.length * 5 + 2)
|
||||
assert.deepEqual(observations.slice(0, 4).map(value => value.boundary), ["authority-before", "native-get", "authority-after", "shape"])
|
||||
// Catalogs are read concurrently; each catalog keeps its own boundary order.
|
||||
for (const catalog of new Set(observations.flatMap(value => "catalog" in value && value.catalog ? [value.catalog] : []))) {
|
||||
assert.deepEqual(observations.filter(value => "catalog" in value && value.catalog === catalog).map(value => value.boundary).slice(0, 4), ["authority-before", "native-get", "authority-after", "shape"], catalog)
|
||||
}
|
||||
assert.ok(observations.every(value => value.serverId === 7))
|
||||
assert.deepEqual(observations.at(-2), { serverId: 7, boundary: "final-authority" })
|
||||
assert.deepEqual(observations.at(-1), { serverId: 7, boundary: "snapshot-validation" })
|
||||
|
||||
@@ -108,16 +108,19 @@ export async function readVrfProductionSnapshot(input: ReadVrfProductionInput):
|
||||
}
|
||||
const catalogs: VrfProductionSnapshot["catalogs"] = {}
|
||||
let totalBytes = 0
|
||||
const read = async (resource: string): Promise<void> => {
|
||||
const fetchCatalog = async (resource: string, reach: (boundary: VrfRawReadObservation["boundary"]) => void = () => {}): Promise<VrfProductionRow[]> => {
|
||||
const at = (boundary: VrfRawReadObservation["boundary"]) => { reach(boundary); notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary }) }
|
||||
insist(Object.hasOwn(VRF_PRODUCTION_RAW_PATHS, resource), "unknown raw resource")
|
||||
notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "authority-before" })
|
||||
at("authority-before")
|
||||
const source = await fence()
|
||||
notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "native-get" })
|
||||
at("native-get")
|
||||
const payload = await source.client.get<unknown>(VRF_PRODUCTION_RAW_PATHS[resource], 10000, undefined, MAX_BYTES)
|
||||
notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "authority-after" })
|
||||
at("authority-after")
|
||||
await fence()
|
||||
notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "shape" })
|
||||
const catalog = rows(resource, payload)
|
||||
at("shape")
|
||||
return rows(resource, payload)
|
||||
}
|
||||
const store = (resource: string, catalog: VrfProductionRow[]): void => {
|
||||
// Catalog shape succeeded. Budgets and subsequent closure validation are
|
||||
// snapshot work, not a failure of this completed native GET/catalog shape.
|
||||
notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, boundary: "snapshot-validation" })
|
||||
@@ -126,8 +129,22 @@ export async function readVrfProductionSnapshot(input: ReadVrfProductionInput):
|
||||
insist(size < MAX_BYTES && totalBytes < MAX_TOTAL_BYTES, "raw evidence byte budget")
|
||||
catalogs[resource] = catalog
|
||||
}
|
||||
for (const resource of VRF_PRODUCTION_REQUIRED_CATALOGS) await read(resource)
|
||||
for (const resource of [...(input.requiredCatalogs ?? [])]) if (!Object.hasOwn(catalogs, resource)) await read(resource)
|
||||
const read = async (resource: string): Promise<void> => store(resource, await fetchCatalog(resource))
|
||||
// The fixed catalogs are independent: read them concurrently (bounded by the REST pool), which also
|
||||
// narrows the window in which RouterOS can show an address and its route out of step. Stored in the
|
||||
// fixed order, so the snapshot is identical to a sequential read.
|
||||
const initial = [...new Set<string>([...VRF_PRODUCTION_REQUIRED_CATALOGS, ...(input.requiredCatalogs ?? [])])]
|
||||
const reached = new Map<string, VrfRawReadObservation["boundary"]>()
|
||||
const observeRead = input.observeRead
|
||||
const fetched = await Promise.allSettled(initial.map(resource => fetchCatalog(resource, boundary => reached.set(resource, boundary))))
|
||||
const failed = fetched.findIndex(result => result.status === "rejected")
|
||||
if (failed !== -1) {
|
||||
// Every read has settled; the last diagnostic names the first failing catalog in the fixed order.
|
||||
const boundary = reached.get(initial[failed])
|
||||
if (boundary) notifyVrfDiagnostic(observeRead, { serverId: input.serverId, catalog: initial[failed], boundary })
|
||||
throw (fetched[failed] as PromiseRejectedResult).reason
|
||||
}
|
||||
initial.forEach((resource, index) => store(resource, (fetched[index] as PromiseFulfilledResult<VrfProductionRow[]>).value))
|
||||
const selected = new Set<string>()
|
||||
if (request.action === "create") insist(request.id === undefined && request.fields.interfaces !== undefined, "CREATE selection")
|
||||
else {
|
||||
|
||||
@@ -149,12 +149,21 @@ export function noteVrfTransitionSettling(error: unknown): void {
|
||||
export function isVrfTransitionSettling(error: unknown): boolean {
|
||||
return !!error && typeof error === "object" && settling.has(error)
|
||||
}
|
||||
/** First successful classification wins. Any other refusal, read or authority
|
||||
* error propagates at once; after the bound the last refusal object is rethrown. */
|
||||
/** First successful classification wins. RouterOS applies addresses, routes and
|
||||
* link state separately and the raw after-read is a series of GETs, so a refusal
|
||||
* seen once may describe a half-applied state: a tagged refusal is final only when
|
||||
* the same guard repeats on a fresh read. Refusals marked settling keep re-reading
|
||||
* up to the bound. Untagged errors (read, authority, shape) propagate at once;
|
||||
* after the bound the last refusal object is rethrown. */
|
||||
export async function settleVrfTransition<S, T>(first: S, classify: (after: S) => T, reread: () => Promise<S>, pause = () => new Promise<void>(resolve => setTimeout(resolve, VRF_SETTLING_PAUSE_MS)), rereads = VRF_SETTLING_REREADS): Promise<T> {
|
||||
let after = first
|
||||
let after = first, previous: VrfTransitionGuard | undefined
|
||||
for (let attempt = 0; ; attempt++) {
|
||||
try { return classify(after) } catch (error) { if (attempt >= rereads || !isVrfTransitionSettling(error)) throw error }
|
||||
try { return classify(after) } catch (error) {
|
||||
const guard = readVrfTransitionRefusal(error)?.guard
|
||||
const repeated = guard !== undefined && guard === previous && !isVrfTransitionSettling(error)
|
||||
if (attempt >= rereads || guard === undefined && !isVrfTransitionSettling(error) || repeated) throw error
|
||||
previous = guard
|
||||
}
|
||||
await pause()
|
||||
after = await reread()
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user