From 6584644112be70731ab300437ce336037ae5b88e Mon Sep 17 00:00:00 2001 From: Denozordec Date: Thu, 19 Nov 2026 11:06:29 +0700 Subject: [PATCH] =?UTF-8?q?fix(mcp):=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5?= =?UTF-8?q?=D1=80=D1=8F=D1=82=D1=8C=20=D0=BF=D1=80=D0=B0=D0=B2=D0=B0=20?= =?UTF-8?q?=D1=80=D0=B0=D0=B7=20=D0=BD=D0=B0=20=D1=8D=D1=82=D0=B0=D0=BF,?= =?UTF-8?q?=20=D1=87=D0=B8=D1=82=D0=B0=D1=82=D1=8C=20=D0=BA=D0=B0=D1=82?= =?UTF-8?q?=D0=B0=D0=BB=D0=BE=D0=B3=D0=B8=20=D0=BF=D0=B0=D1=80=D0=B0=D0=BB?= =?UTF-8?q?=D0=BB=D0=B5=D0=BB=D1=8C=D0=BD=D0=BE,=20=D0=BE=D1=82=D0=BA?= =?UTF-8?q?=D0=B0=D0=B7=20VRF=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE=20?= =?UTF-8?q?=D0=BF=D0=BE=D0=B2=D1=82=D0=BE=D1=80=D0=BD=D1=8B=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Решение владельца 2026-11-19 (ускорение и упрощение проверок при той же итоговой защите): - права шага RouterOS доказываются свежей проверкой на границах этапа (начало шага, прямо перед отправкой, сразу после ACK, перед каждой записью); чтения внутри этапа повторно используют доказательство моложе 2 с. Сервисные шаги проверяют права на каждый вызов, как раньше: их проверка общая для чтений и записей; - независимые каталоги при проверках и в снимке VRF читаются параллельно (пул REST ограничивает 2 соединениями на роутер), порядок снимка прежний; после ACK изменённый каталог читается один раз; - конфликт каталога в проверке перед отправкой помечает операцию «не отправлено»; - VRF: помеченный отказ классификатора окончательный, только если тот же отказ повторился на свежем чтении (в пределах 8 повторов). Интеграционный набор PostgreSQL: 164 проверки пройдены; общие 92 проверки 401 -> 354 с. Co-Authored-By: Claude Opus 5.5 --- .../modules/mcp/authorization-stage.test.ts | 21 ++++++++++++ .../src/modules/mcp/authorization-stage.ts | 25 ++++++++++++++ .../mcp/native-conservation-acceptance.ts | 3 +- backend/src/modules/mcp/ros-prefix.test.ts | 17 +++++++++- backend/src/modules/mcp/ros-prefix.ts | 10 +++--- backend/src/modules/mcp/runtime.ts | 34 ++++++++++++------- .../src/services/config-mutation-policy.ts | 16 +++++---- .../vrf-production-gre-link-local.test.ts | 28 ++++++++------- .../src/services/vrf-production-read.test.ts | 9 +++-- backend/src/services/vrf-production-read.ts | 33 +++++++++++++----- .../src/services/vrf-transition-refusal.ts | 17 +++++++--- 11 files changed, 162 insertions(+), 51 deletions(-) create mode 100644 backend/src/modules/mcp/authorization-stage.test.ts create mode 100644 backend/src/modules/mcp/authorization-stage.ts diff --git a/backend/src/modules/mcp/authorization-stage.test.ts b/backend/src/modules/mcp/authorization-stage.test.ts new file mode 100644 index 00000000..805f5a0b --- /dev/null +++ b/backend/src/modules/mcp/authorization-stage.test.ts @@ -0,0 +1,21 @@ +import assert from "node:assert/strict" +import { coalesceAuthorization } from "./authorization-stage.js" + +let clock = 0, checks = 0, deny = false +const stage = coalesceAuthorization(async () => { checks++; await Promise.resolve(); if (deny) throw Error("revoked"); return checks }, 2000, () => clock) + +// Concurrent reads inside one stage share a single check. +assert.deepEqual(await Promise.all([stage.coalesced(), stage.coalesced(), stage.coalesced()]), [1, 1, 1]) +assert.equal(checks, 1) +clock = 1999; assert.equal(await stage.coalesced(), 1); assert.equal(checks, 1) +// The window is measured from the start of the check; at its end a read re-proves rights. +clock = 2000; assert.equal(await stage.coalesced(), 2); assert.equal(checks, 2) +// A stage boundary always re-proves rights, even inside the window. +clock = 2001; assert.equal(await stage.fresh(), 3); assert.equal(checks, 3) +// Revocation seen by any check is never masked by an earlier success. +deny = true +await assert.rejects(stage.fresh(), /revoked/) +await assert.rejects(stage.coalesced(), /revoked/) +assert.equal(checks, 5) +deny = false; assert.equal(await stage.coalesced(), 6) +console.log("MCP authorization stage: shared in-flight check, 2 s window from check start, fresh at boundaries, failures never cached") diff --git a/backend/src/modules/mcp/authorization-stage.ts b/backend/src/modules/mcp/authorization-stage.ts new file mode 100644 index 00000000..e3b5f238 --- /dev/null +++ b/backend/src/modules/mcp/authorization-stage.ts @@ -0,0 +1,25 @@ +/** Rights are proven at each stage boundary with `fresh()` (step start, right before + * dispatch, right after ACK). Reads inside a stage call `coalesced()`, which reuses + * a proof that started less than MCP_AUTHORIZATION_STAGE_MS ago and joins a check + * already in flight. A failed check is never reused. */ +export const MCP_AUTHORIZATION_STAGE_MS = 2000 + +export function coalesceAuthorization(check: () => Promise, windowMs = MCP_AUTHORIZATION_STAGE_MS, now = () => performance.now()) { + let last: { at: number; value: T } | undefined, pending: Promise | undefined + const fresh = async (): Promise => { + const started = now() + try { + const value = await check() + if (!last || last.at <= started) last = { at: started, value } + return value + } catch (error) { + last = undefined + throw error + } + } + const coalesced = (): Promise => { + if (last && now() - last.at < windowMs) return Promise.resolve(last.value) + return pending ??= fresh().finally(() => { pending = undefined }) + } + return { fresh, coalesced } +} diff --git a/backend/src/modules/mcp/native-conservation-acceptance.ts b/backend/src/modules/mcp/native-conservation-acceptance.ts index 3181a65e..2a9126a6 100644 --- a/backend/src/modules/mcp/native-conservation-acceptance.ts +++ b/backend/src/modules/mcp/native-conservation-acceptance.ts @@ -130,7 +130,8 @@ export async function acceptNativeConservation(actor: AuthUser) { reset(); const plan = await prepare(action), before = writes, beforePolls = polls, result = await apply(plan.planId) assert.equal(result.state, "applied", `normal ${action}`); assert.equal(result.verification, "passed"); assert.equal(writes, before + 1) const sealed = await evidence(plan.planId) - assert.equal(polls - beforePolls, new Set(sealed.snapshots.map(snapshot => snapshot.resource)).size * 4 + 1, `normal ${action}: one sweep per execution boundary and changed-catalog readback`) + // Pre-intent, pre-dispatch, post-ACK (the changed catalog is read once there) and pre-terminal sweeps. + assert.equal(polls - beforePolls, new Set(sealed.snapshots.map(snapshot => snapshot.resource)).size * 4, `normal ${action}: one sweep per execution boundary, changed catalog read once after ACK`) const observed = await invokeMcpTool(principal, "mm_operation_reconcile", { operationId: result.operation.id }) as { automaticRetry: boolean } assert.equal(observed.automaticRetry, false); assert.equal(writes, before + 1) if (action !== "delete") { diff --git a/backend/src/modules/mcp/ros-prefix.test.ts b/backend/src/modules/mcp/ros-prefix.test.ts index f7599280..ab436161 100644 --- a/backend/src/modules/mcp/ros-prefix.test.ts +++ b/backend/src/modules/mcp/ros-prefix.test.ts @@ -49,4 +49,19 @@ actual.set(old.resource, old); actual.set(dependency.resource, dependency) await unadvanced.check() await assert.rejects(unadvanced.check(async () => { throw new McpError("PERMISSION_DENIED", "revoked") }), error => error instanceof McpError && error.code === "PERMISSION_DENIED") assert.equal(unadvanced.before(step).rows[0].comment, "old") -console.log("ROS prefix: version/source/fingerprint and duplicate-baseline guards, verified advancement, inter-step drift/source changes, failed conservation/dependency do not advance, and live revocation") + +// Catalogs are read concurrently; advance trusts the caller's fresh target read and re-reads only the others. +const reads: string[] = [] +let inFlight = 0, peak = 0 +const counted = async (_serverId: number, resource: string, beforeRead: () => Promise = async () => {}) => { + reads.push(resource); inFlight++; peak = Math.max(peak, inFlight) + await beforeRead(); await new Promise(resolve => setTimeout(resolve, 5)); inFlight-- + return structuredClone(actual.get(resource)!) +} +const concurrent = createRosPrefix([old, dependency], counted) +await concurrent.check() +assert.deepEqual(reads.sort(), ["ip_filter", "routing_table"]); assert.equal(peak, 2) +reads.length = 0; actual.set(old.resource, changed) +await concurrent.advance(step, {}, changed) +assert.deepEqual(reads, ["routing_table"], "advance does not re-read the target it was just given") +console.log("ROS prefix: version/source/fingerprint and duplicate-baseline guards, verified advancement, inter-step drift/source changes, failed conservation/dependency do not advance, live revocation, concurrent reads, no target re-read on advance") diff --git a/backend/src/modules/mcp/ros-prefix.ts b/backend/src/modules/mcp/ros-prefix.ts index 75e769d6..19a4ffa9 100644 --- a/backend/src/modules/mcp/ros-prefix.ts +++ b/backend/src/modules/mcp/ros-prefix.ts @@ -23,11 +23,12 @@ export function createRosPrefix(snapshots: RosSnapshot[], read: typeof readRosSn if (previous && !matches(previous, snapshot)) throw new McpError("STATE_CONFLICT", "Неоднозначный исходный сетевой снимок", 409) expected.set(key(snapshot), structuredClone(snapshot)) } - const checkMap = async (map: Map, beforeRead: () => Promise, code: string) => { - for (const snapshot of map.values()) { + // Independent catalogs are read concurrently (the shared REST pool bounds it to 2 per router). + const checkMap = async (map: Map, beforeRead: () => Promise, code: string, alreadyRead?: string) => { + await Promise.all([...map.values()].filter(snapshot => key(snapshot) !== alreadyRead).map(async snapshot => { const current = await read(snapshot.serverId, snapshot.resource, beforeRead, snapshot.sourceProof) if (!matches(snapshot, current)) throw new McpError(code, "Сетевой источник или ожидаемая конфигурация изменились; повтор записи запрещён", 409) - } + })) await beforeRead() } return { @@ -44,7 +45,8 @@ export function createRosPrefix(snapshots: RosSnapshot[], read: typeof readRosSn if (!before || !valid(after) || before.sourceProof !== after.sourceProof || !verifyRosStepSnapshots(step, result, before, after)) throw new McpError("VERIFICATION_FAILED", "Изменение сетевого каталога не совпало с подтверждённым шагом", 409) const next = new Map(expected) next.set(key(after), structuredClone(after)) - await checkMap(next, beforeRead, "VERIFICATION_FAILED") + // `after` is the caller's fresh post-ACK read of the target; re-read only the other catalogs. + await checkMap(next, beforeRead, "VERIFICATION_FAILED", key(after)) expected = next }, } diff --git a/backend/src/modules/mcp/runtime.ts b/backend/src/modules/mcp/runtime.ts index 583611d8..40239e0a 100644 --- a/backend/src/modules/mcp/runtime.ts +++ b/backend/src/modules/mcp/runtime.ts @@ -1,3 +1,4 @@ +import { coalesceAuthorization } from "./authorization-stage.js" import { partitionWriteFailureDiagnostic, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js" import { pollErrorLog } from "../../services/server-poll-error-provenance.js" import { manualRoutingPredispatchDiagnostic } from "./routing-community-predispatch-diagnostic.js" @@ -528,6 +529,8 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin const serverIds = evidence.target.scope === "servers" ? evidence.target.serverIds : evidence.objects.flatMap(object => object.serverId ? [object.serverId] : []) let attempted = false, confirmed = 0, activeServiceStep = false let ecmpStoppedBeforeDispatch = false + // A sealed catalog changed before the native write was sent: nothing reached the router. + let stoppedBeforeDispatch = false let vrfDiagnosticFallback: ReturnType | undefined let reservedForwardPassed = false let reservedRecoverySummary: Record | undefined @@ -574,16 +577,16 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin assertOperationConfigurationLockScope() return connection } - const authorize = async () => { + const authorizeStage = coalesceAuthorization(async () => { assertOperationConfigurationLockScope() const connection = await authorizeGrant() assertOperationConfigurationLockScope() await verifyOspfOwnerGraph(evidence.ospfOwnerGraph, serverIds, authorizeGrant) assertOperationConfigurationLockScope() return connection - } + }) let ownerSources: (() => Promise) | undefined - const authorizeCatalog = async () => { + const catalogStage = coalesceAuthorization(async () => { assertOperationConfigurationLockScope() const connection = await authorizeGrant() assertOperationConfigurationLockScope() @@ -591,7 +594,10 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin await ownerSources() assertOperationConfigurationLockScope() return connection - } + }) + // Reads inside a stage reuse a recent proof; stage boundaries below call fresh(). + const authorize = async () => { assertOperationConfigurationLockScope(); return authorizeStage.coalesced() } + const authorizeCatalog = async () => { assertOperationConfigurationLockScope(); return catalogStage.coalesced() } try { if (productionVrf && (evidence.steps.length !== 1 || evidence.steps[0].kind !== "ros" || evidence.steps[0].step.resource !== "vrf")) throw new McpError("STATE_CONFLICT", "VRF execution требует exact single typed step", 409) if (await routingReservationEvidence(plan, evidence) && (!routingReservationForward || !routingReservationId)) throw new McpError("STATE_CONFLICT", "No confirmed reserved forward admission", 409) @@ -631,10 +637,11 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin await verifyManagementChannel(channel.serverId, channel.channelId) } if (evidence.validUntil && (!Number.isFinite(Date.parse(evidence.validUntil)) || Date.now() >= Date.parse(evidence.validUntil))) throw new McpError("MEASUREMENT_UNKNOWN", "Измерения плана устарели; подготовьте новый план", 409) - const connection = await authorize() + const connection = await authorizeStage.fresh() const step = evidence.steps[index] - let vrfBeforeDispatch: (() => Promise) | undefined + let vrfBeforeDispatch: (() => Promise) | undefined, stepDispatches = 0 if (step.kind === "ros") { + // Refuse before any intent row exists (drift detection treats a newer step row as superseding). await prefix!.check(authorizeCatalog) const before = productionVrf ? evidence.snapshots.find(snapshot => snapshot.resource === "vrf" && snapshot.serverId === step.step.serverId) : prefix!.before(step.step) if (!before) throw new McpError("STATE_CONFLICT", "Нет исходного VRF source snapshot", 409) @@ -655,7 +662,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin vrfDiagnosticFallback = productionVrf && step.kind === "ros" && step.step.resource === "vrf" && ["ros.vrf.create", "ros.vrf.update", "ros.vrf.delete", "routing.vrf.update"].includes(plan.operation_id) ? createVrfRuntimeDiagnosticBoundary(operationId, step.step.action === "update" ? "patch" : step.step.action, diagnostic => { console.error(JSON.stringify(diagnostic)) }) : undefined const beforeDispatch = step.kind === "ros" ? async () => { - await prefix!.check(authorizeCatalog) + try { await prefix!.check(authorizeCatalog) } catch (error) { if (stepDispatches === 0) stoppedBeforeDispatch = true; throw error } if (checksResolvedNextHop) { try { await verifyHealthChecks(evidence, serverIds, false, authorizeCatalog) @@ -664,14 +671,15 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin throw error } } - await authorize() - await authorizeCatalog() + await authorizeStage.fresh() + await catalogStage.fresh() if (productionVrf) { if (!vrfBeforeDispatch) throw new McpError("STATE_CONFLICT", "VRF command guard отсутствует", 409) await vrfBeforeDispatch() } + stepDispatches++ } : undefined - const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, beforeDispatch }, + const result = await withVerifiedConfigurationChange({ actorId: connection.owner_id, connectionId: connection.id, serverIds, objects: evidence.objects, approved: !!plan.approved_by, serverWide: step.kind === "service", ...(serviceExecution?{serviceExecution}:{}), authorize: async () => { await (step.kind === "ros" ? authorizeCatalog() : authorize()) }, authorizeWrite: async () => { await (step.kind === "ros" ? catalogStage.fresh() : authorizeStage.fresh()) }, beforeDispatch }, async () => { vrfDiagnosticFallback?.executorEntered() if (step.kind === "ros") { @@ -723,6 +731,7 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin const after = await Promise.all(evidence.snapshots.map(snapshot => readRosSnapshot(snapshot.serverId, snapshot.resource))) await pool.query("UPDATE mcp_operation_checkpoints SET after_sealed=$3 WHERE operation_id=$1 AND step_index=$2", [operationId, index, await encryptSecret(JSON.stringify(after))]) } + if (step.kind === "ros") await catalogStage.fresh() // post-ACK stage boundary if (step.kind === "ros" && productionVrf) { await prefix!.check(authorizeCatalog) } else if (step.kind === "ros") { @@ -775,9 +784,10 @@ async function executeOperationWithLease(operationId: string, principal: McpPrin const diagnosticNotStarted = diagnosticOnly && confirmed === 0 && code === "READ_CANCELLED" && (error as { diagnosticNotStarted?: unknown } | null)?.diagnosticNotStarted === true && diagnosticProgress?.state === "cancelled" && diagnosticProgress.attemptedStages === 0 && diagnosticProgress.completedStages === 0 - const state = ecmpStoppedBeforeDispatch ? confirmed === 0 ? "rejected" : "partial" + const notDispatched = ecmpStoppedBeforeDispatch || stoppedBeforeDispatch + const state = notDispatched ? confirmed === 0 ? "rejected" : "partial" : !attempted || diagnosticNotStarted ? "rejected" : code === "VERIFICATION_FAILED" ? "partial" : "unknown" - const dispatchOutcome = ecmpStoppedBeforeDispatch ? { failedStepDispatched: false } : {} + const dispatchOutcome = notDispatched ? { failedStepDispatched: false } : {} // Private operator STDERR only; never HTTP/result/audit payload. try { const [step] = evidence.steps diff --git a/backend/src/services/config-mutation-policy.ts b/backend/src/services/config-mutation-policy.ts index 274123cf..45bb9e8a 100644 --- a/backend/src/services/config-mutation-policy.ts +++ b/backend/src/services/config-mutation-policy.ts @@ -20,7 +20,9 @@ export interface VerifiedMcpServiceMetadata { /** Snapshot of the exact server-decrypted sealed step, never public plan JSON. */ serviceInput:Record } -interface VerifiedChange { active: boolean; actorId: string; connectionId: string; approved: boolean; serverIds: number[]; objects: McpObjectRef[]; serverWide: boolean; authorize?: () => Promise; beforeDispatch?: () => Promise; serviceExecution?:VerifiedMcpServiceMetadata } +// `authorize` guards reads inside a stage and may reuse a recent proof; `authorizeWrite`, when given, is a +// fresh check before every native or application write (each write is a stage boundary). +interface VerifiedChange { active: boolean; actorId: string; connectionId: string; approved: boolean; serverIds: number[]; objects: McpObjectRef[]; serverWide: boolean; authorize?: () => Promise; authorizeWrite?: () => Promise; beforeDispatch?: () => Promise; serviceExecution?:VerifiedMcpServiceMetadata } const verified = new AsyncLocalStorage() declare const serviceExecutionBrand:unique symbol export type VerifiedMcpServiceExecution=Readonly<{[serviceExecutionBrand]:true}> @@ -67,14 +69,14 @@ function checkService(entry:ServiceExecutionScope){ export async function captureVerifiedMcpServiceExecution():Promise{ const context=verified.getStore(),entry=context&&serviceScopes.get(context) if(!entry)return deniedService();checkService(entry) - await entry.context.authorize!();checkService(entry) + await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry) const token=Object.freeze(Object.create(null)) as VerifiedMcpServiceExecution serviceTokens.set(token,entry);return token } export async function inspectVerifiedMcpServiceExecution(token:unknown){ const entry=token&&typeof token==="object"?serviceTokens.get(token):undefined if(!entry)return deniedService();checkService(entry) - await entry.context.authorize!();checkService(entry) + await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry) return freezeService({...entry.metadata,ownerId:entry.scope.actorId,connectionId:entry.scope.connectionId, approved:entry.scope.approved,serverWide:entry.scope.serverWide,serverIds:[...entry.scope.serverIds],objects:structuredClone(entry.scope.objects)}) } @@ -83,7 +85,7 @@ export async function inspectVerifiedMcpServiceExecution(token:unknown){ export async function inspectVerifiedMcpServiceExecutionLifetime(token:unknown):Promise>{ const entry=token&&typeof token==="object"?serviceTokens.get(token):undefined if(!entry)return deniedService();checkService(entry) - await entry.context.authorize!();checkService(entry) + await (entry.context.authorizeWrite??entry.context.authorize!)();checkService(entry) return Object.freeze({expiresAt:entry.metadata.originalExpiresAt,monotonicDeadline:entry.monotonicEnd,signal:entry.controller.signal}) } /** Scheduler handoff may leave an active original scope, never a retired one. */ @@ -118,7 +120,7 @@ export async function withVerifiedConfigurationChange(input: Omit { const context = verified.getStore() if (!context?.active || !context.authorize) throw new McpError("PERMISSION_DENIED", "Нет действующего контекста операции измерения", 403) - await context.authorize() + await (context.authorizeWrite ?? context.authorize)() } /** A full database replacement affects every protected domain, including non-router objects. */ @@ -146,7 +148,7 @@ export async function executeGuardedConfigurationWrite(serverId: number, meth const inherited = verified.getStore(), context = inherited?.active ? inherited : undefined // A service step can contain many native writes. Revalidate its live access // for each dispatch, rather than only once at the beginning of that step. - await context?.authorize?.() + await (context?.authorizeWrite ?? context?.authorize)?.() const actorId = context?.actorId ?? actor.getStore() ?? "system" const { rows } = await pool.query("SELECT object_ref FROM mcp_protections WHERE object_ref->>'serverId'=$1 OR (object_ref->>'kind'='server' AND object_ref->>'id'=$1)", [String(serverId)]) for (const row of rows) { @@ -192,7 +194,7 @@ export async function executeGuardedApplicationWrite(object: { kind: string; const serverIds = dependencies === "all" ? (await pool.query("SELECT id FROM servers")).rows.map(row => Number(row.id)) : typeof dependencies === "function" ? await dependencies() : dependencies if (serverIds.some(id => !Number.isSafeInteger(id) || id <= 0)) throw new McpError("INVALID_INPUT", "Некорректная зависимость сервера", 400) const inherited = verified.getStore(), context = inherited?.active ? inherited : undefined, actorId = context?.actorId ?? actor.getStore() ?? "system" - await context?.authorize?.() + await (context?.authorizeWrite ?? context?.authorize)?.() if (context && serverIds.some(id => !context.serverIds.includes(id))) throw new McpError("SERVER_DENIED", "Сервис затрагивает зависимость вне проверенного плана") const objects = affectedObjects ? await affectedObjects() : [] const { rows } = await pool.query("SELECT object_ref FROM mcp_protections WHERE object_ref->>'serverId'=ANY($1::text[]) OR (object_ref->>'kind'='server' AND object_ref->>'id'=ANY($1::text[])) OR (object_ref->>'kind'=$2 AND object_ref->>'id'=$3) OR EXISTS(SELECT 1 FROM jsonb_array_elements($4::jsonb) AS target WHERE target->>'kind'=object_ref->>'kind' AND target->>'id'=object_ref->>'id')", [serverIds.map(String), object.kind, object.id, JSON.stringify(objects)]) diff --git a/backend/src/services/vrf-production-gre-link-local.test.ts b/backend/src/services/vrf-production-gre-link-local.test.ts index 16f9a0b8..0c8939cb 100644 --- a/backend/src/services/vrf-production-gre-link-local.test.ts +++ b/backend/src/services/vrf-production-gre-link-local.test.ts @@ -234,7 +234,8 @@ for (const [label, current, mutate, guard] of midFlap) { // (d) mutation check: without re-reads the mid-flap read is refused. await assert.rejects(settleVrfTransition(flapping, classify, async () => structuredClone(current.after), noPause, 0), /VRF production transition refused/, label); cases++ } -// (c) genuine configuration changes are refused at once, with no re-read. +// (c) a genuine configuration change persists on the router: the same refusal on one fresh read is final. +// A refusal that does not repeat on the fresh read was a half-applied read and the fresh state is classified. for (const [label, current, mutate] of [ ["static address changed", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv4_address.find(row => greNames(s).includes(String(row.interface)))!.address = "10.99.99.1/30" }], ["foreign route row", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv6_route.push({ ".id": "*ffab", opaque: "foreign" }) }], @@ -243,8 +244,11 @@ for (const [label, current, mutate] of [ ] as const) { const changed = structuredClone(current.after); mutate(changed) let rereads = 0 - await assert.rejects(settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(current.after) }, noPause), /VRF production transition refused/, label) - assert.equal(rereads, 0, label); cases++ + await assert.rejects(settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(changed) }, noPause), /VRF production transition refused/, label) + assert.equal(rereads, 1, label); cases++ + rereads = 0 + const healed = await settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(current.after) }, noPause) + assert.equal(healed.status, "explained-native-transition", label); assert.equal(rereads, 1, label); cases++ } // Owner decision 2026-11-18 (stand experiment, docs/mcp-stand-run-a-plan.md 1dda615b): a membership PATCH that moves an // empty concrete bridge out of the VRF may lose its automatic LL; the loss is revival debt settled once by a reappearance. @@ -295,11 +299,11 @@ for (const [label, current, mutate] of [ // (d) still absent on the rejoin: the debt is carried, neither adopted nor refused. const carried = classifyVrfProductionTransition(input(left, withoutLl(rejoined, leaving), rejoin, leftPrevious)) assert.deepEqual(durable(carried.checkpoint).leftBridgeLl.map((item: { member: string }) => item.member), [leaving]); cases++ - // (e) refused at once, no settling re-read: non-empty bridge, changed bridge row, missing static address, extra missing row. + // (e) refused after one fresh read repeats the refusal: non-empty bridge, changed bridge row, missing static address, extra missing row. const refusedNow = async (before: VrfProductionSnapshot, after: VrfProductionSnapshot, label: string, reason: RegExp) => { let count = 0 await assert.rejects(settleVrfTransition(after, next => classifyVrfProductionTransition(input(before, next, transfer, chainPrevious)), async () => { count++; return after }, async () => {}), reason, label) - assert.equal(count, 0, label); cases++ + assert.equal(count, 1, label); cases++ } const bridgeId = String(reenabled.catalogs.bridge.find(row => row.name === leaving)![".id"]) const ported = (s: VrfProductionSnapshot) => { const copy = structuredClone(s); copy.catalogs.bridge_vlan.push({ ".id": "*ff70", bridge: leaving, "vlan-ids": "10", tagged: leaving, untagged: "", disabled: "false" }); return copy } @@ -331,15 +335,15 @@ for (const [label, current, mutate] of [ count = 0 const exhausted = await settleVrfTransition(lingering, classifyLeave, async () => { count++; return structuredClone(lingering) }, async () => {}).then(() => undefined, (error: unknown) => error) assert.equal(readVrfTransitionRefusal(exhausted)?.guard, "PATCH_BRIDGE_LL_CONNECTED_ROUTE_CLOSURE"); assert.equal(count, VRF_SETTLING_REREADS); cases++ - // The same lingering route with a non-empty bridge or a changed bridge row: refused at once (earlier checks). + // The same lingering route with a non-empty bridge or a changed bridge row: refused once the fresh read repeats it (earlier checks). const portedLinger = ported(lingering) count = 0 await assert.rejects(settleVrfTransition(portedLinger, after => classifyVrfProductionTransition(input(ported(reenabled), after, transfer, chainPrevious)), async () => { count++; return portedLinger }, async () => {}), /PATCH bridge LL empty configuration/) - assert.equal(count, 0); cases++ + assert.equal(count, 1); cases++ const changedLinger = structuredClone(lingering); changedLinger.catalogs.bridge.find(row => row.name === leaving)!.mtu = "1400" count = 0 await assert.rejects(settleVrfTransition(changedLinger, classifyLeave, async () => { count++; return changedLinger }, async () => {}), /opaque configuration drift/) - assert.equal(count, 0); cases++ + assert.equal(count, 1); cases++ } // Owner decision 2026-11-18 (run v24 step 31): the rejoin post-read holds the owed bridge's fe80 route but not yet its // LL address (non-atomic read). That shape alone is re-read (settling); nothing is accepted without the address. @@ -356,11 +360,11 @@ for (const [label, current, mutate] of [ count = 0 const exhausted = await settleVrfTransition(routeOnly, classifyRejoin(leftPrevious), async () => { count++; return structuredClone(routeOnly) }, async () => {}).then(() => undefined, (error: unknown) => error) assert.equal(readVrfTransitionRefusal(exhausted)?.guard, "FOREIGN_CATALOG_CONFIGURATION_ORDER_CHANGED"); assert.equal(count, VRF_SETTLING_REREADS); cases++ - // Immediate refusal (no re-read): no debt, an extra foreign row, a route of a bridge without debt, the route of a GRE. + // Refused when one fresh read repeats it: no debt, an extra foreign row, a route of a bridge without debt, the route of a GRE. const immediate = async (first: VrfProductionSnapshot, classify: (after: VrfProductionSnapshot) => unknown, label: string) => { let rereads = 0 - await assert.rejects(settleVrfTransition(first, classify as (after: VrfProductionSnapshot) => never, async () => { rereads++; return structuredClone(rejoined) }, async () => {}), /foreign ipv6_route configuration\/order changed/, label) - assert.equal(rereads, 0, label); cases++ + await assert.rejects(settleVrfTransition(first, classify as (after: VrfProductionSnapshot) => never, async () => { rereads++; return structuredClone(first) }, async () => {}), /foreign ipv6_route configuration\/order changed/, label) + assert.equal(rereads, 1, label); cases++ } await immediate(routeOnly, classifyRejoin(undefined), "no debt") const extraForeign = structuredClone(routeOnly); extraForeign.catalogs.ipv6_route.push({ ".id": "*ffc1", "dst-address": "2001:db8::/64", gateway: "foreign", dynamic: "false", "routing-table": "main" }) @@ -376,7 +380,7 @@ for (const [label, current, mutate] of [ const greAfter = structuredClone(gre.after); greAfter.catalogs.ipv6_route.push({ ...structuredClone(greRoute), ".id": "*ffc3" }) let rereads = 0 await assert.rejects(settleVrfTransition(greAfter, after => classifyVrfProductionTransition({ ...gre, after }), async () => { rereads++; return greAfter }, async () => {}), /VRF production transition refused/) - assert.equal(rereads, 0); cases++ + assert.equal(rereads, 1); cases++ } } // Owner decision 2026-11-18 (run v21 step 31): the reconcile of a verified checkpoint may see the owed bridge LL come diff --git a/backend/src/services/vrf-production-read.test.ts b/backend/src/services/vrf-production-read.test.ts index 331819f5..941533b5 100644 --- a/backend/src/services/vrf-production-read.test.ts +++ b/backend/src/services/vrf-production-read.test.ts @@ -63,7 +63,9 @@ for (const change of ["revoke", "disable", "source"] as const) { const m = model() m.afterGet(() => { if (change === "revoke") m.revoke(); if (change === "disable") m.disable(); if (change === "source") m.changeSource() }) await assert.rejects(readVrfProductionSnapshot(m.input)) - assert.equal(m.calls.length, 1) + // The fixed catalogs are fetched as one concurrent batch: reads already in flight finish, the snapshot + // is refused, and no closure read starts after the change was observed. + assert.ok(m.calls.length >= 1 && m.calls.length <= VRF_PRODUCTION_REQUIRED_CATALOGS.length, String(m.calls.length)) } const resolving = model(); resolving.afterResolve(() => resolving.revoke()) await assert.rejects(readVrfProductionSnapshot(resolving.input)); assert.equal(resolving.calls.length, 0) @@ -155,7 +157,10 @@ assert.deepEqual(await readVrfProductionSnapshot({ ...observed.input, observeRea assert.deepEqual(observed.calls, happy.calls) assert.deepEqual(observed.counters(), happy.counters()) assert.equal(observations.length, observed.calls.length * 5 + 2) -assert.deepEqual(observations.slice(0, 4).map(value => value.boundary), ["authority-before", "native-get", "authority-after", "shape"]) +// Catalogs are read concurrently; each catalog keeps its own boundary order. +for (const catalog of new Set(observations.flatMap(value => "catalog" in value && value.catalog ? [value.catalog] : []))) { + assert.deepEqual(observations.filter(value => "catalog" in value && value.catalog === catalog).map(value => value.boundary).slice(0, 4), ["authority-before", "native-get", "authority-after", "shape"], catalog) +} assert.ok(observations.every(value => value.serverId === 7)) assert.deepEqual(observations.at(-2), { serverId: 7, boundary: "final-authority" }) assert.deepEqual(observations.at(-1), { serverId: 7, boundary: "snapshot-validation" }) diff --git a/backend/src/services/vrf-production-read.ts b/backend/src/services/vrf-production-read.ts index 4a33762b..979d51a7 100644 --- a/backend/src/services/vrf-production-read.ts +++ b/backend/src/services/vrf-production-read.ts @@ -108,16 +108,19 @@ export async function readVrfProductionSnapshot(input: ReadVrfProductionInput): } const catalogs: VrfProductionSnapshot["catalogs"] = {} let totalBytes = 0 - const read = async (resource: string): Promise => { + const fetchCatalog = async (resource: string, reach: (boundary: VrfRawReadObservation["boundary"]) => void = () => {}): Promise => { + const at = (boundary: VrfRawReadObservation["boundary"]) => { reach(boundary); notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary }) } insist(Object.hasOwn(VRF_PRODUCTION_RAW_PATHS, resource), "unknown raw resource") - notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "authority-before" }) + at("authority-before") const source = await fence() - notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "native-get" }) + at("native-get") const payload = await source.client.get(VRF_PRODUCTION_RAW_PATHS[resource], 10000, undefined, MAX_BYTES) - notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "authority-after" }) + at("authority-after") await fence() - notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, catalog: resource, boundary: "shape" }) - const catalog = rows(resource, payload) + at("shape") + return rows(resource, payload) + } + const store = (resource: string, catalog: VrfProductionRow[]): void => { // Catalog shape succeeded. Budgets and subsequent closure validation are // snapshot work, not a failure of this completed native GET/catalog shape. notifyVrfDiagnostic(input.observeRead, { serverId: input.serverId, boundary: "snapshot-validation" }) @@ -126,8 +129,22 @@ export async function readVrfProductionSnapshot(input: ReadVrfProductionInput): insist(size < MAX_BYTES && totalBytes < MAX_TOTAL_BYTES, "raw evidence byte budget") catalogs[resource] = catalog } - for (const resource of VRF_PRODUCTION_REQUIRED_CATALOGS) await read(resource) - for (const resource of [...(input.requiredCatalogs ?? [])]) if (!Object.hasOwn(catalogs, resource)) await read(resource) + const read = async (resource: string): Promise => store(resource, await fetchCatalog(resource)) + // The fixed catalogs are independent: read them concurrently (bounded by the REST pool), which also + // narrows the window in which RouterOS can show an address and its route out of step. Stored in the + // fixed order, so the snapshot is identical to a sequential read. + const initial = [...new Set([...VRF_PRODUCTION_REQUIRED_CATALOGS, ...(input.requiredCatalogs ?? [])])] + const reached = new Map() + const observeRead = input.observeRead + const fetched = await Promise.allSettled(initial.map(resource => fetchCatalog(resource, boundary => reached.set(resource, boundary)))) + const failed = fetched.findIndex(result => result.status === "rejected") + if (failed !== -1) { + // Every read has settled; the last diagnostic names the first failing catalog in the fixed order. + const boundary = reached.get(initial[failed]) + if (boundary) notifyVrfDiagnostic(observeRead, { serverId: input.serverId, catalog: initial[failed], boundary }) + throw (fetched[failed] as PromiseRejectedResult).reason + } + initial.forEach((resource, index) => store(resource, (fetched[index] as PromiseFulfilledResult).value)) const selected = new Set() if (request.action === "create") insist(request.id === undefined && request.fields.interfaces !== undefined, "CREATE selection") else { diff --git a/backend/src/services/vrf-transition-refusal.ts b/backend/src/services/vrf-transition-refusal.ts index 6e866687..c9880bd2 100644 --- a/backend/src/services/vrf-transition-refusal.ts +++ b/backend/src/services/vrf-transition-refusal.ts @@ -149,12 +149,21 @@ export function noteVrfTransitionSettling(error: unknown): void { export function isVrfTransitionSettling(error: unknown): boolean { return !!error && typeof error === "object" && settling.has(error) } -/** First successful classification wins. Any other refusal, read or authority - * error propagates at once; after the bound the last refusal object is rethrown. */ +/** First successful classification wins. RouterOS applies addresses, routes and + * link state separately and the raw after-read is a series of GETs, so a refusal + * seen once may describe a half-applied state: a tagged refusal is final only when + * the same guard repeats on a fresh read. Refusals marked settling keep re-reading + * up to the bound. Untagged errors (read, authority, shape) propagate at once; + * after the bound the last refusal object is rethrown. */ export async function settleVrfTransition(first: S, classify: (after: S) => T, reread: () => Promise, pause = () => new Promise(resolve => setTimeout(resolve, VRF_SETTLING_PAUSE_MS)), rereads = VRF_SETTLING_REREADS): Promise { - let after = first + let after = first, previous: VrfTransitionGuard | undefined for (let attempt = 0; ; attempt++) { - try { return classify(after) } catch (error) { if (attempt >= rereads || !isVrfTransitionSettling(error)) throw error } + try { return classify(after) } catch (error) { + const guard = readVrfTransitionRefusal(error)?.guard + const repeated = guard !== undefined && guard === previous && !isVrfTransitionSettling(error) + if (attempt >= rereads || guard === undefined && !isVrfTransitionSettling(error) || repeated) throw error + previous = guard + } await pause() after = await reread() }