diff --git a/backend/src/services/vrf-execution-diagnostic.test.mjs b/backend/src/services/vrf-execution-diagnostic.test.mjs index 4fb0cbc3..06db5323 100644 --- a/backend/src/services/vrf-execution-diagnostic.test.mjs +++ b/backend/src/services/vrf-execution-diagnostic.test.mjs @@ -22,6 +22,7 @@ async function pureModuleUri(name){ } const diagnosticUri=await pureModuleUri('./vrf-execution-diagnostic.ts') const diagnostic=await import(diagnosticUri) +const refusalUri=await pureModuleUri('./vrf-transition-refusal.ts'),refusal=await import(refusalUri) const id='12345678-1234-4123-8123-123456789abc' let cases=0 for(const [error,expected]of [[Error('SECRET'),'Error'],[new TypeError('SECRET'),'TypeError'],[new RangeError('SECRET'),'RangeError'],[new SyntaxError('SECRET'),'SyntaxError'],[new ReferenceError('SECRET'),'ReferenceError'],[new URIError('SECRET'),'URIError'],[new EvalError('SECRET'),'EvalError'],[{message:'SECRET'},'NonError'],[new Proxy(Error('SECRET'),{get(){throw Error('getter')}}),'NonError']]){ @@ -51,12 +52,12 @@ const functions=['readVrfProductionSnapshot','claimStoredVrfProductionTicket','i const stubUri=uri(`export const McpError=globalThis.${key}.McpError;export const pool={query:(...args)=>globalThis.${key}.query(...args)};`+functions.map(name=>`export const ${name}=(...args)=>globalThis.${key}.${name}(...args);`).join('\n')) let emitted=await compile('./vrf-production-execution.ts') const ast=ts.createSourceFile('executor.js',emitted,ts.ScriptTarget.ES2022,true,ts.ScriptKind.JS) -const edits=ast.statements.filter(ts.isImportDeclaration).map(node=>({start:node.moduleSpecifier.getStart(ast),end:node.moduleSpecifier.end,value:node.moduleSpecifier.text.endsWith('/vrf-execution-diagnostic.js')?diagnosticUri:stubUri})) +const edits=ast.statements.filter(ts.isImportDeclaration).map(node=>({start:node.moduleSpecifier.getStart(ast),end:node.moduleSpecifier.end,value:node.moduleSpecifier.text.endsWith('/vrf-execution-diagnostic.js')?diagnosticUri:node.moduleSpecifier.text.endsWith('/vrf-transition-refusal.js')?refusalUri:stubUri})) for(const edit of edits.reverse())emitted=emitted.slice(0,edit.start)+JSON.stringify(edit.value)+emitted.slice(edit.end) const api=await import(uri(emitted)) function fixture(options={}){ const failure=Object.assign(Error('PRIVATE_PASSWORD native-body SQL /private/path'),{code:options.errno??'ETIMEDOUT'}) - const events=[],calls=[];let reads=0,deps=0,access=0,inspect=0,routers=0,guard,clientCalls=0 + const events=[],calls=[];let settles=0,reads=0,deps=0,access=0,inspect=0,routers=0,guard,clientCalls=0 const proof='a'.repeat(64),snapshot={projection:'raw-native-v1',serverId:1,sourceProof:proof,catalogs:{vrf:[{'.id':'*0',name:'main',interfaces:'all',disabled:'false'}],interface_inventory:[]}} const fail=name=>{calls.push(name);if(options.fail===name)throw failure} const f={McpError, @@ -83,7 +84,9 @@ function fixture(options={}){ async encryptSecret(){return'PRIVATE_CIPHERTEXT'},async decryptSecretExisting(){throw Error('not used')}, inspectVrfProductionAdmission(){fail('inspect-'+(++inspect));return{admission:{}}}, assertVrfProductionHistoryProvenance(){},assertVrfProductionTransitionCheckpoint(){},openVrfProductionTransitionCheckpoint(){},vrfProductionCheckpointCatalogs(){return[]}, - classifyVrfProductionTransition(){fail('classification');return{checkpoint:{}}},sealVrfProductionTransitionCheckpoint(){return'PRIVATE_HISTORY'}, + classifyVrfProductionTransition(){ + if(settles<(options.settling??0)){settles++;const error=Error('PRIVATE mid-flap');refusal.noteVrfTransitionRefusal(error,'address/route absence outside exact acquisition');refusal.noteVrfTransitionSettling(error);throw error} + fail('classification');return{checkpoint:{}}},sealVrfProductionTransitionCheckpoint(){return'PRIVATE_HISTORY'}, } globalThis[key]=f const input={operationId:id,planId:'plan',connectionId:'connection',ownerId:'owner',operationName:options.action==='delete'?'ros.vrf.delete':'ros.vrf.create',operationVersion:1,stepIndex:0, @@ -92,7 +95,7 @@ function fixture(options={}){ async authorize(){},async assertDependencies(){fail('dependencies-'+(++deps))},installBeforeDispatch(value){fail('install');guard=value}, onDiagnostic(value){events.push(value);if(options.observerThrows)throw Error('logging failed');if(options.observerRejects)return Promise.reject(Error('async logging failed'))}, } - return{failure,events,calls,input,clientCalls:()=>clientCalls,run:()=>api.executeVrfProductionStep(input)} + return{failure,events,calls,input,clientCalls:()=>clientCalls,reads:()=>reads,run:()=>api.executeVrfProductionStep(input)} } for(const [fail,stage,ack]of [ ['access-1','initial-authorization',false],['sealing-key','sealing-key',false],['read-1','raw-before',false], @@ -121,6 +124,11 @@ const history=fixture({action:'delete',fail:'history-load'});await assert.reject const malformed=fixture({badAck:true});await assert.rejects(malformed.run(),error=>error.code==='STATE_CONFLICT'&&!(error instanceof api.VrfProductionAcknowledgedFailure));assert.equal(malformed.events[0].stage,'ack-identity');assert.equal(malformed.events[0].executorAckObserved,true);cases++ const malformedDelete=fixture({action:'delete',badDeleteBody:true});await assert.rejects(malformedDelete.run(),error=>error instanceof api.VrfProductionAcknowledgedFailure&&error.code==='VERIFICATION_FAILED');assert.equal(malformedDelete.events[0].executorAckObserved,true);assert.equal(malformedDelete.events[0].stage,'ack-identity');cases++ const happy=fixture();const receipt=await happy.run();assert.equal(receipt.configurationVerified,true);assert.equal(happy.events.length,0);assert.equal(happy.clientCalls(),1);cases++ +// Bounded post-ACK settling: real executor pauses, re-reads the full raw-after and never re-dispatches. +const settledOnce=fixture({settling:1});assert.equal((await settledOnce.run()).configurationVerified,true);assert.equal(settledOnce.reads(),5);assert.equal(settledOnce.clientCalls(),1);assert.equal(settledOnce.events.length,0);cases++ +const neverSettled=fixture({settling:99});await assert.rejects(neverSettled.run(),error=>error instanceof api.VrfProductionAcknowledgedFailure&&error.code==='VERIFICATION_FAILED') +assert.equal(neverSettled.reads(),3+1+refusal.VRF_SETTLING_REREADS);assert.equal(neverSettled.clientCalls(),1);assert.equal(neverSettled.events.length,1) +assert.equal(neverSettled.events[0].stage,'transition-classification');assert.equal(neverSettled.events[0].attempts,1+refusal.VRF_SETTLING_REREADS);assert.deepEqual(neverSettled.events[0].transition,{guard:'ADDRESS_ROUTE_ABSENCE_OUTSIDE_EXACT_ACQUISITION'});cases++ const silent=fixture({fail:'read-1'});delete silent.input.onDiagnostic;await assert.rejects(silent.run(),error=>error===silent.failure);cases++ // Exercise the exact emitted runtime policy call/callback/fallback/clear wiring. // Only its surrounding policy and executor I/O are synthetic. Extracting the diff --git a/backend/src/services/vrf-execution-diagnostic.ts b/backend/src/services/vrf-execution-diagnostic.ts index 4f4dfa7e..65ede6ca 100644 --- a/backend/src/services/vrf-execution-diagnostic.ts +++ b/backend/src/services/vrf-execution-diagnostic.ts @@ -25,10 +25,12 @@ export interface VrfExecutionDiagnostic { executorAckObserved: boolean read?: VrfRawReadObservation transition?: VrfTransitionRefusalTag + /** Post-ACK classification attempts, present only when bounded re-reads ran. */ + attempts?: number } /** Closed private diagnostic only. Never read error messages, stacks, getters, * causes, request bodies, SQL or native values. No outcome/authority is minted. */ -export function vrfExecutionDiagnostic(operationId: string, action: string, stage: VrfExecutionStage, error: unknown, executorAckObserved: boolean, read?: VrfRawReadObservation): VrfExecutionDiagnostic | undefined { +export function vrfExecutionDiagnostic(operationId: string, action: string, stage: VrfExecutionStage, error: unknown, executorAckObserved: boolean, read?: VrfRawReadObservation, attempts?: number): VrfExecutionDiagnostic | undefined { if (typeof operationId !== "string" || !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(operationId) || !["create", "patch", "delete"].includes(action) || !VRF_EXECUTION_STAGES.includes(stage) || typeof executorAckObserved !== "boolean") return undefined const result: VrfExecutionDiagnostic = { event: "mcp.vrf.execution-failed", operationId, action: action as VrfExecutionDiagnostic["action"], stage, errorClass: "NonError", category: "other", executorAckObserved } if (types.isNativeError(error)) { @@ -39,6 +41,7 @@ export function vrfExecutionDiagnostic(operationId: string, action: string, stag if (read && Number.isSafeInteger(read.serverId) && read.serverId > 0 && VRF_READ_BOUNDARIES.includes(read.boundary) && (read.catalog === undefined || typeof read.catalog === "string" && catalogs.has(read.catalog))) result.read = { serverId: read.serverId, boundary: read.boundary, ...(read.catalog === undefined ? {} : { catalog: read.catalog }) } const transition = readVrfTransitionRefusal(error) if (transition) result.transition = transition + if (Number.isSafeInteger(attempts) && attempts! > 1 && attempts! <= 16) result.attempts = attempts return result } /** Observers are in-process only and cannot fail, await, or replace the work. */ diff --git a/backend/src/services/vrf-production-execution.ts b/backend/src/services/vrf-production-execution.ts index b7c8f580..dfa06dcc 100644 --- a/backend/src/services/vrf-production-execution.ts +++ b/backend/src/services/vrf-production-execution.ts @@ -9,6 +9,7 @@ import { assertVrfActionContext, vrfNativeActionFor, vrfActionContractsCurrent, import { findMcpOperation } from "../modules/mcp/registry.js" import { decryptSecretExisting, encryptSecret, fingerprint, McpError, withVrfProductionSealingKey } from "../modules/mcp/security.js" import { configurationRows, resourceById, routerClient, rosSnapshotSourceProof, type RosSnapshot, type RosStep } from "../modules/mcp/resources.js" +import { settleVrfTransition } from "./vrf-transition-refusal.js" import { notifyVrfDiagnostic, vrfExecutionDiagnostic, type VrfExecutionDiagnostic, type VrfExecutionStage, type VrfRawReadObservation } from "./vrf-execution-diagnostic.js" export interface VrfProductionExecutionInput { @@ -134,14 +135,14 @@ export async function executeVrfProductionStep(original: VrfProductionExecutionI } let acknowledged = false let executorAckObserved = false, diagnosticReported = false - let stage: VrfExecutionStage = "initial-authorization", read: VrfRawReadObservation | undefined + let stage: VrfExecutionStage = "initial-authorization", read: VrfRawReadObservation | undefined, rereads = 0 const mark = (next: VrfExecutionStage) => { stage = next; read = undefined } const observeRead = (value: VrfRawReadObservation) => { read = value } const report = (error: unknown) => { if (diagnosticReported) return diagnosticReported = true try { - const diagnostic = vrfExecutionDiagnostic(input.operationId, request.action, stage, error, executorAckObserved, read) + const diagnostic = vrfExecutionDiagnostic(input.operationId, request.action, stage, error, executorAckObserved, read, rereads ? rereads + 1 : undefined) if (diagnostic) notifyVrfDiagnostic(input.onDiagnostic, diagnostic) } catch { /* Original exception and acknowledgement handling stay authoritative. */ } } @@ -219,9 +220,19 @@ export async function executeVrfProductionStep(original: VrfProductionExecutionI mark("post-ack-authorization") await authorize() mark("raw-after") - const after = await readVrfProductionSnapshot({ serverId: binding.serverId, expectedSourceProof: binding.sourceProof, request: vrfProductionReadSelection(input.step, before.catalogs.vrf.find(row => row[".id"] === request.id)?.interfaces), requiredCatalogs: Object.keys(before.catalogs), assertAuthority: authorize, observeRead }) - mark("transition-classification") - const classified = classifyVrfProductionTransition({ before, after, binding, request, sealed, secret, now: Date.now(), acknowledgedId, previous }) + const afterSelection = vrfProductionReadSelection(input.step, before.catalogs.vrf.find(row => row[".id"] === request.id)?.interfaces) + const readAfter = () => readVrfProductionSnapshot({ serverId: binding.serverId, expectedSourceProof: binding.sourceProof, request: afterSelection, requiredCatalogs: Object.keys(before.catalogs), assertAuthority: authorize, observeRead }) + // Same fenced read and full classification per attempt; the native write is never repeated. + const classified = await settleVrfTransition(await readAfter(), after => { + mark("transition-classification") + return classifyVrfProductionTransition({ before, after, binding, request, sealed, secret, now: Date.now(), acknowledgedId, previous }) + }, async () => { + rereads++ + mark("post-ack-authorization") + await authorize() + mark("raw-after") + return readAfter() + }) mark("post-ack-dependencies") await input.assertDependencies() mark("post-ack-final-authorization") diff --git a/backend/src/services/vrf-production-gre-link-local.test.ts b/backend/src/services/vrf-production-gre-link-local.test.ts index f6ed1161..116d41ea 100644 --- a/backend/src/services/vrf-production-gre-link-local.test.ts +++ b/backend/src/services/vrf-production-gre-link-local.test.ts @@ -2,6 +2,8 @@ import assert from "node:assert/strict" import { createHmac } from "node:crypto" import { readFile } from "node:fs/promises" import { sealVrfProductionAdmission, VRF_PRODUCTION_REQUIRED_CATALOGS, type VrfProductionRequest, type VrfProductionSnapshot, type VrfProductionRow as Row } from "./vrf-production-admission.js" +import { readVrfTransitionRefusal, settleVrfTransition, VRF_SETTLING_REREADS } from "./vrf-transition-refusal.js" +import { vrfExecutionDiagnostic } from "./vrf-execution-diagnostic.js" import { vrfProductionDependencyObjects, classifyVrfProductionTransition, sealVrfProductionTransitionCheckpoint, openVrfProductionTransitionCheckpoint, assertVrfProductionHistoryProvenance, type VrfProductionTransitionCheckpoint } from "./vrf-production-transition.js" const secret = Buffer.alloc(32, 7), now = 1000, sourceProof = "a".repeat(64) @@ -207,4 +209,41 @@ assertVrfProductionHistoryProvenance(secretOpened, secretAfter); cases++ const currentSecretDrift = structuredClone(secretAfter) currentSecretDrift.catalogs.gre[0]["ipsec-secret"] = "changed-during-current-transition" assert.throws(() => classifyVrfProductionTransition(input(secretBefore, currentSecretDrift, base.request, undefined, base.acknowledgedId))); cases++ +// Bounded post-ACK settling over a non-atomic raw-after read taken while RouterOS +// cycles the active GRE member. Mid-flap shapes are synthetic edits of the model above. +const greNames = (s: VrfProductionSnapshot) => s.catalogs.gre.map(row => String(row.name)) +const midFlap: Array<[string, ReturnType, (s: VrfProductionSnapshot) => void, string]> = [ + ["create GRE LL absent", inputs[0], s => { s.catalogs.ipv6_address = s.catalogs.ipv6_address.filter(row => !(row.dynamic === "true" && greNames(s).includes(String(row.interface)))) }, "ADDRESS_ROUTE_ABSENCE_OUTSIDE_EXACT_ACQUISITION"], + ["create GRE down", inputs[0], s => { for (const row of s.catalogs.interface_inventory) if (greNames(s).includes(String(row.name))) row.running = "false" }, "BOUNDED_GRE_CYCLE_ACTIVE_STATE"], + ["patch GRE route without immediate-gw", inputs[4], s => { for (const row of s.catalogs.ipv6_route) if (greNames(s).some(name => String(row.gateway).startsWith(name))) delete row["immediate-gw"] }, "MISSING_AMBIGUOUS_PROVEN_CONNECTED_ROUTE"], + ["delete GRE LL not yet revived", inputs[5], s => { s.catalogs.ipv6_address = s.catalogs.ipv6_address.filter(row => !(row.dynamic === "true" && greNames(s).includes(String(row.interface)))) }, "DELETE_MISSING_EXPECTED_LL_REVIVAL"], +] +const noPause = async () => {} +for (const [label, current, mutate, guard] of midFlap) { + const flapping = structuredClone(current.after); mutate(flapping) + const classify = (after: VrfProductionSnapshot) => classifyVrfProductionTransition({ ...current, after }) + // (a) first read mid-flap, second read stable: classification passes after one re-read. + let rereads = 0 + const settled = await settleVrfTransition(flapping, classify, async () => { rereads++; return structuredClone(current.after) }, noPause) + assert.equal(settled.status, "explained-native-transition", label); assert.equal(rereads, 1, label); cases++ + // (b) every read mid-flap: the same refusal after the bound, attempts counted. + rereads = 0 + const exhausted = await settleVrfTransition(flapping, classify, async () => { rereads++; return structuredClone(flapping) }, noPause).then(() => undefined, (error: unknown) => error) + assert.deepEqual(readVrfTransitionRefusal(exhausted)?.guard, guard, label); assert.equal(rereads, VRF_SETTLING_REREADS, label) + assert.equal(vrfExecutionDiagnostic("12345678-1234-4123-8123-123456789abc", current.request.action, "transition-classification", exhausted, true, undefined, rereads + 1)?.attempts, 1 + VRF_SETTLING_REREADS, label); cases++ + // (d) mutation check: without re-reads the mid-flap read is refused. + await assert.rejects(settleVrfTransition(flapping, classify, async () => structuredClone(current.after), noPause, 0), /VRF production transition refused/, label); cases++ +} +// (c) genuine configuration changes are refused at once, with no re-read. +for (const [label, current, mutate] of [ + ["static address changed", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv4_address.find(row => greNames(s).includes(String(row.interface)))!.address = "10.99.99.1/30" }], + ["foreign route row", inputs[0], (s: VrfProductionSnapshot) => { s.catalogs.ipv6_route.push({ ".id": "*ffab", opaque: "foreign" }) }], + ["GRE endpoint changed", inputs[4], (s: VrfProductionSnapshot) => { s.catalogs.gre[0]["remote-address"] = "198.18.245.99" }], + ["GRE disabled", inputs[0], (s: VrfProductionSnapshot) => { for (const row of s.catalogs.interface_inventory) if (greNames(s).includes(String(row.name))) { row.disabled = "true"; row.running = "false" } }], +] as const) { + const changed = structuredClone(current.after); mutate(changed) + let rereads = 0 + await assert.rejects(settleVrfTransition(changed, after => classifyVrfProductionTransition({ ...current, after }), async () => { rereads++; return structuredClone(current.after) }, noPause), /VRF production transition refused/, label) + assert.equal(rereads, 0, label); cases++ +} console.log("GRE captured auto-LL, synthetic lifecycle, durable endpoints, negative conservation cases:", cases) diff --git a/backend/src/services/vrf-production-transition.ts b/backend/src/services/vrf-production-transition.ts index 73a2eb1b..b7e96c8f 100644 --- a/backend/src/services/vrf-production-transition.ts +++ b/backend/src/services/vrf-production-transition.ts @@ -1,4 +1,4 @@ -import { noteVrfTransitionRefusal, withVrfTransitionSection, type VrfTransitionGuard } from "./vrf-transition-refusal.js" +import { noteVrfTransitionRefusal, noteVrfTransitionSettling, withVrfTransitionSection, type VrfTransitionGuard } from "./vrf-transition-refusal.js" import { isIP } from "node:net" import { isDeepStrictEqual } from "node:util" import { createHash, createHmac, timingSafeEqual } from "node:crypto" @@ -97,6 +97,12 @@ function must(value: unknown, reason: string, guard?: VrfTransitionGuard): asser throw error } } +/** Marks a refusal as possibly mid-flap only when `transient()` holds for the + * RouterOS-derived after-state of a no-MAC GRE member; the refusal is unchanged. */ +function settling(transient: () => unknown, work: () => T): T { + try { return work() } catch (error) { try { if (transient()) noteVrfTransitionSettling(error) } catch { /* Keep the original refusal. */ } throw error } +} +const noMacGre = (snapshot: VrfProductionSnapshot, name: string) => snapshot.catalogs.interface_inventory.some(row => row.name === name && row.type === "gre-tunnel" && !Object.hasOwn(row, "mac-address")) function nativeId(value: unknown): string { must(typeof value === "string" && /^\*[a-f0-9]+$/i.test(value), "native ID"); return value.slice(1).replace(/^0+/, "").toLowerCase() || "0" } const projected = vrfProductionConfiguration function same(resource: string, old: Row, current: Row, omit: readonly string[] = []): void { must(isDeepStrictEqual(projected(resource, old, omit), projected(resource, current, omit)), "opaque configuration drift") } @@ -293,7 +299,7 @@ function memberInventoryOmissions(before: VrfProductionSnapshot, after: VrfProdu if (isDeepStrictEqual(witness(old, greMemberChronology), witness(current, greMemberChronology))) return omit const previousContext = context(before.catalogs.vrf, name), nextContext = context(after.catalogs.vrf, name) must(old.type === "gre-tunnel" && current.type === "gre-tunnel" && old.name === name && current.name === name && old[".id"] === current[".id"] && !Object.hasOwn(old, "mac-address") && !Object.hasOwn(current, "mac-address") && previousContext !== nextContext && old.vrf === previousContext && current.vrf === nextContext, "bounded GRE cycle member identity/context") - must(no(old.disabled) && no(current.disabled) && old.running === "true" && current.running === "true", "bounded GRE cycle active state") + settling(() => no(old.disabled) && no(current.disabled) && old.running === "true", () => must(no(old.disabled) && no(current.disabled) && old.running === "true" && current.running === "true", "bounded GRE cycle active state")) const a = greParent(before, old), b = greParent(after, current) const parent = (snapshot: VrfProductionSnapshot, resource: string, id: unknown) => snapshot.catalogs[resource]?.find(row => row[".id"] === id) const oldGre = before.catalogs.gre.find(row => row.name === name), newGre = after.catalogs.gre.find(row => row.name === name) @@ -371,7 +377,7 @@ export function classifyVrfProductionTransition(input: VrfProductionTransitionIn must(a[0].vrf === oldContext.get(name) && b[0].vrf === newContext.get(name), "member native VRF context") if (a[0].status !== b[0].status) { const x = String(a[0].status), y = String(b[0].status) - must(x.length === 24 && y.length === 24 && x.slice(0, 20) === y.slice(0, 20) && x.slice(21) === y.slice(21) && x[20] === (oldContext.get(name) === "main" ? "-" : "V") && y[20] === (newContext.get(name) === "main" ? "-" : "V"), "member status native shape") + settling(() => noMacGre(before, name) && noMacGre(after, name) && no(a[0].disabled) && no(b[0].disabled) && a[0].running === "true" && b[0].running !== "true", () => must(x.length === 24 && y.length === 24 && x.slice(0, 20) === y.slice(0, 20) && x.slice(21) === y.slice(21) && x[20] === (oldContext.get(name) === "main" ? "-" : "V") && y[20] === (newContext.get(name) === "main" ? "-" : "V"), "member status native shape")) } same("interface_inventory", a[0], b[0], memberInventoryOmissions(before, after, name, a[0], b[0])); explain("interface_inventory", a[0], b[0]) } @@ -388,10 +394,10 @@ export function classifyVrfProductionTransition(input: VrfProductionTransitionIn const successor = successors[0], oldRoute = route(before, source) if (!successor) { if (request.action === "delete" && source.ll) { - withVrfTransitionSection("LL-comparison", () => deletedBridgeLinkLocal(before, after, old!, source, oldRoute, beforeSources, afterSources)) + settling(() => changes.includes(source.member) && noMacGre(before, source.member) && noMacGre(after, source.member), () => withVrfTransitionSection("LL-comparison", () => deletedBridgeLinkLocal(before, after, old!, source, oldRoute, beforeSources, afterSources))) explain(source.resource, source.row, undefined); explain(routeResource(source), oldRoute, undefined); continue } - must(source.ll && transfer && current!.interfaces === source.member && oldContext.get(source.member) === "main", "address/route absence outside exact acquisition") + settling(() => source.ll && changes.includes(source.member) && noMacGre(before, source.member) && noMacGre(after, source.member), () => must(source.ll && transfer && current!.interfaces === source.member && oldContext.get(source.member) === "main", "address/route absence outside exact acquisition")) absences.push({ address: structuredClone(source.row), route: structuredClone(oldRoute), member: source.member }); explain(source.resource, source.row, undefined); explain(routeResource(source), oldRoute, undefined); continue } usedAfter.add(successor) @@ -401,7 +407,7 @@ export function classifyVrfProductionTransition(input: VrfProductionTransitionIn withVrfTransitionSection("LL-comparison", () => same(source.resource, source.row, successor.row, moved && concreteNoMacGre ? [".id", "vrf"] : [".id"])) } else { must(source.row[".id"] === successor.row[".id"], "static address identity changed"); same(source.resource, source.row, successor.row, ["vrf"]) } - const newRoute = route(after, successor) + const newRoute = settling(() => moved && noMacGre(before, source.member) && noMacGre(after, source.member) && !after.catalogs[routeResource(successor)].some(row => routeFor(row, successor, after.catalogs.vrf)), () => route(after, successor)) must(Object.hasOwn(oldRoute, "local-address") === Object.hasOwn(newRoute, "local-address"), "connected route local-address shape changed") const ownedDelete = request.action === "delete" && oldContext.get(source.member) === old!.name withVrfTransitionSection("connected-route", () => same(routeResource(source), oldRoute, newRoute, moved ? [".id", "gateway", "immediate-gw", "routing-table", "local-address"] : transfer || ownedDelete ? [".id"] : [])) @@ -416,7 +422,7 @@ export function classifyVrfProductionTransition(input: VrfProductionTransitionIn same(routeResource(successor), missing[0].route, nextRoute, [".id", "gateway", "immediate-gw", "routing-table", "local-address"]) explain(successor.resource, undefined, successor.row); explain(routeResource(successor), undefined, nextRoute) } - for (const missing of history?.absentLl ?? []) if (!afterSources.some(source => source.member === missing.member && source.ll)) { must(request.action !== "delete", "DELETE missing expected LL revival"); absences.push(structuredClone(missing)) } + for (const missing of history?.absentLl ?? []) if (!afterSources.some(source => source.member === missing.member && source.ll)) { settling(() => changes.includes(missing.member) && noMacGre(before, missing.member) && noMacGre(after, missing.member), () => must(request.action !== "delete", "DELETE missing expected LL revival")); absences.push(structuredClone(missing)) } let active: Row | undefined, dormant: Row | undefined withVrfTransitionSection("virtual", () => { for (const resource of ["routing_table", "interface_inventory"]) { diff --git a/backend/src/services/vrf-transition-refusal.ts b/backend/src/services/vrf-transition-refusal.ts index 9bd95b99..1fa7ac6a 100644 --- a/backend/src/services/vrf-transition-refusal.ts +++ b/backend/src/services/vrf-transition-refusal.ts @@ -128,6 +128,31 @@ export function noteVrfTransitionRefusal(error: Error, reason: string, explicit? if (types.isNativeError(error) && guard && guards.has(guard)) owned.set(error, Object.freeze({ guard })) } catch { /* Diagnostic recording cannot replace the original refusal. */ } } +/** Bounded post-ACK settling. RouterOS cycles an active no-MAC GRE member when + * its VRF context changes (link-downs +1 in the same second the VRF comes up), + * and the raw-after read is not atomic. Only refusals the classifier marked at + * the throw site as RouterOS-derived after-state of such a member re-read the + * router; the guard and the error object stay unchanged. Worst case adds + * VRF_SETTLING_REREADS full fenced raw reads plus their pauses. */ +export const VRF_SETTLING_REREADS = 3 +export const VRF_SETTLING_PAUSE_MS = 1000 +const settling = new WeakSet() +export function noteVrfTransitionSettling(error: unknown): void { + try { if (owned.has(error as object)) settling.add(error as object) } catch { /* Never replace the original refusal. */ } +} +export function isVrfTransitionSettling(error: unknown): boolean { + return !!error && typeof error === "object" && settling.has(error) +} +/** First successful classification wins. Any other refusal, read or authority + * error propagates at once; after the bound the last refusal object is rethrown. */ +export async function settleVrfTransition(first: S, classify: (after: S) => T, reread: () => Promise, pause = () => new Promise(resolve => setTimeout(resolve, VRF_SETTLING_PAUSE_MS)), rereads = VRF_SETTLING_REREADS): Promise { + let after = first + for (let attempt = 0; ; attempt++) { + try { return classify(after) } catch (error) { if (attempt >= rereads || !isVrfTransitionSettling(error)) throw error } + await pause() + after = await reread() + } +} export function readVrfTransitionRefusal(error: unknown): VrfTransitionRefusalTag | undefined { if (!error || typeof error !== "object") return undefined const value = owned.get(error)