Files
MikrotikManager/backend/src/services/bandwidth-admission.test.ts
T

247 lines
21 KiB
TypeScript

import assert from "node:assert/strict"
import { createHmac } from "node:crypto"
import { test } from "node:test"
import { setImmediate as turn, setTimeout as delay } from "node:timers/promises"
import { createBandwidthAdmission, BandwidthAdmissionError, type BandwidthAdmissionInput, type BandwidthAdmissionDependencies, type BandwidthBinding, type BandwidthDispatchCapability, type BandwidthJournalAck, type BandwidthTransition } from "./bandwidth-admission.js"
const canonical = (value: unknown): string => Array.isArray(value) ? "[" + value.map(canonical).join(",") + "]" : value && typeof value === "object" ? "{" + Object.entries(value).sort(([a], [b]) => a.localeCompare(b)).map(([key, item]) => JSON.stringify(key) + ":" + canonical(item)).join(",") + "}" : JSON.stringify(value)
const proof = async (purpose: string, value: unknown) => createHmac("sha256", "test-key-not-a-human-credential").update(purpose + ":" + canonical(value)).digest("hex")
const deferred = <T>() => { let resolve!: (value: T) => void; const promise = new Promise<T>(done => { resolve = done }); return { promise, resolve } }
const hasCode = (code: string) => (error: unknown) => error instanceof BandwidthAdmissionError && error.code === code
function fixture() {
const input: BandwidthAdmissionInput = { attemptId: "11111111-1111-4111-8111-111111111111", callerKind: "rest-speed", operationId: "run-1", request: { runId: "run-1", rate: 100 }, source: { canonicalNodeId: "physical:A", endpointProof: "a".repeat(64) }, destination: { canonicalNodeId: "physical:B", endpointProof: "b".repeat(64) }, authorityProof: "c".repeat(64), authorityGeneration: "actor-7", controlGeneration: "enrolled-2", expiresAt: Date.now() + 10000, durationSec: 10, nativeBody: { address: "192.0.2.2", user: "destination-user", password: "destination-secret", protocol: "tcp", direction: "both", duration: "10s", "local-tx-speed": "1000000" } }
const events: string[] = [], faults = new Map<BandwidthTransition, string>()
let occupancy = "free", sends = 0, authorized = true, endpointsCurrent = true, ackNumber = 0, lastBinding: Readonly<BandwidthBinding> | undefined
let sendBehavior = async (): Promise<unknown> => [{ status: "done testing", "tx-total-average": "100" }]
let armDelay: Promise<void> | undefined
let authorizeHook: (() => void) | undefined
let completeHook: (() => void) | undefined
let terminalHook: (() => void) | undefined
const ack = (transition: BandwidthTransition, binding: Readonly<BandwidthBinding>): BandwidthJournalAck => {
const kind = faults.get(transition)
if (kind === "throw") throw new Error("lost transaction ACK")
if (kind === "missing") return undefined as unknown as BandwidthJournalAck
return { outcome: kind === "existing" ? "existing" as "new" : "new", transition, ackId: kind === "duplicate" ? "ack-1" : `ack-${++ackNumber}`, attemptId: kind === "wrong" ? "foreign" : binding.attemptId, bindingProof: binding.bindingProof, controlGeneration: binding.controlGeneration }
}
const deps: BandwidthAdmissionDependencies = {
now: Date.now, cleanupBudgetMs: 100,
keyedProof: async (purpose, value) => { if (purpose === "bandwidth-terminal-v1") terminalHook?.(); return proof(purpose, value) },
authorize: async binding => { events.push("authorize"); authorizeHook?.(); if (!authorized) throw new Error("revoked"); return { authorityProof: binding.authorityProof, authorityGeneration: binding.authorityGeneration, controlGeneration: binding.controlGeneration, expiresAt: binding.expiresAt } },
endpoints: async binding => { events.push("endpoints"); return { source: { endpointProof: endpointsCurrent ? binding.source.endpointProof : "f".repeat(64), canonicalNodeId: binding.source.canonicalNodeId }, destination: { ...binding.destination } } },
journal: {
reserve: async binding => { events.push("reserve"); lastBinding = binding; assert.equal(occupancy, "free", "a different attempt cannot reclaim busy nodes"); occupancy = "reserved"; return ack("reserve", binding) },
arm: async binding => { events.push("arm"); assert.equal(occupancy, "reserved"); occupancy = "armed"; if (armDelay) await armDelay; return ack("arm", binding) },
complete: async (binding, evidence, scope) => { events.push("complete"); completeHook?.(); assert.match(evidence.terminalProof, /^[a-f0-9]{64}$/); assert.equal(scope.signal, undefined, "cleanup does not inherit execute cancellation"); assert.equal(occupancy, "armed"); occupancy = "free"; return ack("complete", binding) },
releaseBeforeDispatch: async (binding, evidence, scope) => { events.push("release-before-dispatch"); assert.equal(evidence.proof, "private-no-send"); assert.equal(sends, 0); assert.equal(scope.signal, undefined); assert.equal(occupancy, evidence.from); occupancy = "free"; return ack("release-before-dispatch", binding) },
quarantine: async binding => { events.push("quarantine"); if (occupancy === "free") throw new Error("old attempt no longer owns occupancy"); occupancy = "unknown-busy"; return ack("quarantine", binding) },
},
send: async (_binding, body, scope) => { events.push("send"); sends++; assert.equal(occupancy, "armed"); assert.equal(body.password, input.nativeBody.password); assert.ok(scope.deadlineAt <= input.expiresAt); return sendBehavior() },
}
return { input, deps, events, faults, gate: () => createBandwidthAdmission(input, deps), options: () => ({ deadlineAt: Date.now() + 5000 }), sends: () => sends, occupancy: () => occupancy, binding: () => lastBinding!, revoke: () => { authorized = false }, staleEndpoints: () => { endpointsCurrent = false }, send: (fn: typeof sendBehavior) => { sendBehavior = fn }, delayArm: (promise: Promise<void>) => { armDelay = promise }, onAuthorize: (fn: () => void) => { authorizeHook = fn }, onComplete: (fn: () => void) => { completeHook = fn }, onTerminal: (fn: () => void) => { terminalHook = fn } }
}
test("pure construction performs no I/O; new acknowledged arm permits exactly one fixed send", async () => {
const f = fixture(), gate = f.gate(); assert.equal(f.events.length, 0)
const cap = await gate.reserveAndArm(f.options()); assert.equal(f.sends(), 0); assert.equal(f.occupancy(), "armed")
const result = await gate.dispatch(cap)
assert.equal(result.remoteCompletionConfirmed, true); assert.equal(f.occupancy(), "free"); assert.equal(f.sends(), 1)
assert.deepEqual(gate.status(), { phase: "released", remoteCompletionConfirmed: true })
await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); await assert.rejects(gate.reserveAndArm(f.options()), hasCode("REFUSED"))
assert.equal(f.events.filter(event => event === "arm").length, 1)
const index = f.events.indexOf("send"); assert.deepEqual(f.events.slice(index - 3, index + 1), ["authorize", "endpoints", "authorize", "send"])
})
test("missing, existing, mismatched or duplicate ACK never mints/adopts a dispatch capability", async () => {
for (const transition of ["reserve", "arm"] as const) for (const fault of ["missing", "existing", "wrong", "throw", ...(transition === "arm" ? ["duplicate"] : [])]) {
const f = fixture(); f.faults.set(transition, fault); const gate = f.gate()
await assert.rejects(gate.reserveAndArm(f.options()), hasCode("PERSISTENCE_UNKNOWN"))
assert.equal(f.sends(), 0); assert.equal(gate.status().phase, "unknown-busy"); assert.notEqual(f.occupancy(), "free")
await assert.rejects(gate.reserveAndArm(f.options()), hasCode("REFUSED"))
assert.equal(f.events.filter(event => event === transition).length, 1, "no repeated CAS after lost ACK")
}
})
test("capability cannot be copied, serialized, forged, transferred to another gate or reused", async () => {
const f = fixture(), gate = f.gate(), cap = await gate.reserveAndArm(f.options())
for (const fake of [{}, { ...cap }, JSON.parse(JSON.stringify(cap)), { attemptId: f.input.attemptId }]) await assert.rejects(gate.dispatch(fake as BandwidthDispatchCapability), hasCode("REFUSED"))
const foreign = fixture().gate(); await assert.rejects(foreign.dispatch(cap), hasCode("REFUSED"))
assert.equal(f.sends(), 0); await gate.cancel(cap)
await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); assert.equal(f.occupancy(), "free")
})
test("original body, destination credentials, request, identities and generation all enter keyed exact proofs", async () => {
const baseline = fixture(); await baseline.gate().reserveAndArm(baseline.options())
const initial = baseline.binding()
for (const mutate of [(i: BandwidthAdmissionInput) => { i.nativeBody.password += "changed" }, (i: BandwidthAdmissionInput) => { i.nativeBody["remote-tx-speed"] = "99" }, (i: BandwidthAdmissionInput) => { i.request.rate = 101 }, (i: BandwidthAdmissionInput) => { i.destination.canonicalNodeId = "physical:C" }, (i: BandwidthAdmissionInput) => { i.authorityGeneration = "actor-8" }]) {
const f = fixture(); f.input.expiresAt = baseline.input.expiresAt; mutate(f.input); await f.gate().reserveAndArm(f.options()); assert.notEqual(f.binding().bindingProof, initial.bindingProof)
assert.ok(!JSON.stringify(f.binding()).includes("destination-secret"))
}
})
test("full duration 2147483647 is preserved and no shorter command duration is introduced", async () => {
const f = fixture(); f.input.durationSec = 2147483647; f.input.nativeBody.duration = "2147483647s"
f.deps.send = async (binding, body) => { assert.equal(binding.durationSec, 2147483647); assert.equal(body.duration, "2147483647s"); return [{ status: "done testing" }] }
const gate = f.gate(); await gate.dispatch(await gate.reserveAndArm(f.options()))
})
test("pre-dispatch revoke, stale endpoint and abort release only proven-not-sent ownership", async () => {
for (const fault of ["revoke", "endpoint", "abort"]) {
const f = fixture(), controller = new AbortController(), gate = f.gate(), cap = await gate.reserveAndArm({ ...f.options(), signal: controller.signal })
if (fault === "revoke") f.revoke(); else if (fault === "endpoint") f.staleEndpoints(); else controller.abort()
await assert.rejects(gate.dispatch(cap)); assert.equal(f.sends(), 0); assert.equal(f.occupancy(), "free")
assert.equal(f.events.filter(event => event === "release-before-dispatch").length, 1)
await assert.rejects(gate.dispatch(cap), hasCode("REFUSED"))
}
})
test("reserved endpoint refusal releases without arm and failed no-send release never retries CAS", async () => {
const f = fixture(); f.staleEndpoints(); f.faults.set("release-before-dispatch", "throw")
const gate = f.gate(); await assert.rejects(gate.reserveAndArm(f.options()), hasCode("PERSISTENCE_UNKNOWN"))
assert.equal(f.events.includes("arm"), false); assert.equal(f.events.filter(event => event === "release-before-dispatch").length, 1); assert.equal(f.sends(), 0)
})
test("late arm ACK after cancellation cannot create capability or trigger release/send", async () => {
const f = fixture(), delayed = deferred<void>(), controller = new AbortController(); f.delayArm(delayed.promise)
const gate = f.gate(), pending = gate.reserveAndArm({ ...f.options(), signal: controller.signal })
await turn(); assert.equal(f.occupancy(), "armed"); const rejected = assert.rejects(pending); controller.abort(); await rejected
delayed.resolve(); await turn(); assert.equal(gate.status().phase, "unknown-busy"); assert.equal(f.sends(), 0); assert.equal(f.events.includes("release-before-dispatch"), false)
})
test("late endpoint continuation after proven no-dispatch release cannot send", async () => {
const f = fixture(), delayed = deferred<{ source: BandwidthAdmissionInput["source"]; destination: BandwidthAdmissionInput["destination"] }>(), controller = new AbortController()
const endpointReader = f.deps.endpoints
let reads = 0
f.deps.endpoints = async (binding, scope) => ++reads === 1 ? endpointReader(binding, scope) : delayed.promise
const gate = f.gate(), cap = await gate.reserveAndArm({ ...f.options(), signal: controller.signal }), pending = gate.dispatch(cap)
await turn(); assert.equal(reads, 2)
const rejected = assert.rejects(pending, hasCode("ABORTED")); controller.abort(); await rejected
assert.equal(f.occupancy(), "free")
delayed.resolve({ source: f.input.source, destination: f.input.destination }); await turn()
assert.equal(f.sends(), 0); assert.equal(gate.status().phase, "released")
})
test("fresh generation or original expiry drift refuses dispatch before native", async () => {
for (const field of ["authorityGeneration", "controlGeneration", "expiresAt"] as const) {
const f = fixture(), original = f.deps.authorize
let drift = false
f.deps.authorize = async (binding, scope) => { const grant = await original(binding, scope); return drift ? { ...grant, [field]: field === "expiresAt" ? grant.expiresAt + 1 : "changed" } : grant }
const gate = f.gate(), cap = await gate.reserveAndArm(f.options()); drift = true
await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); assert.equal(f.sends(), 0); assert.equal(f.occupancy(), "free")
}
})
test("concurrent arm and concurrent dispatch cannot duplicate CAS or native", async () => {
const f = fixture(), gate = f.gate(), first = gate.reserveAndArm(f.options())
await assert.rejects(gate.reserveAndArm(f.options()), hasCode("REFUSED")); const cap = await first
const dispatch = gate.dispatch(cap); await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); await dispatch
assert.equal(f.sends(), 1); assert.equal(f.events.filter(event => event === "arm").length, 1)
})
test("invoking transport makes throws, nonterminal, malformed and HTTP-like responses unknown-busy", async () => {
for (const raw of [[], [{ status: "testing" }], [{ status: "done testing" }, { status: "running" }], [{ status: "done testing", error: "bad" }], { status: 200 }, null]) {
const f = fixture(); f.send(async () => raw); const gate = f.gate(), cap = await gate.reserveAndArm(f.options())
let error: unknown; try { await gate.dispatch(cap) } catch (caught) { error = caught }
assert.ok(hasCode("REMOTE_UNKNOWN")(error)); assert.equal(f.occupancy(), "unknown-busy"); assert.equal(f.events.includes("complete"), false); assert.equal(f.events.includes("release-before-dispatch"), false)
assert.equal(Object.hasOwn(error as object, "raw"), false); assert.equal(Object.hasOwn(error as object, "response"), false); assert.equal(String(error).includes("done testing"), false)
assert.deepEqual(gate.partialResponse(error), raw); assert.equal(gate.partialResponse({ ...(error as object) }), undefined)
await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); assert.equal(f.sends(), 1)
}
const f = fixture(); f.send(() => { throw new Error("transport threw before known physical send") }); const gate = f.gate()
await assert.rejects(gate.dispatch(await gate.reserveAndArm(f.options())), hasCode("REMOTE_UNKNOWN")); assert.equal(f.occupancy(), "unknown-busy")
})
test("native response preserves more than 2000 rows, 256 opaque keys and input-depth limits", async () => {
const f = fixture()
let nested: unknown = "deep value"
for (let index = 0; index < 40; index++) nested = { next: nested }
const opaque = Object.fromEntries(Array.from({ length: 300 }, (_, index) => [`opaque_${index}`, `value_${index}`]))
const raw = [...Array.from({ length: 2200 }, (_, index) => ({ status: "testing", sample: index })), { ...opaque, nested, status: "done testing" }]
f.send(async () => raw)
const gate = f.gate(), result = await gate.dispatch(await gate.reserveAndArm(f.options()))
assert.deepEqual(result.raw, raw); assert.equal(result.raw.length, 2201)
assert.equal((result.raw[2200] as Record<string, unknown>).opaque_299, "value_299"); assert.equal(f.occupancy(), "free")
assert.notEqual(result.raw, raw); assert.ok(Object.isFrozen(result.raw[2200]))
})
test("native response accepts the exact existing 8 MiB JSON envelope and rejects excess without release", async () => {
const max = 8 * 1048576
const empty = [{ status: "done testing", opaque: "" }]
const overhead = Buffer.byteLength(JSON.stringify(empty))
for (const extra of [0, 1]) {
const f = fixture(), raw = [{ status: "done testing", opaque: "x".repeat(max - overhead + extra) }]
assert.equal(Buffer.byteLength(JSON.stringify(raw)), max + extra)
// A trusted proof provider may stream/canonicalize large evidence; this
// fixture does not require a second large recursive test-only serializer.
f.deps.keyedProof = async (purpose, value) => purpose === "bandwidth-terminal-v1" ? "e".repeat(64) : proof(purpose, value)
f.send(async () => raw)
const gate = f.gate(), cap = await gate.reserveAndArm(f.options())
if (extra === 0) { const result = await gate.dispatch(cap); assert.deepEqual(result.raw, raw); assert.equal(f.occupancy(), "free") }
else {
await assert.rejects(gate.dispatch(cap), hasCode("REMOTE_UNKNOWN")); assert.equal(f.occupancy(), "unknown-busy")
assert.equal(f.events.includes("complete"), false); assert.equal(f.events.includes("release-before-dispatch"), false)
}
}
})
test("cleanup has a separate hard 10-second maximum without narrowing native duration", async () => {
for (const budget of [0, 10001, 2147483647]) {
const f = fixture(); f.deps.cleanupBudgetMs = budget
assert.throws(f.gate, hasCode("INVALID_INPUT")); assert.equal(f.events.length, 0)
}
const f = fixture(); f.deps.cleanupBudgetMs = 10000
const original = f.deps.journal.complete
f.deps.journal.complete = async (binding, evidence, scope) => { const remaining = scope.deadlineAt - Date.now(); assert.ok(remaining > 0 && remaining <= 10000); return original(binding, evidence, scope) }
f.input.durationSec = 2147483647; f.input.nativeBody.duration = "2147483647s"
const gate = f.gate(); await gate.dispatch(await gate.reserveAndArm(f.options()))
})
test("abort and absolute timeout after send never claim remote stop or release on late done", async () => {
for (const fault of ["abort", "timeout"]) {
const f = fixture(), delayed = deferred<unknown>(), controller = new AbortController(); f.send(() => delayed.promise)
const gate = f.gate(), cap = await gate.reserveAndArm({ signal: controller.signal, deadlineAt: Date.now() + (fault === "timeout" ? 40 : 5000) })
const pending = gate.dispatch(cap), rejected = assert.rejects(pending, hasCode("REMOTE_UNKNOWN"))
await turn(); assert.equal(f.sends(), 1); if (fault === "abort") controller.abort(); await rejected
delayed.resolve([{ status: "done testing" }]); await turn()
assert.deepEqual(gate.status(), { phase: "unknown-busy", remoteCompletionConfirmed: false }); assert.equal(f.occupancy(), "unknown-busy"); assert.equal(f.events.includes("complete"), false); assert.equal(f.events.includes("release-before-dispatch"), false)
}
})
test("unknown persistence bookkeeping failure cannot free an armed attempt", async () => {
const f = fixture(); f.send(async () => [{ status: "testing" }]); f.faults.set("quarantine", "throw"); const gate = f.gate()
await assert.rejects(gate.dispatch(await gate.reserveAndArm(f.options())), hasCode("REMOTE_UNKNOWN")); assert.notEqual(f.occupancy(), "free")
})
test("strict terminal cleanup is independent of subsequent authority expiry/history, with no native replay after lost finalize ACK", async () => {
for (const lost of [false, true]) {
const f = fixture(), controller = new AbortController(); f.onTerminal(() => { controller.abort(); f.revoke() }); if (lost) f.faults.set("complete", "throw")
const gate = f.gate(), cap = await gate.reserveAndArm({ ...f.options(), signal: controller.signal })
if (lost) await assert.rejects(gate.dispatch(cap), hasCode("COMPLETION_PERSISTENCE_UNKNOWN"))
else assert.equal((await gate.dispatch(cap)).remoteCompletionConfirmed, true)
assert.equal(gate.status().remoteCompletionConfirmed, true); assert.equal(f.occupancy(), "free")
assert.equal(f.events.filter(event => event === "complete").length, 1); await assert.rejects(gate.dispatch(cap), hasCode("REFUSED")); assert.equal(f.sends(), 1)
// No speed-history callback participates in occupancy release.
assert.equal(f.events.some(event => event.includes("history")), false)
}
})
test("terminal completion can commit after the original execute deadline using only bounded cleanup", async () => {
const f = fixture(), completed = f.deps.journal.complete
f.deps.journal.complete = async (binding, evidence, scope) => { await delay(50); return completed(binding, evidence, scope) }
const gate = f.gate(), cap = await gate.reserveAndArm({ deadlineAt: Date.now() + 40 })
assert.equal((await gate.dispatch(cap)).remoteCompletionConfirmed, true)
assert.equal(f.occupancy(), "free")
})
test("captured generations/expiry and caller bindings cannot be changed through caller objects", async () => {
const f = fixture(), gate = f.gate(), options = f.options(), cap = await gate.reserveAndArm(options)
const oldExpiry = f.binding().expiresAt; options.deadlineAt += 999999; f.input.authorityGeneration = "new-actor"; f.input.expiresAt += 999999
assert.equal(f.binding().expiresAt, oldExpiry); assert.equal(f.binding().authorityGeneration, "actor-7")
await gate.dispatch(cap)
})
test("untrusted kind, equal physical identities, malformed body and inconsistent duration refuse before I/O", () => {
for (const mutate of [(input: BandwidthAdmissionInput) => { input.callerKind = "arbitrary" as "rest-speed" }, (input: BandwidthAdmissionInput) => { input.destination = { ...input.source } }, (input: BandwidthAdmissionInput) => { input.durationSec = 60 }, (input: BandwidthAdmissionInput) => { input.nativeBody.callback = (() => {}) as unknown as string }, (input: BandwidthAdmissionInput) => { Object.assign(input, { unboundExtra: "reject" }) }]) {
const f = fixture(); mutate(f.input); assert.throws(f.gate, hasCode("INVALID_INPUT")); assert.deepEqual(f.events, [])
}
})