280 lines
20 KiB
TypeScript
280 lines
20 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { createHash, createHmac, generateKeyPairSync, randomUUID, sign } from "node:crypto"
|
|
import { mkdtemp, rm, writeFile, unlink } from "node:fs/promises"
|
|
import { join } from "node:path"
|
|
import { tmpdir } from "node:os"
|
|
import { performance } from "node:perf_hooks"
|
|
import { after, test } from "node:test"
|
|
import { canonicalBandwidthNodeValue } from "./bandwidth-node-bootstrap.js"
|
|
import { bandwidthOperatorEnrollmentGrantSchema } from "./bandwidth-operator-enrollment.js"
|
|
import { createBandwidthOperatorOriginalAuthorityReader, createBandwidthOperatorExternalFencer, createBandwidthOperatorWriterWindows,
|
|
BandwidthOperatorWriterFenceUnknown, type BandwidthOperatorRegistryBorrow, type BandwidthOperatorWriterWindowDependencies } from "./bandwidth-operator-writer-window.js"
|
|
|
|
const later = (ms: number) => new Promise<void>(done => setTimeout(done, ms))
|
|
const scope = (ms = 4000) => ({ deadlineAt: Date.now() + ms })
|
|
const cleanup: Array<() => Promise<void>> = []
|
|
after(async () => { for (const close of cleanup) await close() })
|
|
type Result = { canonicalNodeId: string; incarnation: string }
|
|
type Dependencies = BandwidthOperatorWriterWindowDependencies<Result>
|
|
type Mode = "ok" | "query-error" | "commit-lost" | "commit-rows" | "commit-held" | "withdraw-after-commit" | "missing-release" | "extra-after-release"
|
|
|
|
/** Local synthetic process/driver/external-enforcer model. NO real PostgreSQL,
|
|
* network, external service manager or production stop-proof is used. */
|
|
async function fixture(options: { mode?: Mode; ttl?: number; lifecycle?: "register" | "quarantine"; rawFence?: boolean } = {}) {
|
|
const folder = await mkdtemp(join(tmpdir(), "writer-window-candidate-")), keys = generateKeyPairSync("ed25519")
|
|
const trustKeyFile = join(folder, "operator.pub"), grantFile = join(folder, "grant.json")
|
|
const root = { installationId: randomUUID(), issuerKeySha256: "a".repeat(64), quiescenceProof: "b".repeat(64) }
|
|
await writeFile(trustKeyFile, keys.publicKey.export({ type: "spki", format: "pem" }))
|
|
let grant = bandwidthOperatorEnrollmentGrantSchema.parse({ format: "mmapp-bandwidth-operator-enrollment-v1", ...root, grantId: randomUUID(), operatorId: "SYNTHETIC-OPERATOR",
|
|
executorInventorySha256: "c".repeat(64), executorsStoppedAt: Date.now() - 2000, issuedAt: Date.now() - 1000, expiresAt: Date.now() + (options.ttl ?? 4000),
|
|
peerRef: { kind: "external", enrollmentId: randomUUID() }, sourceProof: "d".repeat(64), authorityGeneration: "SYNTHETIC-ORIGINAL", routingEndpointEpoch: randomUUID(),
|
|
claims: [{ kind: "system-id", value: "SYNTHETIC-CANONICAL-CLAIM" }] })
|
|
async function publish(value = grant) { grant = value; const payload = Buffer.from(canonicalBandwidthNodeValue(value)); await writeFile(grantFile,
|
|
JSON.stringify({ format: "mmapp-bandwidth-operator-enrollment-envelope-v1", payload: payload.toString("base64url"), signature: sign(null, payload, keys.privateKey).toString("base64url") })) }
|
|
await publish()
|
|
const originals = createBandwidthOperatorOriginalAuthorityReader({ ...root, trustKeyFile, grantFile,
|
|
trustKeySha256: createHash("sha256").update(keys.publicKey.export({ type: "spki", format: "der" })).digest("hex") })
|
|
const events: string[] = [], queries: string[] = []
|
|
const occupancy = Object.freeze({ canonicalNodeId: randomUUID(), state: "unknown", attempt: randomUUID() })
|
|
const aliases: Array<{ id: string; canonicalNodeId: string; status: "active" | "quarantined" }> = []
|
|
let mode: Mode = options.mode ?? "ok", sharedWriters = 0, held = false, physicalClosed = false, commits = 0, opens = 0, destroys = 0,
|
|
externalAcquires = 0, externalCloses = 0, externalAssertions = 0, finalReads = 0, backend = 73, externalAllowed = true,
|
|
borrowedReleased = false, dml = 0
|
|
let externalDelay: (() => Promise<void>) | undefined, sourceDelay: (() => Promise<void>) | undefined,
|
|
hook: ((event: string) => void | Promise<void>) | undefined
|
|
const external = createBandwidthOperatorExternalFencer({ async acquire(original) {
|
|
externalAcquires++; events.push("external.acquire"); assert.equal(original.grant.executorInventorySha256, grant.executorInventorySha256)
|
|
await externalDelay?.(); if (!externalAllowed) throw Error("Synthetic old executor stop/restart prohibition is NOT held")
|
|
let stopped = false
|
|
return { async assertCurrent(current) {
|
|
externalAssertions++; events.push("external.assert"); assert(!stopped); assert.equal(current.grant.grantId, original.grant.grantId)
|
|
if (borrowedReleased) { finalReads++; assert.equal(physicalClosed, false, "final authority check must precede physical destroy") }
|
|
await hook?.("external.assert"); if (!externalAllowed) throw Error("External restart prohibition revoked")
|
|
}, close() { if (!stopped) { stopped = true; externalCloses++; events.push("external.close") } } }
|
|
} })
|
|
const source: Dependencies["source"] = { async open(_scope, guard) {
|
|
opens++; events.push("source.open"); await guard(); await sourceDelay?.()
|
|
physicalClosed = false
|
|
const close = () => { if (physicalClosed) return; physicalClosed = true; held = false; destroys++; events.push("source.destroy") }
|
|
return { close, async binding() { await guard(); return root }, async keyedProof(purpose, value) {
|
|
await guard(); await hook?.("keyedProof"); return createHmac("sha256", "SYNTHETIC-CANDIDATE-ISSUER").update(purpose + ":" + canonicalBandwidthNodeValue(value)).digest("hex")
|
|
}, connection: { release() { throw Error("Physical connection.release is owner-only; must destroy") }, async query(text) {
|
|
assert(!physicalClosed); await guard(); queries.push(text); events.push("query:" + text); await hook?.(text)
|
|
if (text.includes("pg_try_advisory_lock(8723116)")) { const acquired = sharedWriters === 0 && !held; if (acquired) held = true; return { rows: [{ acquired }] } }
|
|
if (text.includes("pg_locks")) return { rows: [{ backend_pid: backend, held }] }
|
|
if (text.includes("pg_backend_pid()")) return { rows: [{ backend_pid: backend }] }
|
|
if (text === "SELECT pg_catalog.pg_advisory_xact_lock(8723116)") { assert(held, "same backend reenters the SAME session key without deadlocking"); return { rows: [] } }
|
|
if (text === "BEGIN") return { rows: [], command: "BEGIN" }
|
|
if (text === "COMMIT") { commits++; if (mode === "commit-held") await new Promise(() => {}); if (mode === "commit-lost") throw Error("Synthetic lost actual ACK"); return { rows: [], command: mode === "commit-rows" ? undefined : "COMMIT" } }
|
|
if (text === "INSERT SYNTHETIC ALIAS") {
|
|
if (mode === "query-error") throw Error("Synthetic physical driver failed")
|
|
dml++; aliases.push({ id: randomUUID(), canonicalNodeId: occupancy.canonicalNodeId, status: options.lifecycle === "quarantine" ? "quarantined" : "active" }); return { rows: [] }
|
|
}
|
|
throw Error("Unexpected inert SQL: " + text)
|
|
} } }
|
|
} }
|
|
if (options.rawFence) source.openWithOwnershipFence = async (requested, guard, verifyOwnership) => {
|
|
const session = await source.open(requested, guard)
|
|
const dispatch = async <T>(work: () => Promise<T>) => {
|
|
const boundary = async () => {
|
|
events.push("raw.metadata")
|
|
await verifyOwnership({
|
|
async query(text) {
|
|
assert(text.includes("pg_locks")); events.push("raw.ownership")
|
|
return { rows: [{ backend_pid: backend, held }] }
|
|
},
|
|
async verifyNodeBootstrap() { throw Error("unused") },
|
|
async sourceIdentity() { throw Error("unused") },
|
|
})
|
|
events.push("raw.afterfiles"); await guard(); events.push("raw.original-last")
|
|
}
|
|
await boundary(); const result = await work(); await boundary(); return result
|
|
}
|
|
return { ...session,
|
|
binding: () => dispatch(() => session.binding()),
|
|
keyedProof: (purpose, value) => dispatch(() => session.keyedProof(purpose, value)),
|
|
keyedProofs: inputs => dispatch(async () => {
|
|
events.push("raw.batch-select1")
|
|
return inputs.map(input => createHmac("sha256", "SYNTHETIC-CANDIDATE-ISSUER").update(input.purpose + ":" + canonicalBandwidthNodeValue(input.value)).digest("hex"))
|
|
}),
|
|
connection: { release: session.connection.release,
|
|
query: (text, values, queryOptions) => dispatch(() => session.connection.query(text, values, queryOptions)),
|
|
},
|
|
}
|
|
}
|
|
const registry: Dependencies["registry"] = { async register(_original, borrowed) {
|
|
let result: Result | undefined
|
|
try {
|
|
await borrowed.connection.query("BEGIN")
|
|
await borrowed.connection.query("SELECT pg_catalog.pg_advisory_xact_lock(8723116)")
|
|
if (options.rawFence) {
|
|
assert(borrowed.keyedProofs)
|
|
const proofs = await borrowed.keyedProofs([{ purpose: "external-alias", value: { ...root, canonicalNode: occupancy.canonicalNodeId } }])
|
|
assert.equal(proofs.length, 1)
|
|
} else await borrowed.keyedProof("external-alias", { ...root, canonicalNode: occupancy.canonicalNodeId })
|
|
await borrowed.connection.query("INSERT SYNTHETIC ALIAS")
|
|
await borrowed.connection.query("COMMIT")
|
|
if (mode === "withdraw-after-commit") await unlink(grantFile)
|
|
result = { canonicalNodeId: occupancy.canonicalNodeId, incarnation: randomUUID() }
|
|
} finally {
|
|
if (mode !== "missing-release") { borrowed.connection.release(true); borrowedReleased = true; events.push("borrow.logical-release") }
|
|
}
|
|
if (mode === "extra-after-release") await borrowed.connection.query("INSERT SYNTHETIC ALIAS")
|
|
assert(result); return result
|
|
} }
|
|
const dependencies = { originals, external, source, registry }, windows = createBandwidthOperatorWriterWindows(dependencies)
|
|
cleanup.push(async () => { windows.close(); originals.close(); await rm(folder, { recursive: true, force: true }) })
|
|
const authority = await originals.acquire(scope(options.ttl ?? 4000))
|
|
return { windows, originals, dependencies, authority, events, queries, occupancy, aliases, publish,
|
|
grant: () => grant, snapshot: () => ({ opens, destroys, commits, held, physicalClosed, dml, externalAcquires, externalCloses, externalAssertions, finalReads }),
|
|
setShared: (value: number) => { sharedWriters = value }, revokeExternal: () => { externalAllowed = false }, loseSession: () => { held = false }, changeBackend: () => { backend++ },
|
|
sourceDelay: (value: typeof sourceDelay) => { sourceDelay = value }, externalDelay: (value: typeof externalDelay) => { externalDelay = value }, hook: (value: typeof hook) => { hook = value }, setMode: (value: Mode) => { mode = value } }
|
|
}
|
|
|
|
test("signed original BEFORE external fence; one physical source/session lock; logical release preserves final read", async () => {
|
|
const f = await fixture(), before = f.occupancy, window = await f.windows.open(f.authority, scope())
|
|
assert.equal(f.snapshot().externalAcquires, 1); assert.equal(f.snapshot().opens, 1); assert(f.snapshot().held)
|
|
await f.windows.verify(window, f.authority, scope(3000))
|
|
const receipt = await f.windows.register(window)
|
|
assert.equal(receipt.canonicalNodeId, before.canonicalNodeId); assert.deepEqual(f.occupancy, before); assert.equal(f.snapshot().finalReads, 1)
|
|
assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1); assert.equal(f.snapshot().held, false)
|
|
assert(f.events.indexOf("borrow.logical-release") < f.events.indexOf("source.destroy"))
|
|
assert.equal(f.queries.filter(sql => sql.includes("pg_try_advisory_lock")).length, 1)
|
|
assert(f.queries.indexOf("BEGIN") > f.queries.findIndex(sql => sql.includes("pg_try_advisory_lock")), "no outer BEGIN competes with factory BEGIN")
|
|
assert(!f.queries.some(sql => /occupancy|UNLOCK|DELETE|TRUNCATE/.test(sql)))
|
|
await assert.rejects(f.windows.register(window)); await assert.rejects(f.windows.verify(window, f.authority, scope(100)))
|
|
})
|
|
|
|
test("raw source ownership fence checks exact held backend before final files without outer metadata SQL", async () => {
|
|
const healthy = await fixture({ rawFence: true })
|
|
const window = await healthy.windows.open(healthy.authority, scope())
|
|
assert(!healthy.events.includes("raw.ownership"), "opening cannot require the not-yet-acquired lock")
|
|
await healthy.windows.register(window)
|
|
assert(healthy.events.includes("raw.batch-select1"), "batch MAC path must use its fresh raw-fenced dispatch")
|
|
assert(!healthy.queries.some(query => query.includes("pg_locks")), "ownership checks must bypass recursive source metadata")
|
|
const checks = healthy.events.flatMap((event, index) => event === "raw.ownership" ? [index] : [])
|
|
assert(checks.length >= 8)
|
|
for (const index of checks) {
|
|
assert.equal(healthy.events[index - 1], "raw.metadata")
|
|
assert.equal(healthy.events[index + 1], "raw.afterfiles")
|
|
// The original guard runs after the final source file checks. External Q
|
|
// appends its own events before the guard acknowledges this boundary.
|
|
assert(healthy.events.slice(index + 2).includes("raw.original-last"))
|
|
}
|
|
assert.equal(healthy.snapshot().destroys, 1)
|
|
for (const failure of ["held", "pid"] as const) {
|
|
const fixtureValue = await fixture({ rawFence: true })
|
|
const heldWindow = await fixtureValue.windows.open(fixtureValue.authority, scope())
|
|
fixtureValue.hook(event => {
|
|
if (event === "INSERT SYNTHETIC ALIAS") {
|
|
if (failure === "held") fixtureValue.loseSession(); else fixtureValue.changeBackend()
|
|
}
|
|
})
|
|
await assert.rejects(fixtureValue.windows.register(heldWindow))
|
|
assert.equal(fixtureValue.snapshot().destroys, 1)
|
|
assert.equal(fixtureValue.snapshot().commits, 0, "post-dispatch ownership loss refuses before COMMIT")
|
|
}
|
|
})
|
|
|
|
test("constructor and private capability copies refuse before source or external stop acquisition", async () => {
|
|
const f = await fixture()
|
|
assert.throws(() => createBandwidthOperatorWriterWindows({ ...f.dependencies, originals: { ...f.originals } }))
|
|
assert.throws(() => createBandwidthOperatorWriterWindows({ ...f.dependencies, external: { ...f.dependencies.external } }))
|
|
await assert.rejects(f.windows.open({ ...f.authority }, scope())); assert.equal(f.snapshot().externalAcquires, 0); assert.equal(f.snapshot().opens, 0)
|
|
const window = await f.windows.open(f.authority, scope()); await assert.rejects(f.windows.register({ ...window })); f.windows.close()
|
|
})
|
|
|
|
test("shared native handoff holder blocks exclusive window; SQL absence alone does not issue external stop proof", async () => {
|
|
const f = await fixture(); f.setShared(1)
|
|
await assert.rejects(f.windows.open(f.authority, scope())); assert.equal(f.snapshot().held, false); assert.equal(f.snapshot().destroys, 1)
|
|
const missing = await fixture(); missing.revokeExternal()
|
|
await assert.rejects(missing.windows.open(missing.authority, scope())); assert.equal(missing.snapshot().opens, 0); assert.equal(missing.queries.length, 0)
|
|
})
|
|
|
|
test("safe alias register and quarantine preserve an existing canonical UNKNOWN; no blanket occupancy prohibition", async () => {
|
|
for (const lifecycle of ["register", "quarantine"] as const) {
|
|
const f = await fixture({ lifecycle }), before = structuredClone(f.occupancy)
|
|
const receipt = await f.windows.register(await f.windows.open(f.authority, scope()))
|
|
assert.equal(receipt.canonicalNodeId, before.canonicalNodeId); assert.deepEqual(f.occupancy, before)
|
|
assert.equal(f.aliases[0].status, lifecycle === "quarantine" ? "quarantined" : "active")
|
|
}
|
|
})
|
|
|
|
test("physical driver failure, held-lock loss, backend drift and external revoke destroy immediately", async () => {
|
|
for (const fault of ["query-error", "lock", "backend", "external"] as const) {
|
|
const f = await fixture({ mode: fault === "query-error" ? "query-error" : "ok" }), window = await f.windows.open(f.authority, scope())
|
|
if (fault === "lock") f.hook(stage => { if (stage === "BEGIN") f.loseSession() })
|
|
if (fault === "backend") f.hook(stage => { if (stage === "keyedProof") f.changeBackend() })
|
|
if (fault === "external") f.hook(stage => { if (stage === "keyedProof") f.revokeExternal() })
|
|
await assert.rejects(f.windows.register(window)); assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1); assert.equal(f.snapshot().commits, 0)
|
|
}
|
|
})
|
|
|
|
test("lost/rows-only COMMIT ACK, final grant withdrawal, missing logical release and illegal post-release SQL are UNKNOWN", async () => {
|
|
for (const mode of ["commit-lost", "commit-rows", "withdraw-after-commit", "missing-release", "extra-after-release"] as const) {
|
|
const f = await fixture({ mode }), window = await f.windows.open(f.authority, scope()), before = f.occupancy
|
|
await assert.rejects(f.windows.register(window), error => error instanceof BandwidthOperatorWriterFenceUnknown && error.automaticReplay === false)
|
|
assert.equal(f.snapshot().commits, 1); assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1)
|
|
assert.deepEqual(f.occupancy, before); assert.equal(f.snapshot().dml, 1); await assert.rejects(f.windows.register(window))
|
|
}
|
|
})
|
|
|
|
test("cancel during held COMMIT is UNKNOWN and physically destroys before the late driver completes", async () => {
|
|
const f = await fixture({ mode: "commit-held" }), abort = new AbortController()
|
|
const window = await f.windows.open(f.authority, { ...scope(), signal: abort.signal })
|
|
f.hook(stage => { if (stage === "COMMIT") abort.abort() })
|
|
await assert.rejects(f.windows.register(window), BandwidthOperatorWriterFenceUnknown)
|
|
assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1); assert.equal(f.snapshot().held, false)
|
|
})
|
|
|
|
test("cancelled source/external acquisitions destroy late resources and cannot publish a held window", async () => {
|
|
for (const stage of ["source", "external"] as const) {
|
|
const f = await fixture(), abort = new AbortController(); let resume!: () => void
|
|
const held = new Promise<void>(done => { resume = done })
|
|
if (stage === "source") f.sourceDelay(() => { abort.abort(); return held })
|
|
else f.externalDelay(() => { abort.abort(); return held })
|
|
await assert.rejects(f.windows.open(f.authority, { ...scope(), signal: abort.signal }))
|
|
resume(); await later(20)
|
|
assert.equal(f.snapshot().externalCloses, 1)
|
|
assert.equal(f.snapshot().destroys, stage === "source" ? 1 : 0)
|
|
assert(!f.queries.includes("BEGIN"))
|
|
}
|
|
})
|
|
|
|
test("original signed grant/inventory cannot be renewed or replaced between authority and quiescence", async () => {
|
|
const f = await fixture(), before = f.snapshot()
|
|
await f.publish({ ...f.grant(), executorInventorySha256: "e".repeat(64), expiresAt: f.grant().expiresAt + 1000 })
|
|
await assert.rejects(f.windows.open(f.authority, scope()))
|
|
assert.equal(f.snapshot().externalAcquires, before.externalAcquires); assert.equal(f.snapshot().opens, before.opens)
|
|
})
|
|
|
|
test("the original signed-authority cancellation survives later scopes and retires an already held owner", async () => {
|
|
const f = await fixture(), abort = new AbortController()
|
|
const authority = await f.originals.acquire({ ...scope(), signal: abort.signal })
|
|
const window = await f.windows.open(authority, scope())
|
|
abort.abort()
|
|
assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1)
|
|
await assert.rejects(f.windows.register(window)); await assert.rejects(f.originals.inspect(authority, scope()))
|
|
})
|
|
|
|
test("a shorter verifier scope and its cancellation bound a stalled fence without renewing the held window", async () => {
|
|
for (const kind of ["deadline", "abort"] as const) {
|
|
const f = await fixture(), window = await f.windows.open(f.authority, scope()), abort = new AbortController()
|
|
let resume!: () => void; const stalled = new Promise<void>(done => { resume = done })
|
|
f.hook(stage => { if (stage === "external.assert") { if (kind === "abort") abort.abort(); return stalled } })
|
|
try { await assert.rejects(f.windows.verify(window, f.authority, { ...scope(kind === "deadline" ? 50 : 3000), signal: abort.signal })) }
|
|
finally { resume() }
|
|
assert.equal(f.snapshot().destroys, 1); assert.equal(f.snapshot().externalCloses, 1); assert(!f.queries.includes("BEGIN"))
|
|
}
|
|
})
|
|
|
|
test("backward wall jump while external fence is stalled preserves original monotonic grant expiry and no source checkout", async () => {
|
|
const f = await fixture({ ttl: 250 }), realNow = Date.now
|
|
let resume!: () => void; const held = new Promise<void>(done => { resume = done })
|
|
f.externalDelay(() => { Date.now = () => realNow() - 500; return held })
|
|
const began = performance.now()
|
|
try { await assert.rejects(f.windows.open(f.authority, scope())); assert(performance.now() - began < 750); assert.equal(f.snapshot().opens, 0) }
|
|
finally { Date.now = realNow; resume() }
|
|
await later(20); assert.equal(f.snapshot().externalCloses, 1)
|
|
})
|