Files
MikrotikManager/backend/src/services/server-probe-storage-connection.test.ts
T

385 lines
32 KiB
TypeScript

import assert from "node:assert/strict"
import { test } from "node:test"
import { capturePollError, carryPollError, pollErrorLog } from "./server-poll-error-provenance.js"
import { EventEmitter } from "node:events"
import { DatabaseError, type QueryConfig } from "pg"
import { createProbeStorageConnectionConnector, connectProbeStorage, type ProbeStoragePgClient } from "./server-probe-storage-connection.js"
import { ProbeStorageRefused, ProbeStorageUnknown } from "./server-probe-journal-maintenance.js"
import { principalSourceSql, principalSourceStatementName } from "../modules/mcp/principal-security-sql.js"
type Result = Awaited<ReturnType<ProbeStoragePgClient["query"]>>
const result = (command = "SELECT"): Result => ({ rows: [], command })
function deferred<T>() { let resolve!: (value: T) => void, reject!: (error: unknown) => void; const promise = new Promise<T>((yes, no) => { resolve = yes; reject = no }); return { promise, resolve, reject } }
const tick = () => new Promise<void>(resolve => setImmediate(resolve))
function fixture(handler: (config: QueryConfig & { query_timeout: number }) => Promise<Result> = async config => result(config.text.trim() === "COMMIT" ? "COMMIT" : "SELECT")) {
const queries: Array<QueryConfig & { query_timeout: number }> = [], releases: boolean[] = [], listeners = new Set<(error: Error) => void>()
let active = 0, maximumActive = 0, checkouts = 0
const client: ProbeStoragePgClient = {
async query(config) { queries.push(config); active++; maximumActive = Math.max(maximumActive, active); try { return await handler(config) } finally { active-- } },
release(unsafe = false) { releases.push(unsafe) },
on(_event, listener) { listeners.add(listener) }, removeListener(_event, listener) { listeners.delete(listener) },
}
const connect = createProbeStorageConnectionConnector(async () => { checkouts++; return client })
return { client, connect, queries, releases, checkouts: () => checkouts, maximumActive: () => maximumActive, emitError: () => { for (const listener of listeners) listener(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC")) } }
}
const refused = (error: unknown) => error instanceof ProbeStorageRefused && !error.message.includes("NEVER_")
const unknown = (error: unknown) => error instanceof ProbeStorageUnknown && !error.message.includes("NEVER_")
test("only exact fixed source is named; fresh arguments/timeouts survive and near matches stay unnamed", async () => {
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 30000 })
try {
for (const values of [["credential-a", "connection-a", true], ["credential-b", "connection-b", false]]) {
await lease.query(principalSourceSql, values, { timeoutMs: 543 })
const sent = f.queries.at(-1)!
assert.equal(sent.name, principalSourceStatementName); assert.equal(sent.text, principalSourceSql)
assert.equal(sent.values, values); assert(sent.query_timeout > 0 && sent.query_timeout <= 543)
}
assert(principalSourceStatementName.length <= 63)
for (const text of [principalSourceSql + " ", "/* mcp-principal:coherent-source-v1 */ SELECT 1", "SELECT 1"]) {
await lease.query(text); assert.equal(Object.hasOwn(f.queries.at(-1)!, "name"), false)
}
} finally { lease.release() }
})
test("fixed-source schema failure destroys once without unnamed fallback or retry", async () => {
const f = fixture(async () => { throw Object.assign(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC"), { code: "0A000" }) })
const lease = await f.connect({ deadlineAt: Date.now() + 30000 })
await assert.rejects(lease.query(principalSourceSql, ["a", "b", true]), unknown)
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true]); assert.equal(lease.lifetimeSignal?.aborted, true)
await assert.rejects(lease.query(principalSourceSql, ["a", "b", true]), refused)
assert.equal(f.queries.length, 1); lease.release(); assert.deepEqual(f.releases, [true])
})
test("connector import/construction is inert; invalid/pre-aborted checkout performs no work", async () => {
const f = fixture(), aborted = new AbortController(); aborted.abort(Error("NEVER_PRIVATE_ABORT_REASON"))
assert.equal(typeof connectProbeStorage, "function"); assert.equal(f.checkouts(), 0)
await assert.rejects(f.connect({ deadlineAt: Date.now() + 1000, signal: aborted.signal }), refused)
for (const deadlineAt of [0, NaN, Infinity, Date.now() - 1, Date.now() + 0.5]) await assert.rejects(f.connect({ deadlineAt }), refused)
assert.equal(f.checkouts(), 0); assert.equal(f.queries.length, 0); assert.deepEqual(f.releases, [])
})
for (const mode of ["abort", "deadline"] as const) test("bounded checkout " + mode + " destroys an eventual client once", async () => {
const f = fixture(), pendingClient = deferred<ProbeStoragePgClient>(), controller = new AbortController()
let checkouts = 0
const connect = createProbeStorageConnectionConnector(() => { checkouts++; return pendingClient.promise })
const started = Date.now(), waiting = connect({ deadlineAt: Date.now() + (mode === "deadline" ? 25 : 1000), signal: controller.signal })
if (mode === "abort") controller.abort(Error("NEVER_PRIVATE_ABORT_REASON"))
await assert.rejects(waiting, refused)
assert.equal(checkouts, 1); assert.equal(Date.now() - started < 1000, true); assert.deepEqual(f.releases, [])
pendingClient.resolve(f.client); await tick()
assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 0)
})
test("late checkout failure is consumed after cancellation and never retried", async () => {
const pendingClient = deferred<ProbeStoragePgClient>(), controller = new AbortController()
let checkouts = 0, unhandled = 0
const listener = () => { unhandled++ }
process.on("unhandledRejection", listener)
try {
const connect = createProbeStorageConnectionConnector(() => { checkouts++; return pendingClient.promise })
const waiting = connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
controller.abort(); await assert.rejects(waiting, refused)
pendingClient.reject(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC")); await tick(); await tick()
assert.equal(checkouts, 1); assert.equal(unhandled, 0)
} finally { process.removeListener("unhandledRejection", listener) }
})
test("checkout driver rejection and synchronous throw are sanitized", async () => {
for (const checkout of [async () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }, () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }]) {
await assert.rejects(createProbeStorageConnectionConnector(checkout)({ deadlineAt: Date.now() + 1000 }), refused)
}
})
test("healthy lease carries values and pg query_timeout; release is once and closed is unusable", async () => {
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 30000 })
await lease.query("SELECT $1", [7], { timeoutMs: 321 })
assert.equal(f.queries.length, 1); assert.deepEqual(f.queries[0].values, [7])
assert.equal(f.queries[0].query_timeout > 0 && f.queries[0].query_timeout <= 321, true)
await lease.query("SELECT 1")
assert.equal(f.queries[1].query_timeout > 0 && f.queries[1].query_timeout <= 10000, true)
lease.release(); lease.release(true)
assert.deepEqual(f.releases, [false])
await assert.rejects(lease.query("SELECT 2"), refused); assert.equal(f.queries.length, 2)
})
test("actual connector publishes one immutable native lifetime signal; every close revokes BEFORE exact-once physical release",async()=>{
for(const mode of ["healthy","destroy","error","abort","deadline"]){
const f=fixture(),input=new AbortController(),lease=await f.connect({deadlineAt:Date.now()+(mode==="deadline"?30:1000),signal:input.signal})
const signal=lease.lifetimeSignal;assert(signal instanceof AbortSignal);assert.equal(signal.aborted,false)
const descriptor=Object.getOwnPropertyDescriptor(lease,"lifetimeSignal")!
assert.equal(descriptor.value,signal);assert.equal(descriptor.get,undefined);assert.equal(descriptor.writable,false);assert.equal(descriptor.configurable,false)
assert.throws(()=>Object.defineProperty(lease,"lifetimeSignal",{value:new AbortController().signal}))
const order:string[]=[],rawRelease=f.client.release.bind(f.client)
signal.addEventListener("abort",()=>order.push("revoked"),{once:true})
f.client.release=(unsafe)=>{assert.equal(signal.aborted,true);order.push("release");rawRelease(unsafe)}
if(mode==="healthy")lease.release()
if(mode==="destroy")lease.release(true)
if(mode==="error")f.emitError()
if(mode==="abort")input.abort()
if(mode==="deadline")await new Promise<void>(done=>setTimeout(done,45))
assert.equal(signal.aborted,true);assert.deepEqual(order,["revoked","release"])
lease.release(true);f.emitError();assert.deepEqual(f.releases,[mode!=="healthy"])
await assert.rejects(lease.query("SELECT forbidden_after_close"),refused);assert.equal(f.queries.length,0)
}
})
test("idle owner loss while external async work is pending synchronously revokes the retained lease",async()=>{
const f=fixture(),lease=await f.connect({deadlineAt:Date.now()+1000}),pending=deferred<void>()
const signal=lease.lifetimeSignal!;let consumed=false
const afterExternal=(async()=>{await pending.promise;if(signal.aborted)return;consumed=true})()
f.emitError();assert.equal(signal.aborted,true);pending.resolve();await afterExternal
assert.equal(consumed,false);assert.deepEqual(f.releases,[true]);assert.equal(f.checkouts(),1);assert.equal(f.queries.length,0)
})
test("late abandoned checkout retains socket error absorber, never publishes facade or replays checkout",async()=>{
const waiting=deferred<ProbeStoragePgClient>(),controller=new AbortController(),socket=new EventEmitter()
let checkouts=0,releases=0,published=false
const client:ProbeStoragePgClient={on:socket.on.bind(socket),removeListener:socket.removeListener.bind(socket),async query(){assert.fail("late client cannot query")},release(unsafe){assert.equal(unsafe,true);releases++;queueMicrotask(()=>socket.emit("error",Error("NEVER_PRIVATE_LATE_SOCKET")))}}
const connect=createProbeStorageConnectionConnector(()=>{checkouts++;return waiting.promise})
const result=connect({deadlineAt:Date.now()+1000,signal:controller.signal}).then(()=>{published=true})
controller.abort();await assert.rejects(result,refused);waiting.resolve(client);await tick();await tick()
assert.equal(published,false);assert.equal(checkouts,1);assert.equal(releases,1);assert(socket.listenerCount("error")>0)
socket.emit("error",Error("NEVER_PRIVATE_REPEATED_SOCKET"));assert.equal(releases,1)
})
test("genuine signed core/connectRegistry/node strong facades preserve physical lifetime revocation (synthetic PG only)",async()=>{
const fs=await import("node:fs/promises"),path=await import("node:path"),os=await import("node:os"),crypto=await import("node:crypto"),{Pool}=await import("pg")
const core=await import("./bandwidth-control-source.js"),node=await import("./bandwidth-node-source.js"),bootstrap=await import("./bandwidth-node-bootstrap.js")
const controlFixture=await import("../test/fixtures/bandwidth-control-identity.js"),registry=(await import("../test/fixtures/bandwidth-peer-identity.js")).fixture()
const directory=await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()),"mm-probe-lifetime-")),keys=crypto.generateKeyPairSync("ed25519"),eventKey=crypto.randomBytes(32),issuerKey=crypto.randomBytes(32)
const sha=(value:string|Buffer)=>crypto.createHash("sha256").update(value).digest("hex"),signed=(format:string,text:string)=>{const bytes=Buffer.from(text);return {format,payload:bytes.toString("base64url"),signature:crypto.sign(null,bytes,keys.privateKey).toString("base64url")}}
const config={databaseUrl:"postgresql://bandwidth_runtime:[email protected]:1/owned_database",applicationDatabaseUrl:"postgresql://restricted_application:[email protected]:1/application_database",trustKeyFile:path.join(directory,"trust.pem"),eventKeyFile:path.join(directory,"event.key"),attestationFile:path.join(directory,"source.json"),trustKeySha256:sha(keys.publicKey.export({type:"spki",format:"der"}))}
const record={format:"mmapp-bandwidth-control-source-v1" as const,installationId:registry.expected.installationId,controlGeneration:crypto.randomUUID(),databaseName:"owned_database",databaseOid:123,runtimeRole:"bandwidth_runtime",ownerRole:"fixed_owner",endpointSha256:core.bandwidthControlEndpointSha256(config.databaseUrl),schemaSourceSha256:core.bandwidthControlSchemaSourceSha256(),catalogSha256:controlFixture.expected.catalogSha256,eventKeySha256:sha(eventKey),attestationRef:"Synthetic signed lifecycle propagation, not actual PostgreSQL acceptance"}
// Registry fixture pins its own issuer fingerprint; use that signed public
// value and a matching controlled key digest only for core paths below.
const nodeRecord={format:"mmapp-bandwidth-node-bootstrap-v2" as const,installationId:record.installationId,issuerKeySha256:sha(issuerKey),quiescenceProof:registry.expected.quiescenceProof,registrySchemaSourceSha256:bootstrap.bandwidthPeerRegistrySchemaSourceSha256(),registryCatalogSha256:registry.expected.registryCatalogSha256,attestationRef:"Synthetic signed lifecycle bootstrap, not operator enrollment authority"}
// The original pure validator consumes these genuine fixture facts. Replace
// only the synthetic authority's issuer value to match our actual key bytes.
registry.expected.issuerKeySha256=nodeRecord.issuerKeySha256
for(const row of registry.rows)if(row.kind==="authority"){const value=JSON.parse(row.raw_json!);value.issuer_key_sha256=nodeRecord.issuerKeySha256;row.raw_json=JSON.stringify(value)}
Object.assign(controlFixture.expected,{installationId:record.installationId,controlGeneration:record.controlGeneration,sourceRecordSha256:sha(core.canonicalBandwidthControlRecord(record)),eventKeySha256:record.eventKeySha256})
const issuerKeyFile=path.join(directory,"issuer.key"),bootstrapFile=path.join(directory,"bootstrap.json")
await Promise.all([fs.writeFile(config.trustKeyFile,keys.publicKey.export({type:"spki",format:"pem"})),fs.writeFile(config.eventKeyFile,eventKey),fs.writeFile(config.attestationFile,JSON.stringify(signed("mmapp-bandwidth-control-envelope-v1",core.canonicalBandwidthControlRecord(record)))),fs.writeFile(issuerKeyFile,issuerKey),fs.writeFile(bootstrapFile,JSON.stringify(signed("mmapp-bandwidth-node-bootstrap-envelope-v1",bootstrap.canonicalBandwidthNodeBootstrap(nodeRecord))))])
const originalConnect=Pool.prototype.connect,sockets:EventEmitter[]=[],releases:boolean[]=[]
Pool.prototype.connect=(async()=>{const socket=new EventEmitter();sockets.push(socket);return Object.assign(socket,{async query(input:{text:string}){
if(input.text.startsWith("/* bandwidth-control:registry-identity */"))return {command:"SELECT",rows:[...controlFixture.rows().map(row=>({family:"core",...row})),...registry.rows.map(row=>({family:"registry",...row}))]}
if(input.text.startsWith("/* bandwidth-control:identity */"))return {command:"SELECT",rows:controlFixture.rows()}
assert.equal(input.text,"SELECT 1");return {command:"SELECT",rows:[]}
},release(unsafe:boolean){releases.push(unsafe)}})}) as never
const source=core.createConfiguredBandwidthControlSource(config)
let coreSignal:AbortSignal|undefined
const nodes=node.createConfiguredBandwidthNodeSource({...source,async connectRegistry(...args:Parameters<typeof source.connectRegistry>){const value=await source.connectRegistry(...args);coreSignal=value.lifetimeSignal;return value}},{issuerKeyFile,issuerKeySha256:nodeRecord.issuerKeySha256,bootstrapFile})
try{
for(const mode of ["core","registry","node"]){
const input={deadlineAt:Date.now()+5000},session=mode==="node"?await nodes.open(input):undefined
const connection=session?.connection??(mode==="core"?await source.connect(input):await source.connectRegistry(input,async()=>{}))
const signal=connection.lifetimeSignal;assert(signal instanceof AbortSignal);assert.equal(signal.aborted,false)
if(mode==="node")assert.equal(signal,coreSignal)
const property=Object.getOwnPropertyDescriptor(connection,"lifetimeSignal")!;assert.equal(property.value,signal);assert.equal(property.writable,false);assert.equal(property.configurable,false)
sockets.at(-1)!.emit("error",Error("NEVER_PRIVATE_PHYSICAL_LOSS"));assert.equal(signal.aborted,true)
session?.close();connection.release(true);assert.equal(releases.length,sockets.length);assert(releases.every(Boolean))
sockets.at(-1)!.emit("error",Error("NEVER_PRIVATE_LATE_LOSS"))
}
}finally{
nodes.close();await source.close();Pool.prototype.connect=originalConnect;eventKey.fill(0);issuerKey.fill(0)
const actual=await fs.realpath(directory),parent=await fs.realpath(os.tmpdir()),stat=await fs.lstat(directory)
assert(stat.isDirectory()&&!stat.isSymbolicLink());assert.equal(path.dirname(actual),parent);assert.match(path.basename(actual),/^mm-probe-lifetime-[A-Za-z0-9]+$/)
await fs.rm(actual,{recursive:true,force:true})
}
})
test("query budget cannot extend initial deadline, including mutation of caller options", async () => {
const f = fixture(), options = { deadlineAt: Date.now() + 1000 }, original = options.deadlineAt
const lease = await f.connect(options); options.deadlineAt += 100000
await lease.query("SELECT 1", [], { timeoutMs: 10000 })
assert.equal(f.queries[0].query_timeout <= original - Date.now() + 100, true)
assert.equal(f.queries[0].query_timeout <= 1000, true)
lease.release()
})
test("invalid query budgets reject before dispatch without poisoning a healthy idle lease", async () => {
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
for (const timeoutMs of [0, -1, 10001, NaN, Infinity, 1.5]) await assert.rejects(lease.query("SELECT 1", [], { timeoutMs }), refused)
await assert.rejects(lease.query(""), refused)
assert.equal(lease.lifetimeSignal?.aborted,false)
assert.equal(f.queries.length, 0); await lease.query("SELECT 1"); lease.release()
assert.deepEqual(f.releases, [false])
})
for (const source of ["initial", "query"] as const) test("in-flight " + source + " abort destroys once; late success cannot revive the lease", async () => {
const driver = deferred<Result>(), initial = new AbortController(), query = new AbortController(), f = fixture(() => driver.promise)
const lease = await f.connect({ deadlineAt: Date.now() + 1000, signal: initial.signal })
const pending = lease.query("SELECT 1", [], { timeoutMs: 1000, signal: query.signal })
;(source === "initial" ? initial : query).abort(Error("NEVER_PRIVATE_ABORT_REASON"))
await assert.rejects(pending, unknown); assert.deepEqual(f.releases, [true])
assert.equal(lease.lifetimeSignal?.aborted,true)
lease.release(); driver.resolve(result()); await tick()
await assert.rejects(lease.query("SELECT 2"), refused)
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
})
for (const source of ["initial", "query"] as const) test("wall timer bounds hung driver by " + source + " deadline", async () => {
const driver = deferred<Result>(), f = fixture(() => driver.promise)
const options = { deadlineAt: Date.now() + (source === "initial" ? 25 : 1000) }, lease = await f.connect(options)
options.deadlineAt += 100000
const started = Date.now(), pending = lease.query("SELECT 1", [], { timeoutMs: source === "query" ? 25 : 1000 })
await assert.rejects(pending, unknown)
assert.equal(lease.lifetimeSignal?.aborted,true)
assert.equal(Date.now() - started < 1000, true); assert.deepEqual(f.releases, [true])
driver.reject(Error("NEVER_PRIVATE_LATE_FAILURE")); await tick(); lease.release(); assert.deepEqual(f.releases, [true])
})
test("driver failure and idle socket error permanently destroy the lease", async () => {
const f = fixture(async () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
await assert.rejects(lease.query("SELECT 1"), unknown); lease.release(); f.emitError()
assert.equal(lease.lifetimeSignal?.aborted,true)
assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 1)
const idle = fixture(), idleLease = await idle.connect({ deadlineAt: Date.now() + 1000 })
idle.emitError(); await assert.rejects(idleLease.query("SELECT 1"), refused); idleLease.release()
assert.deepEqual(idle.releases, [true]); assert.equal(idle.queries.length, 0)
})
test("overlap never dispatches a second query and invalidates the first attempt", async () => {
const driver = deferred<Result>(), f = fixture(() => driver.promise), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
const first = lease.query("SELECT 1"), second = lease.query("SELECT 2")
await assert.rejects(second, refused); await assert.rejects(first, unknown)
driver.resolve(result()); await tick(); lease.release()
assert.equal(f.queries.length, 1); assert.equal(f.maximumActive(), 1); assert.deepEqual(f.releases, [true])
})
test("release during in-flight work destroys instead of healthy pooling", async () => {
const driver = deferred<Result>(), f = fixture(() => driver.promise), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
const pending = lease.query("SELECT 1"); lease.release(); lease.release()
await assert.rejects(pending, unknown); driver.resolve(result()); await tick()
assert.deepEqual(f.releases, [true])
})
test("COMMIT accepts only acknowledged COMMIT and refuses ROLLBACK or lost ACK", async () => {
const good = fixture(), goodLease = await good.connect({ deadlineAt: Date.now() + 1000 })
await goodLease.query("COMMIT"); goodLease.release(); assert.deepEqual(good.releases, [false])
for (const mode of ["rollback", "lost"] as const) {
for (const statement of ["COMMIT;", "/* owner */ COMMIT TRANSACTION; -- durable", "END WORK;"]) {
const f = fixture(async () => { if (mode === "lost") throw Error("NEVER_PRIVATE_COMMIT_ACK_FAILURE"); return result("ROLLBACK") }), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
await assert.rejects(lease.query(statement), unknown); lease.release()
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
}
}
})
test("COMMIT abort after driver-side completion is unknown and never pooled healthy", async () => {
const controller = new AbortController(), f = fixture(async () => { controller.abort(); return result("COMMIT") })
const lease = await f.connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
await assert.rejects(lease.query("COMMIT"), unknown); lease.release()
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
})
test("idle abort expires only its lease; a fresh quarantine checkout uses fresh options", async () => {
const f = fixture(), fresh = fixture(), clients = [f.client, fresh.client], aborted = new AbortController()
let checkouts = 0
const connect = createProbeStorageConnectionConnector(async () => { checkouts++; return clients.shift()! })
const lease = await connect({ deadlineAt: Date.now() + 1000, signal: aborted.signal })
aborted.abort(); lease.release(); assert.deepEqual(f.releases, [true])
const freshLease = await connect({ deadlineAt: Date.now() + 1000, signal: new AbortController().signal })
await freshLease.query("SELECT 1"); freshLease.release()
assert.deepEqual(fresh.releases, [false]); assert.equal(checkouts, 2); assert.equal(f.queries.length, 0)
})
test("idle initial deadline destroys without waiting for a query or explicit release", async () => {
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 25 })
await new Promise<void>(resolve => setTimeout(resolve, 40))
assert.deepEqual(f.releases, [true]); await assert.rejects(lease.query("SELECT 1"), refused)
lease.release(); assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 0)
})
test("an abandoned late client's throwing release is absorbed and attempted once", async () => {
const f = fixture(), late = deferred<ProbeStoragePgClient>(), controller = new AbortController()
let releases = 0
const client: ProbeStoragePgClient = { ...f.client, release() { releases++; throw Error("NEVER_PRIVATE_RELEASE_DIAGNOSTIC") } }
const connect = createProbeStorageConnectionConnector(() => late.promise), pending = connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
controller.abort(); await assert.rejects(pending, refused)
late.resolve(client); await tick(); assert.equal(releases, 1); assert.equal(f.queries.length, 0)
})
const diagnosticId = "11111111-1111-4111-8111-111111111111"
async function guardFailure(work: Promise<unknown>, branch: string, kind = "other", phase = "query") {
let caught: unknown
try { await work; assert.fail("Expected guard failure") } catch (error) { caught = error }
assert(caught instanceof ProbeStorageUnknown || caught instanceof ProbeStorageRefused)
assert.deepEqual(capturePollError(caught).probe, { phase, queryKind: kind, branch })
assert(!JSON.stringify(capturePollError(caught)).includes("NEVER_PRIVATE"))
return caught
}
test("guard provenance: plain driver rejection differs from synthesized Unknown and exact kinds are diagnostic only", async () => {
const meta = "SELECT o.id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_operation_steps s ON s.operation_id=o.id WHERE o.connection_id=$1 AND p.operation_id=$2 AND s.step_index=0 AND s.intended_state=$3::jsonb LIMIT 2"
for (const [text, kind] of [[principalSourceSql,"principal"],[meta,"meta"],["/* private */ END WORK;","commit"],[meta+" ","other"],["SELECT 'NEVER_PRIVATE SQL/password'","other"]]) {
const f=fixture(async()=>{throw Error("NEVER_PRIVATE driver timeout syntax error 57014")}),lease=await f.connect({deadlineAt:Date.now()+1000})
const error=await guardFailure(lease.query(text,["NEVER_PRIVATE credential"]),"driver-reject",kind)
assert.equal(capturePollError(error).origin.kind,"other");assert.equal(capturePollError(error).origin.sqlState,null)
let final=error as Error;for(const site of ["storage","journal.tx","poll.safe"] as const)final=carryPollError(new Error("public"),final,site)
const log=pollErrorLog(final,diagnosticId,"servers.poll")!
assert.deepEqual(log.provenance.sites,["probe.query","storage","journal.tx","poll.safe"])
assert.equal(log.provenance.probe?.branch,"driver-reject");assert(Buffer.byteLength(JSON.stringify(log))<=1024)
assert(!JSON.stringify(log).includes("NEVER_PRIVATE"));assert.equal(pollErrorLog(final,diagnosticId,"other.operation"),null)
assert.equal(f.queries.length,1);assert.deepEqual(f.releases,[true]);lease.release()
}
const pg=new DatabaseError("NEVER_PRIVATE real pg class",1,"error");pg.code="57014"
const driver=fixture(async()=>{throw pg}),lease=await driver.connect({deadlineAt:Date.now()+1000})
const error=await guardFailure(lease.query(principalSourceSql),"driver-reject","principal")
assert.equal(capturePollError(error).origin.kind,"pg");assert.equal(capturePollError(error).origin.sqlState,"57014")
})
test("guard provenance: synchronous physical query throw has its own branch", async () => {
let calls=0,releases=0
const lease=await createProbeStorageConnectionConnector(async()=>({query(){calls++;throw Error("NEVER_PRIVATE sync")},release(unsafe){assert.equal(unsafe,true);releases++}}))({deadlineAt:Date.now()+1000})
await guardFailure(lease.query("COMMIT"),"driver-throw","commit");assert.equal(calls,1);assert.equal(releases,1)
})
test("guard provenance: query wall timeout, caller abort and lease socket close remain distinct and late success cannot clear them", async () => {
for(const mode of ["deadline","abort","lease-error","lease-closed"] as const){
const driver=deferred<Result>(),controller=new AbortController(),f=fixture(()=>driver.promise),lease=await f.connect({deadlineAt:Date.now()+1000})
const pending=lease.query("SELECT 1",[],{timeoutMs:mode==="deadline"?20:1000,signal:controller.signal})
if(mode==="abort")controller.abort(Error("NEVER_PRIVATE abort"));if(mode==="lease-error")f.emitError();if(mode==="lease-closed")lease.release()
const error=await guardFailure(pending,mode);driver.resolve(result());await tick()
assert.equal(capturePollError(error).probe?.branch,mode);assert.equal(f.queries.length,1);assert.deepEqual(f.releases,[true]);lease.release()
}
})
test("guard provenance: initial lease deadline, result shape, result accessor and COMMIT ACK guard are synthetic, not driver rejection", async () => {
const hung=deferred<Result>(),expired=fixture(()=>hung.promise),old=await expired.connect({deadlineAt:Date.now()+20})
await guardFailure(old.query("SELECT 1",[],{timeoutMs:1000}),"lease-timeout");hung.resolve(result());await tick()
for(const [value,text,branch] of [[{rows:"NEVER_PRIVATE rows",command:"SELECT"},"SELECT 1","result-shape"],[null,"SELECT 1","result-access"],[result("ROLLBACK"),"COMMIT","commit-ack"]] as const){
const f=fixture(async()=>value as unknown as Result),lease=await f.connect({deadlineAt:Date.now()+1000})
await guardFailure(lease.query(text),branch,text==="COMMIT"?"commit":"other");assert.deepEqual(f.releases,[true]);assert.equal(f.queries.length,1)
}
})
test("guard provenance: post-resolution closed and inactive checks retain their precise branch", async () => {
for(const mode of ["result-closed","result-inactive"] as const){
const realNow=Date.now,start=realNow(),socket=new EventEmitter();let clock=start,releases=0
if(mode==="result-inactive")Date.now=()=>clock
try{
const lease=await createProbeStorageConnectionConnector(async()=>({on:socket.on.bind(socket),removeListener:socket.removeListener.bind(socket),query(){return Promise.resolve(result()).then(value=>{queueMicrotask(()=>queueMicrotask(()=>{if(mode==="result-closed")socket.emit("error",Error("NEVER_PRIVATE socket"));else clock=start+2000}));return value})},release(unsafe){assert.equal(unsafe,true);releases++}}))({deadlineAt:start+1000})
await guardFailure(lease.query("SELECT 1"),mode);assert.equal(releases,1)
}finally{Date.now=realNow}
}
})
test("guard provenance: checkout rejected/thrown/aborted/timed out never acquires query authority", async () => {
for(const mode of ["driver-reject","driver-throw","abort","deadline"] as const){
const controller=new AbortController(),later=deferred<ProbeStoragePgClient>(),f=fixture();let calls=0
const checkout:()=>Promise<ProbeStoragePgClient>=mode==="driver-throw"?()=>{calls++;throw Error("NEVER_PRIVATE checkout")}:mode==="driver-reject"?async()=>{calls++;throw Error("NEVER_PRIVATE checkout")}:()=>{calls++;return later.promise}
const pending=createProbeStorageConnectionConnector(checkout)({deadlineAt:Date.now()+(mode==="deadline"?20:1000),signal:controller.signal})
if(mode==="abort")controller.abort();await guardFailure(pending,mode,"other","checkout")
later.resolve(f.client);await tick();assert.equal(calls,1);assert.equal(f.queries.length,0)
}
})
test("guard provenance: pre-dispatch guards preserve Refused and query counts", async () => {
const f=fixture(),lease=await f.connect({deadlineAt:Date.now()+1000})
await guardFailure(lease.query("",[],{timeoutMs:1}),"input");assert.equal(f.queries.length,0)
const aborted=new AbortController();aborted.abort();await guardFailure(lease.query("SELECT 1",[],{timeoutMs:1,signal:aborted.signal}),"inactive")
await guardFailure(lease.query("SELECT 1"),"closed");assert.equal(f.queries.length,0);assert.deepEqual(f.releases,[true])
const waiting=deferred<Result>(),busy=fixture(()=>waiting.promise),borrow=await busy.connect({deadlineAt:Date.now()+1000}),first=borrow.query("SELECT 1")
const firstCheck=guardFailure(first,"lease-closed");await guardFailure(borrow.query("SELECT 2"),"busy");await firstCheck;waiting.resolve(result());await tick();assert.equal(busy.queries.length,1)
})
test("guard provenance: successful result/reference, named principal and public error fields stay unchanged", async () => {
const value=result("COMMIT"),f=fixture(async()=>value),lease=await f.connect({deadlineAt:Date.now()+1000})
assert.equal(await lease.query("COMMIT"),value);lease.release();assert.deepEqual(f.releases,[false])
const plain=new ProbeStorageUnknown(),before=Object.getOwnPropertyDescriptors(plain)
carryPollError(plain,new Error("NEVER_PRIVATE"),"probe.query");assert.deepEqual(Object.getOwnPropertyDescriptors(plain),before)
})