385 lines
32 KiB
TypeScript
385 lines
32 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { test } from "node:test"
|
|
import { capturePollError, carryPollError, pollErrorLog } from "./server-poll-error-provenance.js"
|
|
import { EventEmitter } from "node:events"
|
|
import { DatabaseError, type QueryConfig } from "pg"
|
|
import { createProbeStorageConnectionConnector, connectProbeStorage, type ProbeStoragePgClient } from "./server-probe-storage-connection.js"
|
|
import { ProbeStorageRefused, ProbeStorageUnknown } from "./server-probe-journal-maintenance.js"
|
|
import { principalSourceSql, principalSourceStatementName } from "../modules/mcp/principal-security-sql.js"
|
|
|
|
type Result = Awaited<ReturnType<ProbeStoragePgClient["query"]>>
|
|
const result = (command = "SELECT"): Result => ({ rows: [], command })
|
|
function deferred<T>() { let resolve!: (value: T) => void, reject!: (error: unknown) => void; const promise = new Promise<T>((yes, no) => { resolve = yes; reject = no }); return { promise, resolve, reject } }
|
|
const tick = () => new Promise<void>(resolve => setImmediate(resolve))
|
|
function fixture(handler: (config: QueryConfig & { query_timeout: number }) => Promise<Result> = async config => result(config.text.trim() === "COMMIT" ? "COMMIT" : "SELECT")) {
|
|
const queries: Array<QueryConfig & { query_timeout: number }> = [], releases: boolean[] = [], listeners = new Set<(error: Error) => void>()
|
|
let active = 0, maximumActive = 0, checkouts = 0
|
|
const client: ProbeStoragePgClient = {
|
|
async query(config) { queries.push(config); active++; maximumActive = Math.max(maximumActive, active); try { return await handler(config) } finally { active-- } },
|
|
release(unsafe = false) { releases.push(unsafe) },
|
|
on(_event, listener) { listeners.add(listener) }, removeListener(_event, listener) { listeners.delete(listener) },
|
|
}
|
|
const connect = createProbeStorageConnectionConnector(async () => { checkouts++; return client })
|
|
return { client, connect, queries, releases, checkouts: () => checkouts, maximumActive: () => maximumActive, emitError: () => { for (const listener of listeners) listener(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC")) } }
|
|
}
|
|
const refused = (error: unknown) => error instanceof ProbeStorageRefused && !error.message.includes("NEVER_")
|
|
const unknown = (error: unknown) => error instanceof ProbeStorageUnknown && !error.message.includes("NEVER_")
|
|
|
|
test("only exact fixed source is named; fresh arguments/timeouts survive and near matches stay unnamed", async () => {
|
|
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 30000 })
|
|
try {
|
|
for (const values of [["credential-a", "connection-a", true], ["credential-b", "connection-b", false]]) {
|
|
await lease.query(principalSourceSql, values, { timeoutMs: 543 })
|
|
const sent = f.queries.at(-1)!
|
|
assert.equal(sent.name, principalSourceStatementName); assert.equal(sent.text, principalSourceSql)
|
|
assert.equal(sent.values, values); assert(sent.query_timeout > 0 && sent.query_timeout <= 543)
|
|
}
|
|
assert(principalSourceStatementName.length <= 63)
|
|
for (const text of [principalSourceSql + " ", "/* mcp-principal:coherent-source-v1 */ SELECT 1", "SELECT 1"]) {
|
|
await lease.query(text); assert.equal(Object.hasOwn(f.queries.at(-1)!, "name"), false)
|
|
}
|
|
} finally { lease.release() }
|
|
})
|
|
|
|
test("fixed-source schema failure destroys once without unnamed fallback or retry", async () => {
|
|
const f = fixture(async () => { throw Object.assign(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC"), { code: "0A000" }) })
|
|
const lease = await f.connect({ deadlineAt: Date.now() + 30000 })
|
|
await assert.rejects(lease.query(principalSourceSql, ["a", "b", true]), unknown)
|
|
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true]); assert.equal(lease.lifetimeSignal?.aborted, true)
|
|
await assert.rejects(lease.query(principalSourceSql, ["a", "b", true]), refused)
|
|
assert.equal(f.queries.length, 1); lease.release(); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("connector import/construction is inert; invalid/pre-aborted checkout performs no work", async () => {
|
|
const f = fixture(), aborted = new AbortController(); aborted.abort(Error("NEVER_PRIVATE_ABORT_REASON"))
|
|
assert.equal(typeof connectProbeStorage, "function"); assert.equal(f.checkouts(), 0)
|
|
await assert.rejects(f.connect({ deadlineAt: Date.now() + 1000, signal: aborted.signal }), refused)
|
|
for (const deadlineAt of [0, NaN, Infinity, Date.now() - 1, Date.now() + 0.5]) await assert.rejects(f.connect({ deadlineAt }), refused)
|
|
assert.equal(f.checkouts(), 0); assert.equal(f.queries.length, 0); assert.deepEqual(f.releases, [])
|
|
})
|
|
|
|
for (const mode of ["abort", "deadline"] as const) test("bounded checkout " + mode + " destroys an eventual client once", async () => {
|
|
const f = fixture(), pendingClient = deferred<ProbeStoragePgClient>(), controller = new AbortController()
|
|
let checkouts = 0
|
|
const connect = createProbeStorageConnectionConnector(() => { checkouts++; return pendingClient.promise })
|
|
const started = Date.now(), waiting = connect({ deadlineAt: Date.now() + (mode === "deadline" ? 25 : 1000), signal: controller.signal })
|
|
if (mode === "abort") controller.abort(Error("NEVER_PRIVATE_ABORT_REASON"))
|
|
await assert.rejects(waiting, refused)
|
|
assert.equal(checkouts, 1); assert.equal(Date.now() - started < 1000, true); assert.deepEqual(f.releases, [])
|
|
pendingClient.resolve(f.client); await tick()
|
|
assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 0)
|
|
})
|
|
|
|
test("late checkout failure is consumed after cancellation and never retried", async () => {
|
|
const pendingClient = deferred<ProbeStoragePgClient>(), controller = new AbortController()
|
|
let checkouts = 0, unhandled = 0
|
|
const listener = () => { unhandled++ }
|
|
process.on("unhandledRejection", listener)
|
|
try {
|
|
const connect = createProbeStorageConnectionConnector(() => { checkouts++; return pendingClient.promise })
|
|
const waiting = connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
|
|
controller.abort(); await assert.rejects(waiting, refused)
|
|
pendingClient.reject(Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC")); await tick(); await tick()
|
|
assert.equal(checkouts, 1); assert.equal(unhandled, 0)
|
|
} finally { process.removeListener("unhandledRejection", listener) }
|
|
})
|
|
|
|
test("checkout driver rejection and synchronous throw are sanitized", async () => {
|
|
for (const checkout of [async () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }, () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }]) {
|
|
await assert.rejects(createProbeStorageConnectionConnector(checkout)({ deadlineAt: Date.now() + 1000 }), refused)
|
|
}
|
|
})
|
|
|
|
test("healthy lease carries values and pg query_timeout; release is once and closed is unusable", async () => {
|
|
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 30000 })
|
|
await lease.query("SELECT $1", [7], { timeoutMs: 321 })
|
|
assert.equal(f.queries.length, 1); assert.deepEqual(f.queries[0].values, [7])
|
|
assert.equal(f.queries[0].query_timeout > 0 && f.queries[0].query_timeout <= 321, true)
|
|
await lease.query("SELECT 1")
|
|
assert.equal(f.queries[1].query_timeout > 0 && f.queries[1].query_timeout <= 10000, true)
|
|
lease.release(); lease.release(true)
|
|
assert.deepEqual(f.releases, [false])
|
|
await assert.rejects(lease.query("SELECT 2"), refused); assert.equal(f.queries.length, 2)
|
|
})
|
|
|
|
test("actual connector publishes one immutable native lifetime signal; every close revokes BEFORE exact-once physical release",async()=>{
|
|
for(const mode of ["healthy","destroy","error","abort","deadline"]){
|
|
const f=fixture(),input=new AbortController(),lease=await f.connect({deadlineAt:Date.now()+(mode==="deadline"?30:1000),signal:input.signal})
|
|
const signal=lease.lifetimeSignal;assert(signal instanceof AbortSignal);assert.equal(signal.aborted,false)
|
|
const descriptor=Object.getOwnPropertyDescriptor(lease,"lifetimeSignal")!
|
|
assert.equal(descriptor.value,signal);assert.equal(descriptor.get,undefined);assert.equal(descriptor.writable,false);assert.equal(descriptor.configurable,false)
|
|
assert.throws(()=>Object.defineProperty(lease,"lifetimeSignal",{value:new AbortController().signal}))
|
|
const order:string[]=[],rawRelease=f.client.release.bind(f.client)
|
|
signal.addEventListener("abort",()=>order.push("revoked"),{once:true})
|
|
f.client.release=(unsafe)=>{assert.equal(signal.aborted,true);order.push("release");rawRelease(unsafe)}
|
|
if(mode==="healthy")lease.release()
|
|
if(mode==="destroy")lease.release(true)
|
|
if(mode==="error")f.emitError()
|
|
if(mode==="abort")input.abort()
|
|
if(mode==="deadline")await new Promise<void>(done=>setTimeout(done,45))
|
|
assert.equal(signal.aborted,true);assert.deepEqual(order,["revoked","release"])
|
|
lease.release(true);f.emitError();assert.deepEqual(f.releases,[mode!=="healthy"])
|
|
await assert.rejects(lease.query("SELECT forbidden_after_close"),refused);assert.equal(f.queries.length,0)
|
|
}
|
|
})
|
|
|
|
test("idle owner loss while external async work is pending synchronously revokes the retained lease",async()=>{
|
|
const f=fixture(),lease=await f.connect({deadlineAt:Date.now()+1000}),pending=deferred<void>()
|
|
const signal=lease.lifetimeSignal!;let consumed=false
|
|
const afterExternal=(async()=>{await pending.promise;if(signal.aborted)return;consumed=true})()
|
|
f.emitError();assert.equal(signal.aborted,true);pending.resolve();await afterExternal
|
|
assert.equal(consumed,false);assert.deepEqual(f.releases,[true]);assert.equal(f.checkouts(),1);assert.equal(f.queries.length,0)
|
|
})
|
|
|
|
test("late abandoned checkout retains socket error absorber, never publishes facade or replays checkout",async()=>{
|
|
const waiting=deferred<ProbeStoragePgClient>(),controller=new AbortController(),socket=new EventEmitter()
|
|
let checkouts=0,releases=0,published=false
|
|
const client:ProbeStoragePgClient={on:socket.on.bind(socket),removeListener:socket.removeListener.bind(socket),async query(){assert.fail("late client cannot query")},release(unsafe){assert.equal(unsafe,true);releases++;queueMicrotask(()=>socket.emit("error",Error("NEVER_PRIVATE_LATE_SOCKET")))}}
|
|
const connect=createProbeStorageConnectionConnector(()=>{checkouts++;return waiting.promise})
|
|
const result=connect({deadlineAt:Date.now()+1000,signal:controller.signal}).then(()=>{published=true})
|
|
controller.abort();await assert.rejects(result,refused);waiting.resolve(client);await tick();await tick()
|
|
assert.equal(published,false);assert.equal(checkouts,1);assert.equal(releases,1);assert(socket.listenerCount("error")>0)
|
|
socket.emit("error",Error("NEVER_PRIVATE_REPEATED_SOCKET"));assert.equal(releases,1)
|
|
})
|
|
|
|
test("genuine signed core/connectRegistry/node strong facades preserve physical lifetime revocation (synthetic PG only)",async()=>{
|
|
const fs=await import("node:fs/promises"),path=await import("node:path"),os=await import("node:os"),crypto=await import("node:crypto"),{Pool}=await import("pg")
|
|
const core=await import("./bandwidth-control-source.js"),node=await import("./bandwidth-node-source.js"),bootstrap=await import("./bandwidth-node-bootstrap.js")
|
|
const controlFixture=await import("../test/fixtures/bandwidth-control-identity.js"),registry=(await import("../test/fixtures/bandwidth-peer-identity.js")).fixture()
|
|
const directory=await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()),"mm-probe-lifetime-")),keys=crypto.generateKeyPairSync("ed25519"),eventKey=crypto.randomBytes(32),issuerKey=crypto.randomBytes(32)
|
|
const sha=(value:string|Buffer)=>crypto.createHash("sha256").update(value).digest("hex"),signed=(format:string,text:string)=>{const bytes=Buffer.from(text);return {format,payload:bytes.toString("base64url"),signature:crypto.sign(null,bytes,keys.privateKey).toString("base64url")}}
|
|
const config={databaseUrl:"postgresql://bandwidth_runtime:[email protected]:1/owned_database",applicationDatabaseUrl:"postgresql://restricted_application:[email protected]:1/application_database",trustKeyFile:path.join(directory,"trust.pem"),eventKeyFile:path.join(directory,"event.key"),attestationFile:path.join(directory,"source.json"),trustKeySha256:sha(keys.publicKey.export({type:"spki",format:"der"}))}
|
|
const record={format:"mmapp-bandwidth-control-source-v1" as const,installationId:registry.expected.installationId,controlGeneration:crypto.randomUUID(),databaseName:"owned_database",databaseOid:123,runtimeRole:"bandwidth_runtime",ownerRole:"fixed_owner",endpointSha256:core.bandwidthControlEndpointSha256(config.databaseUrl),schemaSourceSha256:core.bandwidthControlSchemaSourceSha256(),catalogSha256:controlFixture.expected.catalogSha256,eventKeySha256:sha(eventKey),attestationRef:"Synthetic signed lifecycle propagation, not actual PostgreSQL acceptance"}
|
|
// Registry fixture pins its own issuer fingerprint; use that signed public
|
|
// value and a matching controlled key digest only for core paths below.
|
|
const nodeRecord={format:"mmapp-bandwidth-node-bootstrap-v2" as const,installationId:record.installationId,issuerKeySha256:sha(issuerKey),quiescenceProof:registry.expected.quiescenceProof,registrySchemaSourceSha256:bootstrap.bandwidthPeerRegistrySchemaSourceSha256(),registryCatalogSha256:registry.expected.registryCatalogSha256,attestationRef:"Synthetic signed lifecycle bootstrap, not operator enrollment authority"}
|
|
// The original pure validator consumes these genuine fixture facts. Replace
|
|
// only the synthetic authority's issuer value to match our actual key bytes.
|
|
registry.expected.issuerKeySha256=nodeRecord.issuerKeySha256
|
|
for(const row of registry.rows)if(row.kind==="authority"){const value=JSON.parse(row.raw_json!);value.issuer_key_sha256=nodeRecord.issuerKeySha256;row.raw_json=JSON.stringify(value)}
|
|
Object.assign(controlFixture.expected,{installationId:record.installationId,controlGeneration:record.controlGeneration,sourceRecordSha256:sha(core.canonicalBandwidthControlRecord(record)),eventKeySha256:record.eventKeySha256})
|
|
const issuerKeyFile=path.join(directory,"issuer.key"),bootstrapFile=path.join(directory,"bootstrap.json")
|
|
await Promise.all([fs.writeFile(config.trustKeyFile,keys.publicKey.export({type:"spki",format:"pem"})),fs.writeFile(config.eventKeyFile,eventKey),fs.writeFile(config.attestationFile,JSON.stringify(signed("mmapp-bandwidth-control-envelope-v1",core.canonicalBandwidthControlRecord(record)))),fs.writeFile(issuerKeyFile,issuerKey),fs.writeFile(bootstrapFile,JSON.stringify(signed("mmapp-bandwidth-node-bootstrap-envelope-v1",bootstrap.canonicalBandwidthNodeBootstrap(nodeRecord))))])
|
|
const originalConnect=Pool.prototype.connect,sockets:EventEmitter[]=[],releases:boolean[]=[]
|
|
Pool.prototype.connect=(async()=>{const socket=new EventEmitter();sockets.push(socket);return Object.assign(socket,{async query(input:{text:string}){
|
|
if(input.text.startsWith("/* bandwidth-control:registry-identity */"))return {command:"SELECT",rows:[...controlFixture.rows().map(row=>({family:"core",...row})),...registry.rows.map(row=>({family:"registry",...row}))]}
|
|
if(input.text.startsWith("/* bandwidth-control:identity */"))return {command:"SELECT",rows:controlFixture.rows()}
|
|
assert.equal(input.text,"SELECT 1");return {command:"SELECT",rows:[]}
|
|
},release(unsafe:boolean){releases.push(unsafe)}})}) as never
|
|
const source=core.createConfiguredBandwidthControlSource(config)
|
|
let coreSignal:AbortSignal|undefined
|
|
const nodes=node.createConfiguredBandwidthNodeSource({...source,async connectRegistry(...args:Parameters<typeof source.connectRegistry>){const value=await source.connectRegistry(...args);coreSignal=value.lifetimeSignal;return value}},{issuerKeyFile,issuerKeySha256:nodeRecord.issuerKeySha256,bootstrapFile})
|
|
try{
|
|
for(const mode of ["core","registry","node"]){
|
|
const input={deadlineAt:Date.now()+5000},session=mode==="node"?await nodes.open(input):undefined
|
|
const connection=session?.connection??(mode==="core"?await source.connect(input):await source.connectRegistry(input,async()=>{}))
|
|
const signal=connection.lifetimeSignal;assert(signal instanceof AbortSignal);assert.equal(signal.aborted,false)
|
|
if(mode==="node")assert.equal(signal,coreSignal)
|
|
const property=Object.getOwnPropertyDescriptor(connection,"lifetimeSignal")!;assert.equal(property.value,signal);assert.equal(property.writable,false);assert.equal(property.configurable,false)
|
|
sockets.at(-1)!.emit("error",Error("NEVER_PRIVATE_PHYSICAL_LOSS"));assert.equal(signal.aborted,true)
|
|
session?.close();connection.release(true);assert.equal(releases.length,sockets.length);assert(releases.every(Boolean))
|
|
sockets.at(-1)!.emit("error",Error("NEVER_PRIVATE_LATE_LOSS"))
|
|
}
|
|
}finally{
|
|
nodes.close();await source.close();Pool.prototype.connect=originalConnect;eventKey.fill(0);issuerKey.fill(0)
|
|
const actual=await fs.realpath(directory),parent=await fs.realpath(os.tmpdir()),stat=await fs.lstat(directory)
|
|
assert(stat.isDirectory()&&!stat.isSymbolicLink());assert.equal(path.dirname(actual),parent);assert.match(path.basename(actual),/^mm-probe-lifetime-[A-Za-z0-9]+$/)
|
|
await fs.rm(actual,{recursive:true,force:true})
|
|
}
|
|
})
|
|
|
|
test("query budget cannot extend initial deadline, including mutation of caller options", async () => {
|
|
const f = fixture(), options = { deadlineAt: Date.now() + 1000 }, original = options.deadlineAt
|
|
const lease = await f.connect(options); options.deadlineAt += 100000
|
|
await lease.query("SELECT 1", [], { timeoutMs: 10000 })
|
|
assert.equal(f.queries[0].query_timeout <= original - Date.now() + 100, true)
|
|
assert.equal(f.queries[0].query_timeout <= 1000, true)
|
|
lease.release()
|
|
})
|
|
|
|
test("invalid query budgets reject before dispatch without poisoning a healthy idle lease", async () => {
|
|
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
|
|
for (const timeoutMs of [0, -1, 10001, NaN, Infinity, 1.5]) await assert.rejects(lease.query("SELECT 1", [], { timeoutMs }), refused)
|
|
await assert.rejects(lease.query(""), refused)
|
|
assert.equal(lease.lifetimeSignal?.aborted,false)
|
|
assert.equal(f.queries.length, 0); await lease.query("SELECT 1"); lease.release()
|
|
assert.deepEqual(f.releases, [false])
|
|
})
|
|
|
|
for (const source of ["initial", "query"] as const) test("in-flight " + source + " abort destroys once; late success cannot revive the lease", async () => {
|
|
const driver = deferred<Result>(), initial = new AbortController(), query = new AbortController(), f = fixture(() => driver.promise)
|
|
const lease = await f.connect({ deadlineAt: Date.now() + 1000, signal: initial.signal })
|
|
const pending = lease.query("SELECT 1", [], { timeoutMs: 1000, signal: query.signal })
|
|
;(source === "initial" ? initial : query).abort(Error("NEVER_PRIVATE_ABORT_REASON"))
|
|
await assert.rejects(pending, unknown); assert.deepEqual(f.releases, [true])
|
|
assert.equal(lease.lifetimeSignal?.aborted,true)
|
|
lease.release(); driver.resolve(result()); await tick()
|
|
await assert.rejects(lease.query("SELECT 2"), refused)
|
|
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
for (const source of ["initial", "query"] as const) test("wall timer bounds hung driver by " + source + " deadline", async () => {
|
|
const driver = deferred<Result>(), f = fixture(() => driver.promise)
|
|
const options = { deadlineAt: Date.now() + (source === "initial" ? 25 : 1000) }, lease = await f.connect(options)
|
|
options.deadlineAt += 100000
|
|
const started = Date.now(), pending = lease.query("SELECT 1", [], { timeoutMs: source === "query" ? 25 : 1000 })
|
|
await assert.rejects(pending, unknown)
|
|
assert.equal(lease.lifetimeSignal?.aborted,true)
|
|
assert.equal(Date.now() - started < 1000, true); assert.deepEqual(f.releases, [true])
|
|
driver.reject(Error("NEVER_PRIVATE_LATE_FAILURE")); await tick(); lease.release(); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("driver failure and idle socket error permanently destroy the lease", async () => {
|
|
const f = fixture(async () => { throw Error("NEVER_PRIVATE_DRIVER_DIAGNOSTIC") }), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
|
|
await assert.rejects(lease.query("SELECT 1"), unknown); lease.release(); f.emitError()
|
|
assert.equal(lease.lifetimeSignal?.aborted,true)
|
|
assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 1)
|
|
const idle = fixture(), idleLease = await idle.connect({ deadlineAt: Date.now() + 1000 })
|
|
idle.emitError(); await assert.rejects(idleLease.query("SELECT 1"), refused); idleLease.release()
|
|
assert.deepEqual(idle.releases, [true]); assert.equal(idle.queries.length, 0)
|
|
})
|
|
|
|
test("overlap never dispatches a second query and invalidates the first attempt", async () => {
|
|
const driver = deferred<Result>(), f = fixture(() => driver.promise), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
|
|
const first = lease.query("SELECT 1"), second = lease.query("SELECT 2")
|
|
await assert.rejects(second, refused); await assert.rejects(first, unknown)
|
|
driver.resolve(result()); await tick(); lease.release()
|
|
assert.equal(f.queries.length, 1); assert.equal(f.maximumActive(), 1); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("release during in-flight work destroys instead of healthy pooling", async () => {
|
|
const driver = deferred<Result>(), f = fixture(() => driver.promise), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
|
|
const pending = lease.query("SELECT 1"); lease.release(); lease.release()
|
|
await assert.rejects(pending, unknown); driver.resolve(result()); await tick()
|
|
assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("COMMIT accepts only acknowledged COMMIT and refuses ROLLBACK or lost ACK", async () => {
|
|
const good = fixture(), goodLease = await good.connect({ deadlineAt: Date.now() + 1000 })
|
|
await goodLease.query("COMMIT"); goodLease.release(); assert.deepEqual(good.releases, [false])
|
|
for (const mode of ["rollback", "lost"] as const) {
|
|
for (const statement of ["COMMIT;", "/* owner */ COMMIT TRANSACTION; -- durable", "END WORK;"]) {
|
|
const f = fixture(async () => { if (mode === "lost") throw Error("NEVER_PRIVATE_COMMIT_ACK_FAILURE"); return result("ROLLBACK") }), lease = await f.connect({ deadlineAt: Date.now() + 1000 })
|
|
await assert.rejects(lease.query(statement), unknown); lease.release()
|
|
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
|
|
}
|
|
}
|
|
})
|
|
|
|
test("COMMIT abort after driver-side completion is unknown and never pooled healthy", async () => {
|
|
const controller = new AbortController(), f = fixture(async () => { controller.abort(); return result("COMMIT") })
|
|
const lease = await f.connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
|
|
await assert.rejects(lease.query("COMMIT"), unknown); lease.release()
|
|
assert.equal(f.queries.length, 1); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("idle abort expires only its lease; a fresh quarantine checkout uses fresh options", async () => {
|
|
const f = fixture(), fresh = fixture(), clients = [f.client, fresh.client], aborted = new AbortController()
|
|
let checkouts = 0
|
|
const connect = createProbeStorageConnectionConnector(async () => { checkouts++; return clients.shift()! })
|
|
const lease = await connect({ deadlineAt: Date.now() + 1000, signal: aborted.signal })
|
|
aborted.abort(); lease.release(); assert.deepEqual(f.releases, [true])
|
|
const freshLease = await connect({ deadlineAt: Date.now() + 1000, signal: new AbortController().signal })
|
|
await freshLease.query("SELECT 1"); freshLease.release()
|
|
assert.deepEqual(fresh.releases, [false]); assert.equal(checkouts, 2); assert.equal(f.queries.length, 0)
|
|
})
|
|
|
|
test("idle initial deadline destroys without waiting for a query or explicit release", async () => {
|
|
const f = fixture(), lease = await f.connect({ deadlineAt: Date.now() + 25 })
|
|
await new Promise<void>(resolve => setTimeout(resolve, 40))
|
|
assert.deepEqual(f.releases, [true]); await assert.rejects(lease.query("SELECT 1"), refused)
|
|
lease.release(); assert.deepEqual(f.releases, [true]); assert.equal(f.queries.length, 0)
|
|
})
|
|
|
|
test("an abandoned late client's throwing release is absorbed and attempted once", async () => {
|
|
const f = fixture(), late = deferred<ProbeStoragePgClient>(), controller = new AbortController()
|
|
let releases = 0
|
|
const client: ProbeStoragePgClient = { ...f.client, release() { releases++; throw Error("NEVER_PRIVATE_RELEASE_DIAGNOSTIC") } }
|
|
const connect = createProbeStorageConnectionConnector(() => late.promise), pending = connect({ deadlineAt: Date.now() + 1000, signal: controller.signal })
|
|
controller.abort(); await assert.rejects(pending, refused)
|
|
late.resolve(client); await tick(); assert.equal(releases, 1); assert.equal(f.queries.length, 0)
|
|
})
|
|
|
|
const diagnosticId = "11111111-1111-4111-8111-111111111111"
|
|
async function guardFailure(work: Promise<unknown>, branch: string, kind = "other", phase = "query") {
|
|
let caught: unknown
|
|
try { await work; assert.fail("Expected guard failure") } catch (error) { caught = error }
|
|
assert(caught instanceof ProbeStorageUnknown || caught instanceof ProbeStorageRefused)
|
|
assert.deepEqual(capturePollError(caught).probe, { phase, queryKind: kind, branch })
|
|
assert(!JSON.stringify(capturePollError(caught)).includes("NEVER_PRIVATE"))
|
|
return caught
|
|
}
|
|
test("guard provenance: plain driver rejection differs from synthesized Unknown and exact kinds are diagnostic only", async () => {
|
|
const meta = "SELECT o.id FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_operation_steps s ON s.operation_id=o.id WHERE o.connection_id=$1 AND p.operation_id=$2 AND s.step_index=0 AND s.intended_state=$3::jsonb LIMIT 2"
|
|
for (const [text, kind] of [[principalSourceSql,"principal"],[meta,"meta"],["/* private */ END WORK;","commit"],[meta+" ","other"],["SELECT 'NEVER_PRIVATE SQL/password'","other"]]) {
|
|
const f=fixture(async()=>{throw Error("NEVER_PRIVATE driver timeout syntax error 57014")}),lease=await f.connect({deadlineAt:Date.now()+1000})
|
|
const error=await guardFailure(lease.query(text,["NEVER_PRIVATE credential"]),"driver-reject",kind)
|
|
assert.equal(capturePollError(error).origin.kind,"other");assert.equal(capturePollError(error).origin.sqlState,null)
|
|
let final=error as Error;for(const site of ["storage","journal.tx","poll.safe"] as const)final=carryPollError(new Error("public"),final,site)
|
|
const log=pollErrorLog(final,diagnosticId,"servers.poll")!
|
|
assert.deepEqual(log.provenance.sites,["probe.query","storage","journal.tx","poll.safe"])
|
|
assert.equal(log.provenance.probe?.branch,"driver-reject");assert(Buffer.byteLength(JSON.stringify(log))<=1024)
|
|
assert(!JSON.stringify(log).includes("NEVER_PRIVATE"));assert.equal(pollErrorLog(final,diagnosticId,"other.operation"),null)
|
|
assert.equal(f.queries.length,1);assert.deepEqual(f.releases,[true]);lease.release()
|
|
}
|
|
const pg=new DatabaseError("NEVER_PRIVATE real pg class",1,"error");pg.code="57014"
|
|
const driver=fixture(async()=>{throw pg}),lease=await driver.connect({deadlineAt:Date.now()+1000})
|
|
const error=await guardFailure(lease.query(principalSourceSql),"driver-reject","principal")
|
|
assert.equal(capturePollError(error).origin.kind,"pg");assert.equal(capturePollError(error).origin.sqlState,"57014")
|
|
})
|
|
test("guard provenance: synchronous physical query throw has its own branch", async () => {
|
|
let calls=0,releases=0
|
|
const lease=await createProbeStorageConnectionConnector(async()=>({query(){calls++;throw Error("NEVER_PRIVATE sync")},release(unsafe){assert.equal(unsafe,true);releases++}}))({deadlineAt:Date.now()+1000})
|
|
await guardFailure(lease.query("COMMIT"),"driver-throw","commit");assert.equal(calls,1);assert.equal(releases,1)
|
|
})
|
|
test("guard provenance: query wall timeout, caller abort and lease socket close remain distinct and late success cannot clear them", async () => {
|
|
for(const mode of ["deadline","abort","lease-error","lease-closed"] as const){
|
|
const driver=deferred<Result>(),controller=new AbortController(),f=fixture(()=>driver.promise),lease=await f.connect({deadlineAt:Date.now()+1000})
|
|
const pending=lease.query("SELECT 1",[],{timeoutMs:mode==="deadline"?20:1000,signal:controller.signal})
|
|
if(mode==="abort")controller.abort(Error("NEVER_PRIVATE abort"));if(mode==="lease-error")f.emitError();if(mode==="lease-closed")lease.release()
|
|
const error=await guardFailure(pending,mode);driver.resolve(result());await tick()
|
|
assert.equal(capturePollError(error).probe?.branch,mode);assert.equal(f.queries.length,1);assert.deepEqual(f.releases,[true]);lease.release()
|
|
}
|
|
})
|
|
test("guard provenance: initial lease deadline, result shape, result accessor and COMMIT ACK guard are synthetic, not driver rejection", async () => {
|
|
const hung=deferred<Result>(),expired=fixture(()=>hung.promise),old=await expired.connect({deadlineAt:Date.now()+20})
|
|
await guardFailure(old.query("SELECT 1",[],{timeoutMs:1000}),"lease-timeout");hung.resolve(result());await tick()
|
|
for(const [value,text,branch] of [[{rows:"NEVER_PRIVATE rows",command:"SELECT"},"SELECT 1","result-shape"],[null,"SELECT 1","result-access"],[result("ROLLBACK"),"COMMIT","commit-ack"]] as const){
|
|
const f=fixture(async()=>value as unknown as Result),lease=await f.connect({deadlineAt:Date.now()+1000})
|
|
await guardFailure(lease.query(text),branch,text==="COMMIT"?"commit":"other");assert.deepEqual(f.releases,[true]);assert.equal(f.queries.length,1)
|
|
}
|
|
})
|
|
test("guard provenance: post-resolution closed and inactive checks retain their precise branch", async () => {
|
|
for(const mode of ["result-closed","result-inactive"] as const){
|
|
const realNow=Date.now,start=realNow(),socket=new EventEmitter();let clock=start,releases=0
|
|
if(mode==="result-inactive")Date.now=()=>clock
|
|
try{
|
|
const lease=await createProbeStorageConnectionConnector(async()=>({on:socket.on.bind(socket),removeListener:socket.removeListener.bind(socket),query(){return Promise.resolve(result()).then(value=>{queueMicrotask(()=>queueMicrotask(()=>{if(mode==="result-closed")socket.emit("error",Error("NEVER_PRIVATE socket"));else clock=start+2000}));return value})},release(unsafe){assert.equal(unsafe,true);releases++}}))({deadlineAt:start+1000})
|
|
await guardFailure(lease.query("SELECT 1"),mode);assert.equal(releases,1)
|
|
}finally{Date.now=realNow}
|
|
}
|
|
})
|
|
test("guard provenance: checkout rejected/thrown/aborted/timed out never acquires query authority", async () => {
|
|
for(const mode of ["driver-reject","driver-throw","abort","deadline"] as const){
|
|
const controller=new AbortController(),later=deferred<ProbeStoragePgClient>(),f=fixture();let calls=0
|
|
const checkout:()=>Promise<ProbeStoragePgClient>=mode==="driver-throw"?()=>{calls++;throw Error("NEVER_PRIVATE checkout")}:mode==="driver-reject"?async()=>{calls++;throw Error("NEVER_PRIVATE checkout")}:()=>{calls++;return later.promise}
|
|
const pending=createProbeStorageConnectionConnector(checkout)({deadlineAt:Date.now()+(mode==="deadline"?20:1000),signal:controller.signal})
|
|
if(mode==="abort")controller.abort();await guardFailure(pending,mode,"other","checkout")
|
|
later.resolve(f.client);await tick();assert.equal(calls,1);assert.equal(f.queries.length,0)
|
|
}
|
|
})
|
|
test("guard provenance: pre-dispatch guards preserve Refused and query counts", async () => {
|
|
const f=fixture(),lease=await f.connect({deadlineAt:Date.now()+1000})
|
|
await guardFailure(lease.query("",[],{timeoutMs:1}),"input");assert.equal(f.queries.length,0)
|
|
const aborted=new AbortController();aborted.abort();await guardFailure(lease.query("SELECT 1",[],{timeoutMs:1,signal:aborted.signal}),"inactive")
|
|
await guardFailure(lease.query("SELECT 1"),"closed");assert.equal(f.queries.length,0);assert.deepEqual(f.releases,[true])
|
|
const waiting=deferred<Result>(),busy=fixture(()=>waiting.promise),borrow=await busy.connect({deadlineAt:Date.now()+1000}),first=borrow.query("SELECT 1")
|
|
const firstCheck=guardFailure(first,"lease-closed");await guardFailure(borrow.query("SELECT 2"),"busy");await firstCheck;waiting.resolve(result());await tick();assert.equal(busy.queries.length,1)
|
|
})
|
|
test("guard provenance: successful result/reference, named principal and public error fields stay unchanged", async () => {
|
|
const value=result("COMMIT"),f=fixture(async()=>value),lease=await f.connect({deadlineAt:Date.now()+1000})
|
|
assert.equal(await lease.query("COMMIT"),value);lease.release();assert.deepEqual(f.releases,[false])
|
|
const plain=new ProbeStorageUnknown(),before=Object.getOwnPropertyDescriptors(plain)
|
|
carryPollError(plain,new Error("NEVER_PRIVATE"),"probe.query");assert.deepEqual(Object.getOwnPropertyDescriptors(plain),before)
|
|
})
|