210 lines
16 KiB
TypeScript
210 lines
16 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { createHash, createHmac, randomUUID } from "node:crypto"
|
|
import { test } from "node:test"
|
|
import { gzipSync } from "node:zlib"
|
|
import { fingerprint } from "../modules/mcp/security.js"
|
|
import { collectExpiredServerPollStorageForTests } from "./server-poll-storage-retention.js"
|
|
import { PollStorageRefused, PollStorageUnknown } from "./server-poll-storage.js"
|
|
import type { ProbeStorageOptions } from "./server-probe-journal-maintenance.js"
|
|
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
|
|
|
const tables = ["server_poll_acks", "server_poll_intents", "server_poll_receipts", "server_poll_quarantines", "server_poll_executions", "server_poll_archives", "server_poll_storage_admissions"]
|
|
const deferred = () => { let resolve!: () => void; const promise = new Promise<void>(done => { resolve = done }); return { promise, resolve } }
|
|
function fixture(openRetired?: Parameters<typeof collectExpiredServerPollStorageForTests>[1]) {
|
|
const now = Date.now(), candidates: Record<string, unknown>[] = [{ receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: null },
|
|
{ receipt_id: randomUUID(), owner_id: "owned-gc", archive_version: 1 }]
|
|
const retiredRows: Record<string, unknown>[] = []
|
|
const commands: { text: string; values: unknown[]; timeout: number }[] = [], releases: boolean[] = [], options: ProbeStorageOptions[] = []
|
|
let settings: Record<string, unknown>[] = [{ requests_retention_days: 30 }], fail = "", residual = "", removed: Record<string, unknown>[] | undefined, hold: ReturnType<typeof deferred> | undefined,writer=true
|
|
const entered = deferred()
|
|
const run = () => collectExpiredServerPollStorageForTests({ now: () => now, connect: async accepted => {
|
|
options.push(accepted)
|
|
return { async query(text, values = [], queryOptions) {
|
|
commands.push({ text, values, timeout: queryOptions!.timeoutMs })
|
|
if (text === fail) throw Error("PRIVATE_DRIVER_SECRET")
|
|
if (text.includes("pg_try_advisory_xact_lock_shared")) return { rows: [{ acquired:writer }] }
|
|
if (text.includes("current_setting('lock_timeout')")) return { rows: [{ lock_timeout: "0" }] }
|
|
// GC transaction/ordering fixture only. Authentic compact MAC and complete
|
|
// counter conservation are covered by archive-accounting component tests.
|
|
if (text.startsWith("SELECT a.receipt_id AS admission_receipt_id") || text.startsWith("SELECT r.receipt_id FROM server_poll_archives")) return { rows: [] }
|
|
if (text.startsWith("SELECT scope,admissions,compacted_admissions,archive_bytes,reserved_rows,reserved_bytes"))
|
|
return { rows: [{ scope: "global", admissions: 0, compacted_admissions: 0, archive_bytes: 0, reserved_rows: 0, reserved_bytes: 0 }] }
|
|
if (text.includes("requests_retention_days")) { entered.resolve(); if (hold) await hold.promise; return { rows: settings } }
|
|
if (text.startsWith("SELECT receipt_id,owner_id")) return { rows: structuredClone(candidates) }
|
|
if (text.startsWith("SELECT r.version,r.receipt_id")) return { rows: structuredClone(retiredRows) }
|
|
if (text.startsWith("DELETE FROM server_poll_storage_admissions")) return { rows: removed ?? candidates.map(row => ({ receipt_id: row.receipt_id })) }
|
|
if (text.startsWith("SELECT receipt_id FROM ") && text.includes("FROM " + residual + " ")) return { rows: [{ receipt_id: candidates[0]?.receipt_id }] }
|
|
return { rows: [] }
|
|
}, release(unsafe = false) { releases.push(unsafe) } }
|
|
} }, openRetired)
|
|
return { run, now, candidates, retiredRows, commands, releases, options, entered: entered.promise, block: () => { hold = deferred(); return hold },setWriter:(value:boolean)=>{writer=value},
|
|
setting: (rows: Record<string, unknown>[]) => { settings = rows }, fail: (text: string) => { fail = text }, residual: (table: string) => { residual = table }, removed: (rows: Record<string, unknown>[]) => { removed = rows } }
|
|
}
|
|
|
|
test("only a definite shared xact lock miss defers GC before POLL storage or a retention read",async()=>{
|
|
const fixtureBlocked=fixture();fixtureBlocked.setWriter(false)
|
|
assert.deepEqual(await fixtureBlocked.run(),{collected:0,automaticReplay:false})
|
|
const lock="SELECT pg_catalog.pg_try_advisory_xact_lock_shared($1::bigint) AS acquired"
|
|
assert.deepEqual(fixtureBlocked.commands.map(command=>command.text),["BEGIN",lock,"COMMIT"])
|
|
const unexpected=fixture();unexpected.fail(lock)
|
|
await assert.rejects(unexpected.run(),PollStorageRefused)
|
|
assert.deepEqual(unexpected.releases,[true])
|
|
})
|
|
|
|
test("selected GC owns one bounded backend, locks current retention, deletes all children and verifies trigger refund outcome", async () => {
|
|
const f = fixture()
|
|
assert.deepEqual(await f.run(), { collected: 2, automaticReplay: false })
|
|
assert.equal(f.options.length, 1); assert.equal(f.options[0].deadlineAt, f.now + 20000)
|
|
assert(f.commands.every(command => command.timeout >= 1 && command.timeout <= 5000))
|
|
const texts = f.commands.map(command => command.text), setting = texts.findIndex(text => text.includes("requests_retention_days"))
|
|
const writer = texts.findIndex(text => text.includes("pg_advisory_xact_lock_shared(")), storage = texts.findIndex(text => text.includes("server-poll-storage-v1"))
|
|
assert(writer > 0 && writer < storage && storage < setting)
|
|
assert(texts[setting].endsWith("FOR SHARE")); assert(setting < texts.indexOf("SET LOCAL mm.poll_storage_gc='on'"))
|
|
const selection = texts.find(text => text.startsWith("SELECT receipt_id,owner_id"))!
|
|
assert(selection.includes("gc_not_before<clock_timestamp()")); assert(selection.includes("server_poll_storage_gc_allowed(receipt_id)")); assert(selection.endsWith("LIMIT 8 FOR UPDATE"))
|
|
const ids = f.candidates.map(row => row.receipt_id)
|
|
for (const table of tables) {
|
|
const deletion = f.commands.find(command => command.text.startsWith(`DELETE FROM ${table} WHERE`))!
|
|
assert.deepEqual(deletion.values, [ids])
|
|
assert(texts.some(text => text === `SELECT receipt_id FROM ${table} WHERE receipt_id=ANY($1::uuid[]) LIMIT 1`))
|
|
}
|
|
const refund = f.commands.find(command => command.text.startsWith("DELETE FROM server_poll_storage_usage"))!
|
|
assert.deepEqual(refund.values, [["owner:owned-gc"]]); assert(refund.text.includes("admissions=0 AND compacted_admissions=0 AND archive_bytes=0 AND reserved_rows=0 AND reserved_bytes=0"))
|
|
const archiveDelete = texts.findIndex(text => text.startsWith("DELETE FROM server_poll_archives"))
|
|
assert(archiveDelete > texts.findIndex(text => text.startsWith("DELETE FROM server_poll_executions")))
|
|
assert(archiveDelete < texts.findIndex(text => text.startsWith("DELETE FROM server_poll_storage_admissions")))
|
|
const accounting = texts.findIndex(text => text.startsWith("SELECT scope,admissions,compacted_admissions"))
|
|
assert(accounting > texts.indexOf(selection))
|
|
assert(accounting < texts.findIndex(text => text.startsWith("DELETE FROM")))
|
|
assert(texts.indexOf("COMMIT") > texts.findIndex(text => text.startsWith("SELECT receipt_id FROM server_poll_storage_admissions")))
|
|
assert.deepEqual(f.releases, [false])
|
|
})
|
|
|
|
async function retiredFixture() {
|
|
const { createServerPollRetiredArchiveForTests } = await import("./server-poll-retired-archive.js")
|
|
const proof = async (purpose: string, value: unknown) =>
|
|
createHmac("sha256", "gc-fixture-only").update(purpose).update(fingerprint(value)).digest("hex")
|
|
const sha = (value: string | Buffer) => createHash("sha256").update(value).digest("hex")
|
|
let opened = 0
|
|
const codec = createServerPollRetiredArchiveForTests({
|
|
proof,
|
|
decrypt: async value => { opened++; return value },
|
|
encrypt: async () => { throw new Error("Unexpected archive write") },
|
|
openV1: async () => { throw new Error("Unexpected archive fallback") },
|
|
verifyHistory: async () => { throw new Error("Unexpected historical fallback") },
|
|
})
|
|
const f = fixture(codec.open)
|
|
const selected = f.candidates[1]
|
|
selected.archive_version = 2
|
|
const timestamp = "2026-10-01T00:00:00.123456Z", gcNotBefore = "2026-11-01T00:00:00.123456Z"
|
|
const history = { version: 2, operation: "servers.poll", operationVersion: 1, planId: randomUUID(), serverId: 3,
|
|
affectedServerIds: [3], gcNotBefore, terminalAt: timestamp,
|
|
previous: { archiveProof: "1".repeat(64), sourceDigest: "2".repeat(64), sourceSha256: "3".repeat(64) },
|
|
history: { createdAt: timestamp, completedAt: timestamp, stageCount: 12, publications: 8,
|
|
nativeReads: 4, observation: "online", verification: "passed" } }
|
|
const plaintext = Buffer.from(JSON.stringify(history)), compressed = gzipSync(plaintext), ciphertext = compressed.toString("base64")
|
|
const metadata = { version: 2, receipt_id: selected.receipt_id, execution_id: randomUUID(), connection_id: randomUUID(), owner_id: selected.owner_id,
|
|
binding_proof: "a".repeat(64), context_proof: "b".repeat(64), terminal_digest: "c".repeat(64),
|
|
original_activity_at: timestamp, original_expires_at: timestamp, retention_days: 30, safety_ms: 60000,
|
|
source_digest: await proof("server-poll-retired-archive-source-v2", sha(plaintext)), source_sha256: sha(plaintext),
|
|
decoded_bytes: plaintext.length, compressed_bytes: compressed.length, ciphertext_bytes: ciphertext.length, ciphertext_sha256: sha(ciphertext) }
|
|
const archiveProof = await proof("server-poll-retired-archive-record-v2", { ...metadata, identity_kind: "mcp",
|
|
storage_state: "compacted", admission_bytes: 32768, charged_rows: 2 })
|
|
f.retiredRows.push({ ...metadata, ciphertext, archive_proof: archiveProof, admission_gc_not_before: gcNotBefore })
|
|
return { ...f, opened: () => opened }
|
|
}
|
|
|
|
test("GC opens only selected V2 under a row lock and authenticates the original lifetime before any DELETE", async () => {
|
|
const f = await retiredFixture()
|
|
assert.deepEqual(await f.run(), { collected: 2, automaticReplay: false })
|
|
assert.equal(f.opened(), 1)
|
|
const index = f.commands.findIndex(command => command.text.startsWith("SELECT r.version,r.receipt_id"))
|
|
assert.deepEqual(f.commands[index].values, [f.candidates[1].receipt_id])
|
|
assert(f.commands[index].text.endsWith("FOR UPDATE OF r"))
|
|
assert(index < f.commands.findIndex(command => command.text.startsWith("DELETE")))
|
|
})
|
|
|
|
test("missing V2, forged ciphertext or proof, and microsecond lifetime drift refuse before any DELETE", async () => {
|
|
for (const fault of ["missing", "ciphertext", "actual-hash", "proof", "lifetime", "expiry", "version"] as const) {
|
|
const f = await retiredFixture(), row = f.retiredRows[0]
|
|
if (fault === "missing") f.retiredRows.length = 0
|
|
if (fault === "ciphertext") row.ciphertext = "A" + String(row.ciphertext).slice(1)
|
|
if (fault === "actual-hash") row.ciphertext_sha256 = "a".repeat(64)
|
|
if (fault === "proof") row.archive_proof = "a".repeat(64)
|
|
if (fault === "lifetime") row.admission_gc_not_before = "2026-11-01T00:00:00.123455Z"
|
|
if (fault === "expiry") row.original_expires_at = "2026-10-01T00:00:00.123455Z"
|
|
if (fault === "version") f.candidates[1].archive_version = 3
|
|
await assert.rejects(f.run(), PollStorageRefused)
|
|
assert.equal(f.opened(), fault === "lifetime" ? 1 : 0)
|
|
assert(!f.commands.some(command => command.text.startsWith("DELETE") || command.text === "COMMIT"))
|
|
assert.equal(f.commands.filter(command => command.text === "ROLLBACK").length, 1)
|
|
}
|
|
})
|
|
|
|
test("empty candidate set returns zero without deleting or refunding", async () => {
|
|
const f = fixture(); f.candidates.length = 0
|
|
assert.deepEqual(await f.run(), { collected: 0, automaticReplay: false })
|
|
assert(!f.commands.some(command => command.text.includes("pg_catalog.sha256") || command.text.startsWith("SELECT scope,admissions,compacted_admissions")),
|
|
"An idle GC tick must not hash or hydrate the retained history")
|
|
assert(!f.commands.some(command => command.text.startsWith("DELETE"))); assert.deepEqual(f.releases, [false])
|
|
})
|
|
|
|
test("missing, duplicate or invalid authoritative retention never enables GC or deletes", async () => {
|
|
for (const rows of [[], [{ requests_retention_days: 0 }], [{ requests_retention_days: 366 }], [{ requests_retention_days: "30" }], [{ requests_retention_days: 1.5 }], [{ requests_retention_days: 30 }, { requests_retention_days: 30 }]]) {
|
|
const f = fixture(); f.setting(rows)
|
|
await assert.rejects(f.run(), PollStorageRefused)
|
|
assert(!f.commands.some(command => /^(?:DELETE|SET LOCAL mm\.poll_storage_gc)/.test(command.text)))
|
|
assert.equal(f.commands.filter(command => command.text === "ROLLBACK").length, 1); assert.deepEqual(f.releases, [true])
|
|
}
|
|
})
|
|
|
|
test("candidate overflow, duplicate or malformed identity fails before deletion", async () => {
|
|
for (const change of [(f: ReturnType<typeof fixture>) => { while (f.candidates.length < 9) f.candidates.push({ receipt_id: randomUUID(), owner_id: "owned-gc" }) },
|
|
(f: ReturnType<typeof fixture>) => { f.candidates[1] = { ...f.candidates[0] } },
|
|
(f: ReturnType<typeof fixture>) => { f.candidates[0].receipt_id = "not-uuid" }]) {
|
|
const f = fixture(); change(f); await assert.rejects(f.run(), PollStorageRefused)
|
|
assert(!f.commands.some(command => command.text.startsWith("DELETE"))); assert.deepEqual(f.releases, [true])
|
|
}
|
|
})
|
|
|
|
test("missing admission refund or residual journal row prevents COMMIT", async () => {
|
|
for (const corrupt of [(f: ReturnType<typeof fixture>) => { f.removed([]) }, (f: ReturnType<typeof fixture>) => { f.residual("server_poll_receipts") },
|
|
(f: ReturnType<typeof fixture>) => { f.residual("server_poll_archives") }]) {
|
|
const f = fixture(); corrupt(f); await assert.rejects(f.run(), PollStorageRefused)
|
|
assert(!f.commands.some(command => command.text === "COMMIT")); assert(f.commands.some(command => command.text === "ROLLBACK")); assert.deepEqual(f.releases, [true])
|
|
}
|
|
})
|
|
|
|
test("lost GC COMMIT acknowledgement destroys once with no rollback, retry or provider leak", async () => {
|
|
const f = fixture(); f.fail("COMMIT")
|
|
await assert.rejects(f.run(), error => error instanceof PollStorageUnknown && !error.message.includes("PRIVATE_DRIVER_SECRET"))
|
|
assert.equal(f.options.length, 1); assert.equal(f.commands.filter(command => command.text === "COMMIT").length, 1)
|
|
assert(!f.commands.some(command => command.text === "ROLLBACK")); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("SQL failure rolls back once and destroys the sole backend", async () => {
|
|
const f = fixture(); f.fail("DELETE FROM server_poll_intents WHERE receipt_id=ANY($1::uuid[])")
|
|
await assert.rejects(f.run(), error => error instanceof PollStorageRefused && !error.message.includes("PRIVATE_DRIVER_SECRET"))
|
|
assert.equal(f.commands.filter(command => command.text === "ROLLBACK").length, 1)
|
|
assert(!f.commands.some(command => command.text === "COMMIT")); assert.equal(f.options.length, 1); assert.deepEqual(f.releases, [true])
|
|
})
|
|
|
|
test("shared database replacement waits for the full active GC lease and refuses a new GC while paused", async () => {
|
|
const f = fixture(), resume = f.block(); const pending = f.run(); await f.entered
|
|
let replaced = false
|
|
const replacement = sharedDatabaseGate.exclusive(async () => { replaced = true; assert.deepEqual(f.releases, [false]) })
|
|
await Promise.resolve(); assert.equal(replaced, false)
|
|
const other = fixture(); await assert.rejects(other.run(), error => (error as { statusCode?: number }).statusCode === 409); assert.equal(other.options.length, 0)
|
|
resume.resolve(); await pending; await replacement; assert.equal(replaced, true)
|
|
})
|
|
|
|
test("simultaneous GC calls each retain their bounded owned transaction and drain before replacement", async () => {
|
|
const a = fixture(), b = fixture(), resumeA = a.block(), resumeB = b.block()
|
|
const first = a.run(), second = b.run(); await Promise.all([a.entered, b.entered])
|
|
let replaced = false; const replacement = sharedDatabaseGate.exclusive(async () => { replaced = true })
|
|
resumeA.resolve(); await first; assert.equal(replaced, false)
|
|
resumeB.resolve(); await second; await replacement
|
|
assert.equal(replaced, true)
|
|
for (const f of [a, b]) { assert.equal(f.options.length, 1); assert.equal(f.options[0].deadlineAt, f.now + 20000); assert.deepEqual(f.releases, [false]) }
|
|
})
|