463 lines
36 KiB
TypeScript
463 lines
36 KiB
TypeScript
import { MIGRATION_MANIFEST } from "../db/migration-manifest.js"
|
|
import assert from "node:assert/strict"
|
|
import { fork, execFile } from "node:child_process"
|
|
import { once } from "node:events"
|
|
import { createHash, randomUUID } from "node:crypto"
|
|
import { createServer, request } from "node:http"
|
|
import { readFile, writeFile } from "node:fs/promises"
|
|
import { resolve } from "node:path"
|
|
import { fileURLToPath } from "node:url"
|
|
import { promisify } from "node:util"
|
|
import type { BandwidthBinding } from "./bandwidth-admission.js"
|
|
import { assertOwnedBandwidthSql, canonical, completionProof, createOwnedBandwidthHarness, fixtureBinding, fixtureMac, prepareOwnedBandwidthState, scope } from "./bandwidth-admission-process.sql-helper.js"
|
|
|
|
assertOwnedBandwidthSql()
|
|
await prepareOwnedBandwidthState()
|
|
const { pool, closePool } = await import("../db/index.js")
|
|
const { applySqlMigrations } = await import("../db/migrate.js")
|
|
const { BANDWIDTH_ADMISSION_PROPOSAL_DDL } = await import("./bandwidth-admission-schema.js")
|
|
pool.on("error", () => { /* An owned test deliberately terminates a released adapter backend. */ })
|
|
const cases: string[] = [], limitations: string[] = []
|
|
const children: ReturnType<typeof fork>[] = []
|
|
const tables = ["bandwidth_attempts", "bandwidth_node_occupancy", "bandwidth_attempt_events"] as const
|
|
const hash = (value: unknown) => createHash("sha256").update(canonical(value)).digest("hex")
|
|
async function snapshot(query = pool) {
|
|
const result: Record<string, { rows: number; sha256: string }> = {}
|
|
for (const name of tables) {
|
|
const values = (await query.query(`SELECT to_jsonb(t) AS row FROM mmapp_control.${name} t ORDER BY to_jsonb(t)::text`)).rows.map(row => row.row)
|
|
result[name] = { rows: values.length, sha256: hash(values) }
|
|
}
|
|
return result
|
|
}
|
|
const occupancy = async (binding: BandwidthBinding) => (await pool.query("SELECT node_id,attempt_id FROM mmapp_control.bandwidth_node_occupancy WHERE node_id=ANY($1::text[]) ORDER BY node_id", [[binding.source.canonicalNodeId, binding.destination.canonicalNodeId]])).rows
|
|
const fresh = (prefix: string) => fixtureBinding(prefix + ":A", prefix + ":B")
|
|
const barrier = () => { let release!: () => void; const reached = new Promise<void>(resolve => { release = resolve }); return { reached, release } }
|
|
async function child() {
|
|
const processChild = fork(fileURLToPath(new URL("./bandwidth-admission-process.sql-helper.ts", import.meta.url)), [], {
|
|
execArgv: ["--import", "tsx"], stdio: ["ignore", "inherit", "inherit", "ipc"], env: { ...process.env, MMAPP_BANDWIDTH_CHILD: "true" },
|
|
})
|
|
children.push(processChild)
|
|
await Promise.race([once(processChild, "message"), once(processChild, "exit").then(() => { throw Error("Owned bandwidth child exited before ready") })])
|
|
let id = 0
|
|
return async (command: string, binding?: BandwidthBinding): Promise<{ ok: boolean; pid?: number; processId?: number }> => {
|
|
const expected = ++id
|
|
return new Promise((resolve, reject) => {
|
|
const timeout = setTimeout(() => { cleanup(); reject(Error("Owned IPC deadline")) }, 15000)
|
|
const cleanup = () => { clearTimeout(timeout); processChild.off("message", response) }
|
|
const response = (value: unknown) => { const result = value as { id: number; ok: boolean }; if (result.id === expected) { cleanup(); resolve(result) } }
|
|
processChild.on("message", response)
|
|
processChild.send({ id: expected, command, binding })
|
|
})
|
|
}
|
|
}
|
|
async function verifyEvents() {
|
|
const rows = (await pool.query(`SELECT e.*,a.terminal_proof FROM mmapp_control.bandwidth_attempt_events e JOIN mmapp_control.bandwidth_attempts a USING(attempt_id) ORDER BY e.ack_id`)).rows
|
|
for (const event of rows) assert.equal(event.evidence_proof, fixtureMac({ ackId: event.ack_id, attemptId: event.attempt_id,
|
|
transition: event.transition, oldState: event.old_state, newState: event.new_state, bindingProof: event.binding_proof,
|
|
controlGeneration: event.control_generation, terminalProof: event.transition === "complete" ? event.terminal_proof : null }))
|
|
return rows.length
|
|
}
|
|
async function verifyGuardMetadata(query = pool) {
|
|
const triggers = (await query.query(`SELECT t.tgname,t.tgenabled::text AS enabled FROM pg_catalog.pg_trigger t
|
|
JOIN pg_catalog.pg_class c ON c.oid=t.tgrelid JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace
|
|
WHERE n.nspname='mmapp_control' AND c.relname=ANY($1::text[]) AND NOT t.tgisinternal ORDER BY t.tgname`, [[...tables]])).rows
|
|
assert.equal(triggers.length, 9); assert(triggers.every(row => row.enabled === "A"))
|
|
const functions = (await query.query(`SELECT p.proname,p.proconfig FROM pg_catalog.pg_proc p
|
|
JOIN pg_catalog.pg_namespace n ON n.oid=p.pronamespace
|
|
WHERE n.nspname='mmapp_control' AND left(p.proname,length('bandwidth_admission_'))='bandwidth_admission_' ORDER BY p.proname`)).rows
|
|
assert.equal(functions.length, 6)
|
|
for (const fn of functions) assert((fn.proconfig as string[] | null)?.some(value => value.replaceAll(" ", "") === "search_path=pg_catalog,mmapp_control"))
|
|
return { triggers, functions }
|
|
}
|
|
try {
|
|
await applySqlMigrations(pool)
|
|
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM schema_migrations")).rows[0].n), MIGRATION_MANIFEST.length)
|
|
// Externally provisioned namespace in this disposable fixture only. Neither
|
|
// normal migrations nor the journal factory may create/enroll it.
|
|
await pool.query("CREATE SCHEMA mmapp_control")
|
|
await pool.query(BANDWIDTH_ADMISSION_PROPOSAL_DDL)
|
|
assert.equal(Number((await pool.query("SELECT count(*) AS n FROM schema_migrations")).rows[0].n), MIGRATION_MANIFEST.length)
|
|
const h = await createOwnedBandwidthHarness()
|
|
const reserve = async (binding: BandwidthBinding) => { h.register(binding); return h.journal.reserve(binding, scope()) }
|
|
const complete = async (binding: BandwidthBinding) => h.journal.complete(binding, { terminalProof: completionProof(binding) }, scope())
|
|
{
|
|
const b = fresh("rollback")
|
|
h.register(b); h.fault("rollback-before-commit")
|
|
await assert.rejects(h.journal.reserve(b, scope()))
|
|
assert.equal((await occupancy(b)).length, 0)
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].n, 0)
|
|
cases.push("reserve rollback before real COMMIT leaves neither half-pair nor attempt")
|
|
}
|
|
{
|
|
const one = await child(), two = await child(), b = fresh("process")
|
|
const first = await one("reserve", b); assert(first.ok)
|
|
const overlap = fixtureBinding(b.destination.canonicalNodeId, "process:C")
|
|
assert.equal((await two("reserve", overlap)).ok, false)
|
|
assert.equal((await two("reserve", fixtureBinding(b.destination.canonicalNodeId, b.source.canonicalNodeId))).ok, false)
|
|
const independent = fixtureBinding("process:C", "process:D"), parallel = await two("reserve", independent)
|
|
assert(parallel.ok); assert.notEqual(first.processId, parallel.processId); assert.notEqual(first.pid, parallel.pid)
|
|
assert.equal((await occupancy(b)).length, 2); assert.equal((await occupancy(independent)).length, 2)
|
|
assert.equal((await two("reserve", b)).ok, false)
|
|
assert((await one("arm", b)).ok); assert((await one("complete", b)).ok)
|
|
await one("close"); await two("close")
|
|
cases.push("two independent processes/backend PIDs: A-B blocks B-C/reversed B-A; disjoint C-D succeeds; repeat has no new ACK")
|
|
}
|
|
{
|
|
const a = await createOwnedBandwidthHarness(), b = await createOwnedBandwidthHarness()
|
|
const left = fresh("race"), right = fixtureBinding(left.destination.canonicalNodeId, "race:C")
|
|
a.register(left); b.register(right)
|
|
const results = await Promise.allSettled([a.journal.reserve(left, scope()), b.journal.reserve(right, scope())])
|
|
assert.equal(results.filter(row => row.status === "fulfilled").length, 1)
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_node_occupancy WHERE node_id LIKE 'race:%'")).rows[0].n, 2)
|
|
cases.push("concurrent independent connections reserve intersecting pairs atomically: exactly one winner")
|
|
}
|
|
{
|
|
const b = fresh("reserve-ack")
|
|
h.register(b); h.fault("lost-commit")
|
|
await assert.rejects(h.journal.reserve(b, scope()))
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
const observer = await child(); assert.equal((await observer("reserve", fixtureBinding(b.source.canonicalNodeId, "reserve-ack:C"))).ok, false); await observer("close")
|
|
cases.push("actual reserve COMMIT then lost ACK retains pair; independent process cannot reclaim")
|
|
}
|
|
{
|
|
const b = fresh("arm-ack"); await reserve(b); h.fault("lost-commit")
|
|
await assert.rejects(h.journal.arm(b, scope()))
|
|
assert.equal((await pool.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].state, "armed")
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
const restart = await createOwnedBandwidthHarness(); restart.register(b)
|
|
await assert.rejects(restart.journal.reserve(b, scope()))
|
|
cases.push("actual arm COMMIT then lost ACK: persisted armed pair survives fresh store and has no repeat reserve ACK")
|
|
}
|
|
{
|
|
const b = fresh("terminate"), received = barrier(), endRemote = barrier()
|
|
const server = createServer((_req, res) => { received.release(); void endRemote.reached.then(() => { if (!res.destroyed) res.end(JSON.stringify([{ status: "done testing" }])) }) })
|
|
server.listen(0, "127.0.0.1"); server.unref(); await once(server, "listening")
|
|
const address = server.address(); assert(address && typeof address !== "string")
|
|
await reserve(b); await h.journal.arm(b, scope())
|
|
const armPid = h.lastPid()
|
|
const req = request({ host: "127.0.0.1", port: address.port, path: "/synthetic-bandwidth", method: "POST" })
|
|
req.on("error", () => {}); req.on("socket", socket => socket.unref()); req.end(); await received.reached
|
|
const { Client } = await import("pg")
|
|
const observer = new Client({ connectionString: process.env.DATABASE_URL }); await observer.connect()
|
|
try {
|
|
assert.notEqual(Number((await observer.query("SELECT pg_backend_pid() AS pid")).rows[0].pid), armPid)
|
|
assert.equal((await observer.query("SELECT pg_terminate_backend($1) AS killed", [armPid])).rows[0].killed, true)
|
|
} finally { await observer.end() }
|
|
const restart = await child(); assert.equal((await restart("reserve", fixtureBinding(b.destination.canonicalNodeId, "terminate:C"))).ok, false); await restart("close")
|
|
req.destroy(); endRemote.release(); await new Promise<void>(resolve => server.close(() => resolve()))
|
|
await h.journal.quarantine(b, scope()); assert.equal((await occupancy(b)).length, 2)
|
|
cases.push("real pg_terminate_backend of committed arm backend AFTER loopback dispatch, while remote held running: new process remains blocked; socket close/quarantine retains pair")
|
|
}
|
|
for (const mode of ["lost-commit", "rollback-before-commit"] as const) {
|
|
const b = fresh("quarantine-" + mode); await reserve(b); await h.journal.arm(b, scope()); h.fault(mode)
|
|
await assert.rejects(h.journal.quarantine(b, scope())); assert.equal((await occupancy(b)).length, 2)
|
|
cases.push("quarantine " + mode + " never clears armed occupancy")
|
|
}
|
|
{
|
|
const b = fresh("complete-ack"); await reserve(b); await h.journal.arm(b, scope()); h.fault("lost-commit")
|
|
await assert.rejects(complete(b)); assert.equal((await occupancy(b)).length, 0)
|
|
assert.equal((await pool.query("SELECT resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].resolution, "remote_completion")
|
|
const next = fixtureBinding(b.source.canonicalNodeId, b.destination.canonicalNodeId); await reserve(next)
|
|
const before = await snapshot()
|
|
await assert.rejects(h.journal.quarantine(b, scope())); await assert.rejects(complete(b))
|
|
h.markNoSend(b); await assert.rejects(h.journal.releaseBeforeDispatch(b, { from: "armed", proof: "private-no-send" }, scope()))
|
|
assert.deepEqual(await snapshot(), before)
|
|
cases.push("real completion COMMIT ACK loss releases both only with terminal proof; stale complete/quarantine/no-send never clears newer owner")
|
|
}
|
|
{
|
|
const b = fresh("proof"); await reserve(b); await h.journal.arm(b, scope())
|
|
await assert.rejects(h.journal.complete(b, { terminalProof: "f".repeat(64) }, scope()))
|
|
await assert.rejects(h.journal.releaseBeforeDispatch(b, { from: "armed", proof: "private-no-send" }, scope()))
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
const changed = fixtureBinding(b.source.canonicalNodeId, "proof:C", { source: { ...b.source, endpointProof: fixtureMac("changed epoch") } })
|
|
h.register(changed); await assert.rejects(h.journal.reserve(changed, scope()))
|
|
cases.push("forged terminal/no-send evidence refused; changed endpoint epoch does not free canonical node")
|
|
}
|
|
{
|
|
const b = fresh("evidence-bad-first"); await reserve(b); await h.journal.arm(b, scope())
|
|
const evidence = { terminalProof: "f".repeat(64) }
|
|
const pending = h.journal.complete(b, evidence, scope())
|
|
evidence.terminalProof = completionProof(b)
|
|
await assert.rejects(pending)
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
const current = (await pool.query("SELECT state,terminal_proof FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0]
|
|
assert.equal(current.state, "armed"); assert.equal(current.terminal_proof, null)
|
|
cases.push("actual SQL mutable evidence TOCTOU: initially invalid completion changed to valid after call still refuses, preserving both occupied nodes and NULL terminal proof")
|
|
}
|
|
{
|
|
const b = fresh("evidence-valid-first"); await reserve(b); await h.journal.arm(b, scope())
|
|
const originalProof = completionProof(b), evidence = { terminalProof: originalProof }
|
|
const pending = h.journal.complete(b, evidence, scope())
|
|
evidence.terminalProof = "f".repeat(64)
|
|
await pending
|
|
assert.equal((await occupancy(b)).length, 0)
|
|
const current = (await pool.query("SELECT state,terminal_proof FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0]
|
|
assert.equal(current.state, "released"); assert.equal(current.terminal_proof, originalProof)
|
|
cases.push("actual SQL mutable evidence TOCTOU: initially valid completion changed to invalid after call commits only captured valid proof and releases exact pair")
|
|
}
|
|
{
|
|
const b = fresh("evidence-no-send"); await reserve(b); h.markNoSend(b)
|
|
const evidence = { from: "reserved" as const, proof: "private-no-send" as const }
|
|
const pending = h.journal.releaseBeforeDispatch(b, evidence, scope())
|
|
// Mutate caller memory after invocation without making the typed fixture
|
|
// provide a different contract at the call boundary.
|
|
Object.assign(evidence, { from: "armed", proof: "forged-after-call" })
|
|
await pending
|
|
assert.equal((await occupancy(b)).length, 0)
|
|
const current = (await pool.query("SELECT state,resolution,terminal_proof FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0]
|
|
assert.equal(current.state, "released"); assert.equal(current.resolution, "proven_not_dispatched"); assert.equal(current.terminal_proof, null)
|
|
const event = (await pool.query("SELECT old_state FROM mmapp_control.bandwidth_attempt_events WHERE attempt_id=$1 AND transition='release-before-dispatch'", [b.attemptId])).rows[0]
|
|
assert.equal(event.old_state, "reserved")
|
|
cases.push("actual SQL mutable no-send evidence TOCTOU: caller mutation after call cannot change captured reserved release, resolution or event source state")
|
|
}
|
|
{
|
|
const b = fresh("delete"), id = Number((await pool.query("INSERT INTO servers(name,host,enabled) VALUES('owned-bw-delete','127.0.0.1',false) RETURNING id")).rows[0].id)
|
|
await reserve(b); await pool.query("DELETE FROM servers WHERE id=$1", [id]); assert.equal((await occupancy(b)).length, 2)
|
|
await assert.rejects(pool.query("DELETE FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId]))
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
cases.push("server DELETE does not cascade into canonical occupancy; active attempt DELETE rejected and pair preserved")
|
|
}
|
|
{
|
|
const b = fresh("expiry"); await reserve(b); await h.journal.arm(b, scope()); await h.journal.quarantine(b, scope())
|
|
const newer = fixtureBinding(b.source.canonicalNodeId, "expiry:C", { controlGeneration: "new-owner-control", authorityGeneration: "new-credential" })
|
|
h.register(newer); await assert.rejects(h.journal.reserve(newer, scope())); assert.equal((await occupancy(b)).length, 2)
|
|
const expired = { ...b, expiresAt: Date.now() - 1 }; h.register(expired)
|
|
await assert.rejects(h.journal.arm(expired, scope()))
|
|
cases.push("quarantined canonical nodes cannot be reclaimed by new control/credential generation; expired altered binding cannot rearm")
|
|
}
|
|
{
|
|
const b = fresh("cancel"); await reserve(b); h.markNoSend(b)
|
|
const outcomes = await Promise.allSettled([h.journal.arm(b, scope()), h.journal.releaseBeforeDispatch(b, { from: "reserved", proof: "private-no-send" }, scope())])
|
|
assert.equal(outcomes.filter(value => value.status === "fulfilled").length, 1)
|
|
const row = (await pool.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0]
|
|
assert.equal((await occupancy(b)).length, row.state === "released" ? 0 : 2)
|
|
cases.push("same-instance local admission arm versus reserved cancel race: one local winner; not evidence of a database CAS race")
|
|
}
|
|
{
|
|
// Structural SQL fixture only: production evidence/private capability checks
|
|
// are not replaced by this direct UPDATE. Both branches supply valid event
|
|
// HMACs so the database transition/CAS/pair constraints are tested in full.
|
|
const b = fresh("physical-cas"); await reserve(b)
|
|
const { Client } = await import("pg")
|
|
const first = new Client({ connectionString: process.env.DATABASE_URL }), second = new Client({ connectionString: process.env.DATABASE_URL })
|
|
await first.connect(); await second.connect()
|
|
const start = barrier(), pids: number[] = []
|
|
let begun = 0
|
|
const compete = async (client: typeof first, release: boolean): Promise<boolean> => {
|
|
await client.query("BEGIN")
|
|
pids.push(Number((await client.query("SELECT pg_backend_pid() AS pid")).rows[0].pid))
|
|
if (++begun === 2) start.release()
|
|
await start.reached
|
|
try {
|
|
const next = release ? "released" : "armed", transition = release ? "release-before-dispatch" : "arm"
|
|
const changed = await client.query("UPDATE mmapp_control.bandwidth_attempts SET state=$2,resolution=$3 WHERE attempt_id=$1 AND state='reserved' RETURNING attempt_id", [b.attemptId, next, release ? "proven_not_dispatched" : null])
|
|
if (!changed.rows.length) { await client.query("ROLLBACK"); return false }
|
|
assert.equal(changed.rows.length, 1)
|
|
if (release) assert.equal((await client.query("DELETE FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1 RETURNING node_id", [b.attemptId])).rows.length, 2)
|
|
const ackId = randomUUID()
|
|
const evidence = fixtureMac({ ackId, attemptId: b.attemptId, transition, oldState: "reserved", newState: next,
|
|
bindingProof: b.bindingProof, controlGeneration: b.controlGeneration, terminalProof: null })
|
|
await client.query("INSERT INTO mmapp_control.bandwidth_attempt_events(ack_id,attempt_id,transition,old_state,new_state,binding_proof,control_generation,evidence_proof) VALUES($1,$2,$3,'reserved',$4,$5,$6,$7)", [ackId, b.attemptId, transition, next, b.bindingProof, b.controlGeneration, evidence])
|
|
await client.query("COMMIT")
|
|
return true
|
|
} catch (error) { await client.query("ROLLBACK").catch(() => {}); throw error }
|
|
}
|
|
try {
|
|
const winners = await Promise.all([compete(first, false), compete(second, true)])
|
|
assert.equal(new Set(pids).size, 2)
|
|
assert.equal(winners.filter(Boolean).length, 1)
|
|
const result = (await pool.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0]
|
|
assert.equal((await occupancy(b)).length, result.state === "released" ? 0 : 2)
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_attempt_events WHERE attempt_id=$1", [b.attemptId])).rows[0].n, 2)
|
|
} finally { await first.end(); await second.end() }
|
|
cases.push("actual two-client distinct-backend reserved UPDATE CAS race after shared BEGIN barrier: arm versus release, one committed winner, loser rows=0/ROLLBACK, complete event HMAC and 0-or-2 occupancy")
|
|
}
|
|
{
|
|
const b = fresh("immutable-guards"); await reserve(b)
|
|
const before = await snapshot()
|
|
await assert.rejects(pool.query("UPDATE mmapp_control.bandwidth_attempts SET binding=jsonb_set(binding,'{durationSec}','3'::jsonb) WHERE attempt_id=$1", [b.attemptId]))
|
|
await assert.rejects(pool.query("UPDATE mmapp_control.bandwidth_node_occupancy SET node_id='foreign-node' WHERE attempt_id=$1", [b.attemptId]))
|
|
await assert.rejects(pool.query("UPDATE mmapp_control.bandwidth_node_occupancy SET attempt_id=$2 WHERE attempt_id=$1", [b.attemptId, randomUUID()]))
|
|
await assert.rejects(pool.query("DELETE FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", [b.attemptId]))
|
|
await assert.rejects(pool.query("UPDATE mmapp_control.bandwidth_attempt_events SET evidence_proof=$2 WHERE attempt_id=$1", [b.attemptId, "f".repeat(64)]))
|
|
await assert.rejects(pool.query("DELETE FROM mmapp_control.bandwidth_attempt_events WHERE attempt_id=$1", [b.attemptId]))
|
|
await assert.rejects(pool.query("DELETE FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId]))
|
|
await assert.rejects(pool.query("TRUNCATE mmapp_control.bandwidth_attempts,mmapp_control.bandwidth_node_occupancy,mmapp_control.bandwidth_attempt_events"))
|
|
assert.deepEqual(await snapshot(), before)
|
|
cases.push("actual DML guards refuse immutable binding, node identity/owner replacement, active occupancy DELETE, event UPDATE/DELETE, attempt DELETE and full journal TRUNCATE; exact rows unchanged")
|
|
}
|
|
{
|
|
const b = fresh("replica-guards"); await reserve(b)
|
|
const before = await snapshot()
|
|
const client = await pool.connect()
|
|
try {
|
|
for (const sql of ["DELETE FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", "DELETE FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", "TRUNCATE mmapp_control.bandwidth_attempts,mmapp_control.bandwidth_node_occupancy,mmapp_control.bandwidth_attempt_events"]) {
|
|
await client.query("BEGIN")
|
|
try {
|
|
// Disposable superuser fixture only: prove ENABLE ALWAYS survives
|
|
// replica mode. This does not claim application role isolation.
|
|
await client.query("SET LOCAL session_replication_role=replica")
|
|
await assert.rejects(client.query(sql, sql.includes("$1") ? [b.attemptId] : []))
|
|
} finally { await client.query("ROLLBACK") }
|
|
}
|
|
} finally { client.release() }
|
|
await verifyGuardMetadata()
|
|
assert.deepEqual(await snapshot(), before)
|
|
cases.push("owned replica mode in three separate rolled-back transactions cannot DELETE active attempt/occupancy or TRUNCATE journal; nine ALWAYS triggers and six fixed search_path functions verified in pg_catalog")
|
|
}
|
|
{
|
|
const b = fixtureBinding("real-expiry:A", "real-expiry:B", { expiresAt: Date.now() + 1000 })
|
|
await reserve(b)
|
|
await new Promise<void>(resolve => setTimeout(resolve, Math.max(1, b.expiresAt - Date.now() + 20)))
|
|
await assert.rejects(h.journal.arm(b, scope()))
|
|
await h.journal.quarantine(b, scope())
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
assert.equal((await pool.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].state, "unknown_busy")
|
|
cases.push("real wall-clock original execute expiry refuses arm; bounded original cleanup can quarantine without freeing pair")
|
|
}
|
|
{
|
|
const b = fresh("blocked-deadline"), blocker = await pool.connect()
|
|
try {
|
|
await blocker.query("BEGIN"); await blocker.query("LOCK mmapp_control.bandwidth_node_occupancy IN ACCESS EXCLUSIVE MODE")
|
|
h.register(b)
|
|
await assert.rejects(h.journal.reserve(b, scope(200)))
|
|
const failedPid = h.lastPid()
|
|
await blocker.query("ROLLBACK")
|
|
assert.equal((await blocker.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].n, 0)
|
|
const goneBy = Date.now() + 2000
|
|
let remaining = 1
|
|
while (remaining && Date.now() < goneBy) {
|
|
remaining = Number((await blocker.query("SELECT count(*)::int AS n FROM pg_stat_activity WHERE pid=$1", [failedPid])).rows[0].n)
|
|
if (remaining) await new Promise<void>(resolve => setTimeout(resolve, 20))
|
|
}
|
|
assert.equal(remaining, 0)
|
|
assert.equal((await occupancy(b)).length, 0)
|
|
} finally { await blocker.query("ROLLBACK").catch(() => {}); blocker.release() }
|
|
cases.push("real relation lock and 200ms owned deadline destroy blocked adapter backend; after unlock no late COMMIT/occupancy")
|
|
}
|
|
{
|
|
const b = fixtureBinding("uuid-case:A", "uuid-case:B", { attemptId: randomUUID().toUpperCase() })
|
|
assert.equal((await reserve(b)).attemptId, b.attemptId)
|
|
assert.equal((await h.journal.arm(b, scope())).attemptId, b.attemptId)
|
|
assert.equal((await complete(b)).attemptId, b.attemptId)
|
|
assert.equal((await occupancy(b)).length, 0)
|
|
assert.equal((await pool.query("SELECT attempt_id::text FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [b.attemptId])).rows[0].attempt_id, b.attemptId.toLowerCase())
|
|
cases.push("uppercase original UUID spelling is preserved in new ACKs while PostgreSQL identity and event HMACs use canonical UUID")
|
|
}
|
|
{
|
|
// Deliberately misleading PUBLIC objects belong only to this fixture.
|
|
// Real control SQL must be fixed-qualified, independently of search_path.
|
|
for (const name of tables) {
|
|
await pool.query(`CREATE TABLE public.${name}(sentinel text NOT NULL)`)
|
|
await pool.query(`INSERT INTO public.${name} VALUES('PUBLIC-SHADOW-MUST-STAY-UNCHANGED')`)
|
|
}
|
|
const { Pool } = await import("pg")
|
|
const shadowPool = new Pool({ connectionString: process.env.DATABASE_URL, options: "-c search_path=public,pg_catalog,mmapp_control", max: 2 })
|
|
try {
|
|
const fixture = await createOwnedBandwidthHarness(shadowPool), b = fresh("search-path")
|
|
fixture.register(b)
|
|
await fixture.journal.reserve(b, scope()); await fixture.journal.arm(b, scope())
|
|
assert.equal((await occupancy(b)).length, 2)
|
|
await fixture.journal.complete(b, { terminalProof: completionProof(b) }, scope())
|
|
for (const name of tables) assert.deepEqual((await pool.query(`SELECT sentinel FROM public.${name}`)).rows, [{ sentinel: "PUBLIC-SHADOW-MUST-STAY-UNCHANGED" }])
|
|
} finally {
|
|
await shadowPool.end()
|
|
for (const name of tables) await pool.query(`DROP TABLE public.${name}`)
|
|
}
|
|
cases.push("public shadow tables plus public-first search_path cannot stand in for fixed mmapp_control journal; reserve/arm/complete touch only control namespace")
|
|
}
|
|
{
|
|
const { Pool } = await import("pg"), url = new URL(process.env.DATABASE_URL!)
|
|
const dbName = "bandwidth_missing_" + randomUUID().replaceAll("-", "")
|
|
await pool.query(`CREATE DATABASE "${dbName}" TEMPLATE template0`)
|
|
url.pathname = "/" + dbName
|
|
const missing = new Pool({ connectionString: url.toString(), max: 2 })
|
|
const namespaceInventory = async () => (await missing.query(`SELECT n.nspname,c.relname,c.relkind::text AS kind FROM pg_catalog.pg_class c
|
|
JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname IN ('public','mmapp_control') ORDER BY n.nspname,c.relname`)).rows
|
|
try {
|
|
const before = await namespaceInventory(), fixture = await createOwnedBandwidthHarness(missing), b = fresh("missing-control")
|
|
fixture.register(b); await assert.rejects(fixture.journal.reserve(b, scope()))
|
|
assert.deepEqual(await namespaceInventory(), before)
|
|
assert.equal((await missing.query("SELECT count(*)::int AS n FROM pg_catalog.pg_namespace WHERE nspname='mmapp_control'")).rows[0].n, 0)
|
|
// Existing public impostors still cannot make an erased/new database an
|
|
// enrolled source. No source-generation issuer is manufactured here.
|
|
for (const name of tables) await missing.query(`CREATE TABLE public.${name}(sentinel text)`)
|
|
const shadowed = await namespaceInventory(), another = await createOwnedBandwidthHarness(missing), next = fresh("missing-shadow")
|
|
another.register(next); await assert.rejects(another.journal.reserve(next, scope()))
|
|
assert.deepEqual(await namespaceInventory(), shadowed)
|
|
} finally { await missing.end(); await pool.query(`DROP DATABASE "${dbName}"`) }
|
|
cases.push("new/erased owned database lacking mmapp_control refuses journal reserve without ACK or object creation; PUBLIC impostors cannot auto-enroll it")
|
|
}
|
|
let eventCount = await verifyEvents(); assert(eventCount > 0); assert(h.committedLosses() >= 3)
|
|
cases.push("every persisted transition event verified with keyed fixture HMAC including terminal completion proof")
|
|
if (process.env.MMAPP_OWN_BANDWIDTH_RESTORE === "true") {
|
|
const execute = promisify(execFile), url = new URL(process.env.DATABASE_URL!)
|
|
const archive = resolve(process.env.MCP_STATE_DIR!, "bandwidth-proposal.dump"), dbName = "bandwidth_restore_" + randomUUID().replaceAll("-", "")
|
|
const env = { ...process.env, PGHOST: url.hostname, PGPORT: url.port, PGUSER: url.username, PGPASSWORD: decodeURIComponent(url.password) }
|
|
const { runPrivateDatabaseDump } = await import("./database-dump-connection.js")
|
|
const { restorePartitionedDatabase } = await import("./system-database-partition-restore.js")
|
|
const commandIo = async (program: string, args: string[]): Promise<Buffer> => {
|
|
try { return (await execute(program, args, { env, encoding: "buffer", timeout: 30000, maxBuffer: 32 * 1048576, windowsHide: true })).stdout }
|
|
catch { throw Error("Owned PostgreSQL archive command failed: " + program) }
|
|
}
|
|
const markerId = Number((await pool.query("INSERT INTO public.servers(name,host,enabled) VALUES('owned-bandwidth-public-before','127.0.0.1',false) RETURNING id")).rows[0].id)
|
|
const safeArchive = resolve(process.env.MCP_STATE_DIR!, "bandwidth-public-only.dump")
|
|
assert((await pool.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_attempts WHERE state='armed'")).rows[0].n > 0)
|
|
assert((await pool.query("SELECT count(*)::int AS n FROM mmapp_control.bandwidth_attempts WHERE state='unknown_busy'")).rows[0].n > 0)
|
|
const appDump = await runPrivateDatabaseDump(url.toString(), { authorize: async () => {} })
|
|
await writeFile(safeArchive, appDump, { flag: "wx", mode: 0o600 })
|
|
const safeToc = await commandIo("pg_restore", ["--list", safeArchive]); assert(!safeToc.toString("utf8").includes("mmapp_control"))
|
|
// This busy pair did not exist when the application backup was created.
|
|
// A public rollback must preserve the NEWER control state as well.
|
|
const postBackup = fresh("after-backup")
|
|
await reserve(postBackup); await h.journal.arm(postBackup, scope())
|
|
const preserved = await snapshot(), preservedGuards = await verifyGuardMetadata()
|
|
await pool.query("UPDATE public.servers SET name='owned-bandwidth-public-after' WHERE id=$1", [markerId])
|
|
let safeReplacements = 0
|
|
await restorePartitionedDatabase(safeArchive, url.toString(), { command: commandIo,
|
|
prepareSql: file => writeFile(file, "", { flag: "wx", mode: 0o600 }), beforeTargetReplacement: () => { safeReplacements++ } })
|
|
assert.equal(safeReplacements, 1)
|
|
assert.equal((await pool.query("SELECT name FROM public.servers WHERE id=$1", [markerId])).rows[0].name, "owned-bandwidth-public-before")
|
|
assert.deepEqual(await snapshot(), preserved); assert.deepEqual(await verifyGuardMetadata(), preservedGuards)
|
|
const afterAppRestore = await createOwnedBandwidthHarness(), blocked = fixtureBinding(postBackup.source.canonicalNodeId, "after-app-restore:C")
|
|
afterAppRestore.register(blocked); await assert.rejects(afterAppRestore.journal.reserve(blocked, scope()))
|
|
assert.deepEqual(await snapshot(), preserved)
|
|
cases.push("actual application pg_dump excludes mmapp_control; actual public-only restore rolls public marker back while preserving NEWER post-backup armed pair and exact unknown/control hashes plus nine ALWAYS/six fixed-path guards; fresh overlapping reserve still refused")
|
|
const expected = await snapshot(), expectedGuards = await verifyGuardMetadata()
|
|
await execute("pg_dump", ["-Fc", "--no-owner", "-f", archive, "-d", "postgres"], { env })
|
|
let unsafeCommands = 0, unsafePreparations = 0, unsafeReplacements = 0
|
|
await assert.rejects(restorePartitionedDatabase(archive, url.toString(), {
|
|
command: async (program, args) => { unsafeCommands++; return commandIo(program, args) },
|
|
prepareSql: async () => { unsafePreparations++ }, beforeTargetReplacement: () => { unsafeReplacements++ },
|
|
}))
|
|
assert.equal(unsafeCommands, 1); assert.equal(unsafePreparations, 0); assert.equal(unsafeReplacements, 0)
|
|
assert.deepEqual(await snapshot(), expected)
|
|
assert.equal((await pool.query("SELECT name FROM public.servers WHERE id=$1", [markerId])).rows[0].name, "owned-bandwidth-public-before")
|
|
cases.push("actual full archive containing mmapp_control rejected by shared restore immediately after TOC, before prepare/decode/target replacement; control and public marker unchanged")
|
|
await pool.query(`CREATE DATABASE "${dbName}"`)
|
|
const { Pool } = await import("pg"), targetUrl = new URL(url); targetUrl.pathname = "/" + dbName
|
|
const restored = new Pool({ connectionString: targetUrl.toString() })
|
|
try {
|
|
await execute("pg_restore", ["--no-owner", "--no-privileges", "--exit-on-error", "-d", dbName, archive], { env })
|
|
assert.deepEqual(await snapshot(restored), expected)
|
|
assert.deepEqual(await verifyGuardMetadata(restored), expectedGuards)
|
|
assert.equal(Number((await restored.query("SELECT count(*) AS n FROM schema_migrations")).rows[0].n), MIGRATION_MANIFEST.length)
|
|
const content = await readFile(archive); content[0] ^= 0xff
|
|
const bad = resolve(process.env.MCP_STATE_DIR!, "bandwidth-proposal-tampered.dump"); await writeFile(bad, content)
|
|
await assert.rejects(execute("pg_restore", ["--list", bad], { env }))
|
|
cases.push("explicit full control-snapshot clone (NOT application restore) through real pg_dump/pg_restore into separate owned DB preserves exact proposal hashes/unknown occupancy, nine ALWAYS triggers, six fixed search_path functions and 39 markers; corrupted header rejected")
|
|
} finally { await restored.end(); await pool.query(`DROP DATABASE "${dbName}"`) }
|
|
} else limitations.push("Owned proposal dump/restore not requested")
|
|
eventCount = await verifyEvents()
|
|
limitations.push("No production native transport/broker integration or schema/control enrollment: SQL proposal only")
|
|
limitations.push("No real remote-completion/physical identity enrollment; terminal evidence is explicitly synthetic keyed fixture proof")
|
|
limitations.push("No aged retention/automatic GC integration; no automatic unknown release policy is supplied")
|
|
limitations.push("Deleting synthetic servers tests canonical-key independence; there is no implemented server-to-physical-node mapping")
|
|
limitations.push("Production source-generation issuer, authenticated external provisioning/READY and least-privilege connection/restore-role isolation are unproven; no automatic enrollment")
|
|
console.log("Owned bandwidth proposal SQL acceptance:", JSON.stringify({ proposalOnly: true, normalMigrationCount: MIGRATION_MANIFEST.length, normalEnrollmentChanged: false,
|
|
productionRestoreAccepted: false, runtimeDispatchIntegrated: false, nativeChrCalls: 0, actualCommitAckLosses: h.committedLosses(), eventCount, cases, limitations }))
|
|
} finally {
|
|
for (const processChild of children) if (processChild.connected) processChild.kill()
|
|
await closePool()
|
|
}
|