Files
MikrotikManager/backend/src/services/server-partition-maintenance-lease.sql-helper.ts
T

198 lines
15 KiB
TypeScript

import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { basename, dirname, resolve } from "node:path"
import type { PoolClient, QueryConfig } from "pg"
import type { ServerPartitionLeaseOptions } from "./server-partition-maintenance-lease.js"
/** Proposal acceptance only. Called explicitly by the owned disposable SQL
* profile after migrations; import is inert. Installs no production migration,
* deletes no rows and does not claim independent DDL/TRUNCATE writers fenced. */
export async function acceptServerPartitionMaintenanceLeaseSql() {
assert.equal(process.env.MMAPP_OWN_PROBE_JOURNAL, "true")
const database = new URL(process.env.DATABASE_URL ?? "")
assert.equal(database.hostname, "127.0.0.1"); assert.equal(database.pathname, "/postgres")
assert(["postgres:", "postgresql:"].includes(database.protocol))
assert(process.env.MCP_STATE_DIR && basename(dirname(resolve(process.env.MCP_STATE_DIR))).startsWith("mm-mcp-test-"))
const { pool } = await import("../db/index.js")
const { PARTITION_SPECS, ensurePartitionFor } = await import("../db/partitions.js")
const { createServerPartitionMaintenanceLease, SERVER_PARTITION_WRITER_DDL, SERVER_PARTITION_WRITER_LOCK, SERVER_PARTITION_WRITER_INVENTORY_SQL, ServerPartitionLeaseError } = await import("./server-partition-maintenance-lease.js")
assert.equal((await pool.query("SELECT current_database() AS name")).rows[0].name, "postgres")
assert.equal((await pool.query("SELECT to_regprocedure('public.server_partition_writer_guard()') AS name")).rows[0].name, null, "Proposal must not already be installed")
await pool.query(SERVER_PARTITION_WRITER_DDL)
const sampledAt = "2099-04-01T12:00:00.000Z"
const leaf = await ensurePartitionFor(pool, "traffic_samples", "week", new Date(sampledAt))
assert(/^[a-z0-9_]+$/.test(leaf))
const serverId = Number((await pool.query("INSERT INTO servers(name,host,port,enabled) VALUES('partition-lease-owned-fixture','127.0.0.1',9,false) RETURNING id")).rows[0].id)
const options = (): ServerPartitionLeaseOptions => ({ deadlineAt: Date.now() + 20000, serverIds: [serverId] })
const sql = (client: Pick<PoolClient, "query">, text: string, values: unknown[] = []) => client.query({ text, values, query_timeout: 6000 } as QueryConfig & { query_timeout: number })
const pidOf = async (client: Pick<PoolClient, "query">) => Number((await sql(client, "SELECT pg_backend_pid() AS pid")).rows[0].pid)
const waitFor = async <T>(name: string, inspect: () => Promise<T | undefined>): Promise<T> => {
const deadline = Date.now() + 10000
while (Date.now() < deadline) {
const result = await inspect(); if (result !== undefined) return result
// Poll scheduling only; success always requires the SQL predicate below.
await new Promise(resolve => setTimeout(resolve, 10))
}
assert.fail("Timed out observing " + name)
}
const waiting = (pid: number, mode: "ShareLock" | "ExclusiveLock") => waitFor("actual waiting advisory " + mode, async () => {
const rows = (await sql(pool, "SELECT mode,granted FROM pg_locks WHERE locktype='advisory' AND pid=$1 AND classid=0 AND objid=$2 AND objsubid=1 AND mode=$3 AND NOT granted", [pid, SERVER_PARTITION_WRITER_LOCK, mode])).rows
return rows.length === 1 ? rows[0] : undefined
})
const gone = (pid: number) => waitFor("released backend lock", async () => (await sql(pool, "SELECT count(*)::int AS n FROM pg_locks WHERE locktype='advisory' AND pid=$1 AND classid=0 AND objid=$2", [pid, SERVER_PARTITION_WRITER_LOCK])).rows[0].n === 0 ? true : undefined)
const insert = "INSERT INTO traffic_samples(server_id,interface_name,sampled_at,rx_bytes) VALUES($1,$2,$3,1)"
const trace: Array<{ pid: number; text: string }> = []
const handledClients = new WeakSet<object>()
const sqlErrorCodes: string[] = []
let ownerPid = 0, loseCommitAck = false, committedAcksLost = 0, cases = 0
const lease = createServerPartitionMaintenanceLease({
assertAuthority: async opts => { assert(opts.serverIds.length === 0 || opts.serverIds.length === 1 && opts.serverIds[0] === serverId); assert(opts.deadlineAt > Date.now()); assert(!opts.signal?.aborted) },
connect: async () => {
const client = await pool.connect(), pid = await pidOf(client); ownerPid = pid
// An intentionally terminated owned backend must not emit an unhandled
// client error; the next command still rejects at the actual SQL boundary.
if (!handledClients.has(client)) { handledClients.add(client); client.on("error", () => {}) }
return {
query: async (text, values, opts) => {
trace.push({ pid, text })
const result = await client.query({ text, values, query_timeout: Math.min(6000, opts.timeoutMs) } as QueryConfig & { query_timeout: number }).catch((error: unknown) => {
if (error && typeof error === "object" && "code" in error) sqlErrorCodes.push(String(error.code))
throw error
})
if (text === "COMMIT" && loseCommitAck) { loseCommitAck = false; committedAcksLost++; throw Error("Synthetic lost ACK after actual committed SQL") }
return result
},
release: destroy => client.release(destroy),
}
},
})
try {
const inventory = (await sql(pool, SERVER_PARTITION_WRITER_INVENTORY_SQL, [PARTITION_SPECS.map(spec => spec.parent)])).rows
assert.equal(PARTITION_SPECS.length, 12)
assert.equal(new Set(inventory.map(row => row.parent)).size, 12)
assert(inventory.length > 12 && inventory.some(row => row.relname === leaf), "Future partition must inherit the trigger")
assert(inventory.every(row => row.trigger_enabled === "A" && row.trigger_type === 31 && row.unconditional && row.function_name === "server_partition_writer_guard"))
await lease.withLease(async () => {}, options()); cases++
for (const boundary of ["COMMIT", "ROLLBACK"] as const) {
const writer = await pool.connect()
let pending: Promise<void> | undefined
try {
await sql(writer, "BEGIN"); await sql(writer, "SET LOCAL statement_timeout='15000ms'")
await sql(writer, insert, [serverId, randomUUID(), sampledAt])
const writerPid = await pidOf(writer)
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM pg_locks WHERE locktype='advisory' AND pid=$1 AND classid=0 AND objid=$2 AND mode='ShareLock' AND granted", [writerPid, SERVER_PARTITION_WRITER_LOCK])).rows[0].n, 1)
let entered = false; ownerPid = 0
pending = lease.withLease(async () => { entered = true }, options())
void pending.catch(() => {})
const pid = await waitFor("maintenance backend", async () => ownerPid || undefined)
await waiting(pid, "ExclusiveLock"); assert.equal(entered, false)
await sql(writer, boundary); await pending; assert.equal(entered, true); await gone(pid); cases++
} finally { await sql(writer, "ROLLBACK").catch(() => {}); writer.release(); await pending?.catch(() => {}) }
}
const writer = await pool.connect()
let blockedInsert: Promise<unknown> | undefined
try {
await sql(writer, "SET statement_timeout='15000ms'")
const otherPid = await pidOf(writer), name = randomUUID()
await lease.withLease(async () => {
const pid = ownerPid
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM pg_locks WHERE locktype='advisory' AND pid=$1 AND classid=0 AND objid=$2 AND mode='ExclusiveLock' AND granted", [pid, SERVER_PARTITION_WRITER_LOCK])).rows[0].n, 1)
const start = trace.length
await lease.transaction(false, async () => {}, options(), async query => {
assert.equal(Number((await query("SELECT pg_backend_pid() AS pid")).rows[0].pid), pid)
await query(insert, [serverId, name, sampledAt]) // Same backend's shared row trigger is reentrant.
await lease.transaction(true, async () => {}, options(), async read => { assert.equal(Number((await read("SELECT pg_backend_pid() AS pid")).rows[0].pid), pid) })
let nestedWrite = false
await assert.rejects(lease.transaction(false, async () => {}, options(), async () => { nestedWrite = true }), ServerPartitionLeaseError)
assert.equal(nestedWrite, false)
})
assert.equal(new Set(trace.slice(start).map(row => row.pid)).size, 1)
assert(trace.slice(start).some(row => row.text.startsWith("SAVEPOINT ")))
assert(trace.slice(start).some(row => row.text.startsWith("RELEASE SAVEPOINT ")))
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM traffic_samples WHERE server_id=$1 AND interface_name=$2", [serverId, name])).rows[0].n, 1)
blockedInsert = sql(writer, insert, [serverId, randomUUID(), sampledAt]); void blockedInsert.catch(() => {})
await waiting(otherPid, "ShareLock")
}, options())
await blockedInsert; cases++
} finally { writer.release(); await blockedInsert?.catch(() => {}) }
// Expected integration failure: a row trigger takes the shared advisory
// lock AFTER INSERT already owns relation locks. Never prove this with an
// actual DROP: NOWAIT safely exposes the reverse lock ordering as 55P03.
const lateWriter = await pool.connect()
let lateInsert: Promise<unknown> | undefined
try {
await sql(lateWriter, "SET statement_timeout='15000ms'")
const pid = await pidOf(lateWriter), beforeCodes = sqlErrorCodes.length
await assert.rejects(lease.withLease(async () => {
lateInsert = sql(lateWriter, insert, [serverId, randomUUID(), sampledAt]); void lateInsert.catch(() => {})
await waiting(pid, "ShareLock")
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM pg_locks WHERE locktype='relation' AND pid=$1 AND relation='public.traffic_samples'::regclass AND mode='RowExclusiveLock' AND granted", [pid])).rows[0].n, 1)
await lease.transaction(false, async () => {}, options(), async query => { await query("LOCK TABLE public.traffic_samples IN ACCESS EXCLUSIVE MODE NOWAIT") })
}, options()), (error: unknown) => error instanceof ServerPartitionLeaseError && error.outcomeUnknown)
assert(sqlErrorCodes.slice(beforeCodes).includes("55P03"), "Actual SQL must expose row-trigger DDL lock-order gap")
await lateInsert; cases++
} finally { lateWriter.release(); await lateInsert?.catch(() => {}) }
// Positive protocol proof requires application admission BEFORE table SQL.
// This instrumented writer has no traffic relation lock while waiting.
const earlyWriter = await pool.connect()
let earlyAdmission: Promise<unknown> | undefined
try {
await sql(earlyWriter, "BEGIN"); await sql(earlyWriter, "SET LOCAL statement_timeout='15000ms'")
const pid = await pidOf(earlyWriter)
await lease.withLease(async () => {
earlyAdmission = sql(earlyWriter, "SELECT pg_advisory_xact_lock_shared($1::bigint)", [SERVER_PARTITION_WRITER_LOCK]); void earlyAdmission.catch(() => {})
await waiting(pid, "ShareLock")
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM pg_locks WHERE locktype='relation' AND pid=$1 AND relation='public.traffic_samples'::regclass", [pid])).rows[0].n, 0)
await lease.transaction(false, async () => {}, options(), async query => { await query("LOCK TABLE public.traffic_samples IN ACCESS EXCLUSIVE MODE NOWAIT") })
}, options())
await earlyAdmission
await sql(earlyWriter, insert, [serverId, randomUUID(), sampledAt]); await sql(earlyWriter, "COMMIT"); cases++
} finally { await earlyAdmission?.catch(() => {}); await sql(earlyWriter, "ROLLBACK").catch(() => {}); earlyWriter.release() }
let detached: Promise<void> | undefined, releaseDetached!: () => void
const gate = new Promise<void>(resolve => { releaseDetached = resolve })
await lease.withLease(async () => { detached = gate.then(() => lease.transaction(true, async () => {}, options(), async () => {})) }, options())
const beforeDetached = trace.length
releaseDetached(); await assert.rejects(detached!, ServerPartitionLeaseError); assert.equal(trace.length, beforeDetached); cases++
// Reversible proposal-only schema fault on our newly created leaf. No data
// or audit rows are removed, and the trigger is restored even on failure.
await sql(pool, `ALTER TABLE public."${leaf}" DISABLE TRIGGER server_partition_writer_guard`)
try {
let entered = false
await assert.rejects(lease.withLease(async () => { entered = true }, options()), ServerPartitionLeaseError)
assert.equal(entered, false); cases++
} finally { await sql(pool, `ALTER TABLE public."${leaf}" ENABLE ALWAYS TRIGGER server_partition_writer_guard`) }
const lostName = randomUUID(), beforeLost = trace.length
loseCommitAck = true
await assert.rejects(lease.withLease(() => lease.transaction(false, async () => {}, options(), async query => { await query(insert, [serverId, lostName, sampledAt]) }), options()), (error: unknown) => error instanceof ServerPartitionLeaseError && error.outcomeUnknown)
assert.equal(committedAcksLost, 1)
const lostTrace = trace.slice(beforeLost)
assert.equal(lostTrace.filter(row => row.text === "COMMIT").length, 1)
assert.equal(lostTrace.filter(row => row.text.startsWith("INSERT INTO traffic_samples")).length, 1)
assert(!lostTrace.some(row => row.text === "ROLLBACK" || row.text.includes("partition-lease:unlock")))
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM traffic_samples WHERE server_id=$1 AND interface_name=$2", [serverId, lostName])).rows[0].n, 1)
await gone(ownerPid); cases++
const disconnectedName = randomUUID(), beforeDisconnect = trace.length
await assert.rejects(lease.withLease(() => lease.transaction(false, async () => {}, options(), async query => {
await query(insert, [serverId, disconnectedName, sampledAt])
assert.equal((await sql(pool, "SELECT pg_terminate_backend($1) AS terminated", [ownerPid])).rows[0].terminated, true)
await query("SELECT 1")
}), options()), (error: unknown) => error instanceof ServerPartitionLeaseError && error.outcomeUnknown)
assert.equal(trace.slice(beforeDisconnect).filter(row => row.text.startsWith("INSERT INTO traffic_samples")).length, 1)
assert(!trace.slice(beforeDisconnect).some(row => row.text === "COMMIT" || row.text === "ROLLBACK"))
await gone(ownerPid)
assert.equal((await sql(pool, "SELECT count(*)::int AS n FROM traffic_samples WHERE server_id=$1 AND interface_name=$2", [serverId, disconnectedName])).rows[0].n, 0)
cases++
return { cases, roots: 12, descendants: inventory.length - 12, actualPostgres: true, observedAdvisoryWaits: 5, rowTriggerOnlyDdlSafe: false, observedLateWriterDdlRefusal: "55P03", preSqlWriterAdmissionRequired: true, independentDdlAndTruncateFenced: false, migrationApplied: false, productionWiring: false }
} finally {
await sql(pool, "UPDATE servers SET enabled=false WHERE id=$1", [serverId])
}
}