279 lines
14 KiB
TypeScript
279 lines
14 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { fork } from "node:child_process"
|
|
import { once } from "node:events"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import { fileURLToPath } from "node:url"
|
|
import Fastify from "fastify"
|
|
import type { PoolClient } from "pg"
|
|
import { closePool, pool } from "../db/index.js"
|
|
import { SYSTEM_DATABASE_REQUEST_LOCK, withSystemDatabaseOperationLock } from "../db/operation-lock.js"
|
|
import { randomUUID } from "node:crypto"
|
|
import { withDetachedDatabaseApiRequest } from "../modules/mcp/database-request-lease.js"
|
|
import {
|
|
DATABASE_RESTORE_QUARANTINE_DDL, beginDatabaseRestoreQuarantine, completeDatabaseRestoreQuarantine,
|
|
} from "../db/database-restore-quarantine.js"
|
|
import { installDatabaseRequestGate } from "../plugins/database-request-gate.js"
|
|
import { withConfigMutationLock } from "../services/config-mutation-lock.js"
|
|
import { databaseBackgroundWork } from "../services/database-background-work.js"
|
|
import {
|
|
admitPartitionWriter, PARTITION_WRITER_ADVISORY_KEY, PartitionWriteError, partitionWriteFailureDiagnostic, withPartitionWrite,
|
|
} from "../services/partition-write-transaction.js"
|
|
import { deleteTrafficFlowData } from "../services/traffic-flow-purge-execution.js"
|
|
import { enqueueRipeMisses, lookupRipeCached, resetRipeCacheForTests, setRipeFetchForTests } from "../services/traffic-flow-ripe.js"
|
|
import { commitFlowWrite } from "../services/traffic-flow-write-transaction.js"
|
|
|
|
const endpoint = "/api/owned-replica"
|
|
const detachedEndpoint = "/api/owned-replica/detached"
|
|
const detachedAdmissionEndpoint = "/owned-replica/detached-admission"
|
|
const readSql = "SELECT audit_retention_days FROM mcp_settings WHERE id=1"
|
|
const lockedRead = readSql + " FOR UPDATE"
|
|
const finishDetachedSql = "UPDATE mcp_settings SET audit_retention_days=audit_retention_days+1 WHERE id=1"
|
|
const pendingExclusiveLock = "SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid=0 AND objid=$1::oid AND objsubid=1 AND mode='ExclusiveLock' AND NOT granted"
|
|
const quarantineCode = "DATABASE_RESTORE_QUARANTINE_UNAVAILABLE"
|
|
const readQuarantineId = "SELECT operation_id FROM mmapp_control.database_restore_quarantine WHERE id=1"
|
|
|
|
if (process.argv.includes("--replica-child")) {
|
|
assert(process.send && process.env.MMAPP_OWN_REST_REPLICA_ONLY === "true")
|
|
let finishDetached!: () => void
|
|
const detachedLatch = new Promise<void>(resolve => { finishDetached = resolve })
|
|
let detached: Promise<void> | undefined
|
|
const app = Fastify()
|
|
installDatabaseRequestGate(app)
|
|
app.get("/health", async () => ({ ok: true }))
|
|
app.post(endpoint, async () => {
|
|
await pool.query(lockedRead)
|
|
return withConfigMutationLock([721], async () => {
|
|
const { rows } = await pool.query("UPDATE mcp_settings SET audit_retention_days=audit_retention_days+1 WHERE id=1 RETURNING audit_retention_days")
|
|
return rows[0]
|
|
})
|
|
})
|
|
app.post(detachedEndpoint, async (_request, reply) => {
|
|
detached = withDetachedDatabaseApiRequest(async () => {
|
|
process.send?.("detached-started")
|
|
await detachedLatch
|
|
await pool.query(finishDetachedSql)
|
|
})
|
|
void detached.catch(error => console.error("Detached replica work failed", error))
|
|
reply.code(202)
|
|
return { accepted: true }
|
|
})
|
|
// Bypass the HTTP gate to exercise the detached operation's own quarantine check.
|
|
app.post(detachedAdmissionEndpoint, () => withDetachedDatabaseApiRequest(async () => {
|
|
assert.fail("Quarantined detached work entered its callback")
|
|
}))
|
|
const address = await app.listen({ host: "127.0.0.1", port: 0 })
|
|
process.on("message", async message => {
|
|
if (message === "finish-detached") { finishDetached(); return }
|
|
if (message !== "stop") return
|
|
await app.close()
|
|
await detached?.catch(() => {})
|
|
await closePool()
|
|
process.disconnect?.()
|
|
})
|
|
process.send(address)
|
|
}
|
|
|
|
async function verifyRipeCacheWrites(phase: "partition lock held" | "quarantined" | "reopened") {
|
|
const ip = "8.8.8.8", prefix = `${ip}/32`, asn = 64512, holder = "Owned RIPE fixture"
|
|
const canPersist = phase === "reopened"
|
|
setRipeFetchForTests(async input => {
|
|
const { pathname } = new URL(String(input))
|
|
if (pathname === "/data/network-info/data.json") return Response.json({ data: { prefix, asns: [asn] } })
|
|
if (pathname === "/data/maxmind-geo-lite/data.json") return Response.json({ data: {} })
|
|
assert.equal(pathname, "/data/as-overview/data.json")
|
|
return Response.json({ data: { holder } })
|
|
})
|
|
try {
|
|
enqueueRipeMisses([ip])
|
|
await databaseBackgroundWork.drain(4000)
|
|
const entry = lookupRipeCached(ip)
|
|
assert.equal(entry?.holder, holder, `RIPE resolution with admission ${phase}`)
|
|
assert.equal(entry?.asn, asn)
|
|
assert.deepEqual((await pool.query("SELECT asn, holder FROM flow_ip_meta WHERE prefix=$1", [prefix])).rows,
|
|
canPersist ? [{ asn, holder }] : [], `RIPE prefix persistence with admission ${phase}`)
|
|
assert.deepEqual((await pool.query("SELECT holder FROM flow_asn_meta WHERE asn=$1", [asn])).rows,
|
|
canPersist ? [{ holder }] : [], `RIPE ASN persistence with admission ${phase}`)
|
|
} finally {
|
|
resetRipeCacheForTests()
|
|
}
|
|
}
|
|
|
|
export async function verifyDatabaseRestReplica() {
|
|
await pool.query("CREATE SCHEMA IF NOT EXISTS mmapp_control")
|
|
await pool.query(DATABASE_RESTORE_QUARANTINE_DDL)
|
|
const before = (await pool.query(readSql)).rows[0].audit_retention_days
|
|
const child = fork(fileURLToPath(import.meta.url), ["--replica-child"], {
|
|
execArgv: ["--import", "tsx"], stdio: ["ignore", "ignore", "inherit", "ipc"],
|
|
env: { ...process.env, DATABASE_RESTORE_QUARANTINE: "true" },
|
|
})
|
|
const exited = once(child, "exit")
|
|
const signal = AbortSignal.timeout(15000)
|
|
const blocker = await pool.connect()
|
|
const previousQuarantine = process.env.DATABASE_RESTORE_QUARANTINE
|
|
let snapshot: PoolClient | undefined
|
|
let first: Promise<Response> | undefined, maintenance: Promise<void> | undefined
|
|
let partitionWriter: Promise<number> | undefined, quarantine: Promise<void> | undefined
|
|
try {
|
|
const [address] = await Promise.race([
|
|
once(child, "message", { signal }),
|
|
exited.then(() => assert.fail("Replica exited before readiness")),
|
|
])
|
|
assert.equal(typeof address, "string")
|
|
const post = (path = endpoint) => fetch(address + path, { method: "POST", signal })
|
|
const waitFor = async (sql: string, args: unknown[]) => {
|
|
while (!(await pool.query(sql, args)).rowCount) await delay(15, undefined, { signal })
|
|
}
|
|
const refused = async (code = "PROTOCOL_STORAGE_BUSY", path = endpoint) => {
|
|
const response = await post(path)
|
|
assert.equal(response.status, 503)
|
|
const body = await response.json()
|
|
assert.ok(body && typeof body === "object" && "code" in body)
|
|
assert.equal(body.code, code)
|
|
}
|
|
await blocker.query("BEGIN")
|
|
await blocker.query(lockedRead)
|
|
first = post()
|
|
void first.catch(() => {})
|
|
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [lockedRead])
|
|
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
|
|
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, before + 1)
|
|
await refused()
|
|
assert.equal((await fetch(address + "/health", { signal })).status, 200)
|
|
})
|
|
void maintenance.catch(() => {})
|
|
await waitFor(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])
|
|
await refused()
|
|
await blocker.query("ROLLBACK")
|
|
const response = await first
|
|
assert.equal(response.status, 200)
|
|
assert.deepEqual(await response.json(), { audit_retention_days: before + 1 })
|
|
await maintenance
|
|
const resumed = await post()
|
|
assert.equal(resumed.status, 200)
|
|
assert.deepEqual(await resumed.json(), { audit_retention_days: before + 2 })
|
|
console.log("Independent PostgreSQL replica: queued maintenance drained the active REST request and its nested mutation, denied new requests, then reopened admission")
|
|
|
|
await blocker.query("BEGIN")
|
|
await blocker.query(lockedRead)
|
|
const [accepted, [started]] = await Promise.all([
|
|
post(detachedEndpoint), once(child, "message", { signal }),
|
|
])
|
|
assert.equal(accepted.status, 202)
|
|
assert.deepEqual(await accepted.json(), { accepted: true })
|
|
assert.equal(started, "detached-started")
|
|
const afterDetached = before + 3
|
|
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
|
|
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, afterDetached)
|
|
})
|
|
void maintenance.catch(() => {})
|
|
await waitFor(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])
|
|
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, before + 2)
|
|
child.send("finish-detached")
|
|
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [finishDetachedSql])
|
|
assert.ok((await pool.query(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])).rowCount)
|
|
await blocker.query("ROLLBACK")
|
|
await maintenance
|
|
console.log("Independent PostgreSQL replica: detached work held admission after HTTP 202 through its final committed UPDATE")
|
|
|
|
process.env.DATABASE_RESTORE_QUARANTINE = "true"
|
|
await blocker.query("BEGIN")
|
|
await blocker.query("SELECT pg_catalog.pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
|
|
await verifyRipeCacheWrites("partition lock held")
|
|
await blocker.query("ROLLBACK")
|
|
|
|
snapshot = await pool.connect()
|
|
await snapshot.query("BEGIN ISOLATION LEVEL REPEATABLE READ")
|
|
assert.equal((await snapshot.query(readQuarantineId)).rows[0].operation_id, null)
|
|
|
|
let writerCalls = 0
|
|
const writePartition = () => withPartitionWrite(async ({ client }) => {
|
|
writerCalls++
|
|
return (await client.query<{ audit_retention_days: number }>(lockedRead)).rows[0].audit_retention_days
|
|
}, { signal })
|
|
await blocker.query("BEGIN")
|
|
await blocker.query(lockedRead)
|
|
partitionWriter = writePartition()
|
|
void partitionWriter.catch(() => {})
|
|
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [lockedRead])
|
|
|
|
const operationId = randomUUID(), otherId = randomUUID()
|
|
let restoreTargetReached = false
|
|
quarantine = withSystemDatabaseOperationLock(() => pool.connect(), async owner => {
|
|
await owner.query("BEGIN", [])
|
|
await owner.query("SET LOCAL synchronous_commit=on", [])
|
|
await beginDatabaseRestoreQuarantine(owner, operationId)
|
|
await owner.query("COMMIT", [])
|
|
await owner.query("BEGIN", [])
|
|
try {
|
|
await owner.query("SET LOCAL lock_timeout='5s'", [])
|
|
await owner.query("SELECT pg_catalog.pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
|
|
await owner.query(readSql, [])
|
|
restoreTargetReached = true
|
|
} finally {
|
|
await owner.query("ROLLBACK", [])
|
|
}
|
|
})
|
|
void quarantine.catch(() => {})
|
|
await waitFor(pendingExclusiveLock + " AND EXISTS (SELECT 1 FROM pg_stat_activity AS writer WHERE writer.pid=ANY(pg_blocking_pids(pg_locks.pid)) AND writer.query=$2)", [PARTITION_WRITER_ADVISORY_KEY, lockedRead])
|
|
assert.equal(restoreTargetReached, false)
|
|
await blocker.query("ROLLBACK")
|
|
assert.equal(await partitionWriter, afterDetached)
|
|
await quarantine
|
|
assert.equal(restoreTargetReached, true)
|
|
|
|
await refused(quarantineCode)
|
|
await refused(quarantineCode, detachedAdmissionEndpoint)
|
|
await assert.rejects(beginDatabaseRestoreQuarantine(pool, otherId), { code: quarantineCode })
|
|
await assert.rejects(completeDatabaseRestoreQuarantine(pool, otherId), { code: quarantineCode })
|
|
await refused(quarantineCode)
|
|
assert.equal((await pool.query(readQuarantineId)).rows[0].operation_id, operationId)
|
|
assert.equal((await snapshot.query(readQuarantineId)).rows[0].operation_id, null)
|
|
await assert.rejects(admitPartitionWriter(snapshot), { code: quarantineCode })
|
|
await assert.rejects(writePartition(), error => {
|
|
assert(error instanceof PartitionWriteError)
|
|
assert.equal(error.code, "PARTITION_WRITE_UNAVAILABLE")
|
|
assert.equal(partitionWriteFailureDiagnostic(error)?.stage, "admission")
|
|
return true
|
|
})
|
|
assert.equal(writerCalls, 1)
|
|
const flowBatchId = randomUUID()
|
|
await assert.rejects(commitFlowWrite(
|
|
{ id: flowBatchId, kind: "ingest", sourceProof: "a".repeat(64) },
|
|
async () => assert.fail("Quarantined NetFlow batch entered its write callback"),
|
|
), { code: quarantineCode })
|
|
assert.equal((await pool.query("SELECT 1 FROM flow_write_receipts WHERE batch_id=$1", [flowBatchId])).rowCount, 0)
|
|
await assert.rejects(deleteTrafficFlowData(
|
|
() => pool.connect(), async () => assert.fail("Quarantined NetFlow purge entered its delete callback"),
|
|
), { code: quarantineCode })
|
|
await verifyRipeCacheWrites("quarantined")
|
|
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, afterDetached)
|
|
|
|
await withSystemDatabaseOperationLock(() => pool.connect(), async owner => {
|
|
await owner.query("BEGIN", [])
|
|
await owner.query("SET LOCAL synchronous_commit=on", [])
|
|
await completeDatabaseRestoreQuarantine(owner, operationId)
|
|
await owner.query("COMMIT", [])
|
|
})
|
|
await verifyRipeCacheWrites("reopened")
|
|
console.log("Independent PostgreSQL replica: RIPE prefix and ASN writes resumed only after exclusive partition locking and durable quarantine were cleared")
|
|
assert.equal(await writePartition(), afterDetached)
|
|
assert.equal(writerCalls, 2)
|
|
const recovered = await post()
|
|
assert.equal(recovered.status, 200)
|
|
assert.deepEqual(await recovered.json(), { audit_retention_days: afterDetached + 1 })
|
|
console.log("Independent PostgreSQL replica: the exclusive restore lock drained an admitted partition writer before target SQL; durable quarantine refused requests, detached work and partition writes after lock release; exact resolution reopened admission")
|
|
} finally {
|
|
if (child.connected) child.send("finish-detached")
|
|
try { await blocker.query("ROLLBACK") } finally { blocker.release() }
|
|
try { await snapshot?.query("ROLLBACK") } finally { snapshot?.release() }
|
|
await Promise.allSettled([first, maintenance, partitionWriter, quarantine])
|
|
if (previousQuarantine === undefined) delete process.env.DATABASE_RESTORE_QUARANTINE
|
|
else process.env.DATABASE_RESTORE_QUARANTINE = previousQuarantine
|
|
if (child.connected) child.send("stop")
|
|
try {
|
|
const [code] = await Promise.race([exited, delay(2500, undefined, { ref: false }).then(() => assert.fail("Replica did not shut down"))])
|
|
assert.equal(code, 0)
|
|
} catch (error) { child.kill(); throw error }
|
|
}
|
|
}
|