Files
MikrotikManager/backend/src/test/database-rest-replica.ts
T

279 lines
14 KiB
TypeScript

import assert from "node:assert/strict"
import { fork } from "node:child_process"
import { once } from "node:events"
import { setTimeout as delay } from "node:timers/promises"
import { fileURLToPath } from "node:url"
import Fastify from "fastify"
import type { PoolClient } from "pg"
import { closePool, pool } from "../db/index.js"
import { SYSTEM_DATABASE_REQUEST_LOCK, withSystemDatabaseOperationLock } from "../db/operation-lock.js"
import { randomUUID } from "node:crypto"
import { withDetachedDatabaseApiRequest } from "../modules/mcp/database-request-lease.js"
import {
DATABASE_RESTORE_QUARANTINE_DDL, beginDatabaseRestoreQuarantine, completeDatabaseRestoreQuarantine,
} from "../db/database-restore-quarantine.js"
import { installDatabaseRequestGate } from "../plugins/database-request-gate.js"
import { withConfigMutationLock } from "../services/config-mutation-lock.js"
import { databaseBackgroundWork } from "../services/database-background-work.js"
import {
admitPartitionWriter, PARTITION_WRITER_ADVISORY_KEY, PartitionWriteError, partitionWriteFailureDiagnostic, withPartitionWrite,
} from "../services/partition-write-transaction.js"
import { deleteTrafficFlowData } from "../services/traffic-flow-purge-execution.js"
import { enqueueRipeMisses, lookupRipeCached, resetRipeCacheForTests, setRipeFetchForTests } from "../services/traffic-flow-ripe.js"
import { commitFlowWrite } from "../services/traffic-flow-write-transaction.js"
const endpoint = "/api/owned-replica"
const detachedEndpoint = "/api/owned-replica/detached"
const detachedAdmissionEndpoint = "/owned-replica/detached-admission"
const readSql = "SELECT audit_retention_days FROM mcp_settings WHERE id=1"
const lockedRead = readSql + " FOR UPDATE"
const finishDetachedSql = "UPDATE mcp_settings SET audit_retention_days=audit_retention_days+1 WHERE id=1"
const pendingExclusiveLock = "SELECT 1 FROM pg_locks WHERE locktype='advisory' AND classid=0 AND objid=$1::oid AND objsubid=1 AND mode='ExclusiveLock' AND NOT granted"
const quarantineCode = "DATABASE_RESTORE_QUARANTINE_UNAVAILABLE"
const readQuarantineId = "SELECT operation_id FROM mmapp_control.database_restore_quarantine WHERE id=1"
if (process.argv.includes("--replica-child")) {
assert(process.send && process.env.MMAPP_OWN_REST_REPLICA_ONLY === "true")
let finishDetached!: () => void
const detachedLatch = new Promise<void>(resolve => { finishDetached = resolve })
let detached: Promise<void> | undefined
const app = Fastify()
installDatabaseRequestGate(app)
app.get("/health", async () => ({ ok: true }))
app.post(endpoint, async () => {
await pool.query(lockedRead)
return withConfigMutationLock([721], async () => {
const { rows } = await pool.query("UPDATE mcp_settings SET audit_retention_days=audit_retention_days+1 WHERE id=1 RETURNING audit_retention_days")
return rows[0]
})
})
app.post(detachedEndpoint, async (_request, reply) => {
detached = withDetachedDatabaseApiRequest(async () => {
process.send?.("detached-started")
await detachedLatch
await pool.query(finishDetachedSql)
})
void detached.catch(error => console.error("Detached replica work failed", error))
reply.code(202)
return { accepted: true }
})
// Bypass the HTTP gate to exercise the detached operation's own quarantine check.
app.post(detachedAdmissionEndpoint, () => withDetachedDatabaseApiRequest(async () => {
assert.fail("Quarantined detached work entered its callback")
}))
const address = await app.listen({ host: "127.0.0.1", port: 0 })
process.on("message", async message => {
if (message === "finish-detached") { finishDetached(); return }
if (message !== "stop") return
await app.close()
await detached?.catch(() => {})
await closePool()
process.disconnect?.()
})
process.send(address)
}
async function verifyRipeCacheWrites(phase: "partition lock held" | "quarantined" | "reopened") {
const ip = "8.8.8.8", prefix = `${ip}/32`, asn = 64512, holder = "Owned RIPE fixture"
const canPersist = phase === "reopened"
setRipeFetchForTests(async input => {
const { pathname } = new URL(String(input))
if (pathname === "/data/network-info/data.json") return Response.json({ data: { prefix, asns: [asn] } })
if (pathname === "/data/maxmind-geo-lite/data.json") return Response.json({ data: {} })
assert.equal(pathname, "/data/as-overview/data.json")
return Response.json({ data: { holder } })
})
try {
enqueueRipeMisses([ip])
await databaseBackgroundWork.drain(4000)
const entry = lookupRipeCached(ip)
assert.equal(entry?.holder, holder, `RIPE resolution with admission ${phase}`)
assert.equal(entry?.asn, asn)
assert.deepEqual((await pool.query("SELECT asn, holder FROM flow_ip_meta WHERE prefix=$1", [prefix])).rows,
canPersist ? [{ asn, holder }] : [], `RIPE prefix persistence with admission ${phase}`)
assert.deepEqual((await pool.query("SELECT holder FROM flow_asn_meta WHERE asn=$1", [asn])).rows,
canPersist ? [{ holder }] : [], `RIPE ASN persistence with admission ${phase}`)
} finally {
resetRipeCacheForTests()
}
}
export async function verifyDatabaseRestReplica() {
await pool.query("CREATE SCHEMA IF NOT EXISTS mmapp_control")
await pool.query(DATABASE_RESTORE_QUARANTINE_DDL)
const before = (await pool.query(readSql)).rows[0].audit_retention_days
const child = fork(fileURLToPath(import.meta.url), ["--replica-child"], {
execArgv: ["--import", "tsx"], stdio: ["ignore", "ignore", "inherit", "ipc"],
env: { ...process.env, DATABASE_RESTORE_QUARANTINE: "true" },
})
const exited = once(child, "exit")
const signal = AbortSignal.timeout(15000)
const blocker = await pool.connect()
const previousQuarantine = process.env.DATABASE_RESTORE_QUARANTINE
let snapshot: PoolClient | undefined
let first: Promise<Response> | undefined, maintenance: Promise<void> | undefined
let partitionWriter: Promise<number> | undefined, quarantine: Promise<void> | undefined
try {
const [address] = await Promise.race([
once(child, "message", { signal }),
exited.then(() => assert.fail("Replica exited before readiness")),
])
assert.equal(typeof address, "string")
const post = (path = endpoint) => fetch(address + path, { method: "POST", signal })
const waitFor = async (sql: string, args: unknown[]) => {
while (!(await pool.query(sql, args)).rowCount) await delay(15, undefined, { signal })
}
const refused = async (code = "PROTOCOL_STORAGE_BUSY", path = endpoint) => {
const response = await post(path)
assert.equal(response.status, 503)
const body = await response.json()
assert.ok(body && typeof body === "object" && "code" in body)
assert.equal(body.code, code)
}
await blocker.query("BEGIN")
await blocker.query(lockedRead)
first = post()
void first.catch(() => {})
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [lockedRead])
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, before + 1)
await refused()
assert.equal((await fetch(address + "/health", { signal })).status, 200)
})
void maintenance.catch(() => {})
await waitFor(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])
await refused()
await blocker.query("ROLLBACK")
const response = await first
assert.equal(response.status, 200)
assert.deepEqual(await response.json(), { audit_retention_days: before + 1 })
await maintenance
const resumed = await post()
assert.equal(resumed.status, 200)
assert.deepEqual(await resumed.json(), { audit_retention_days: before + 2 })
console.log("Independent PostgreSQL replica: queued maintenance drained the active REST request and its nested mutation, denied new requests, then reopened admission")
await blocker.query("BEGIN")
await blocker.query(lockedRead)
const [accepted, [started]] = await Promise.all([
post(detachedEndpoint), once(child, "message", { signal }),
])
assert.equal(accepted.status, 202)
assert.deepEqual(await accepted.json(), { accepted: true })
assert.equal(started, "detached-started")
const afterDetached = before + 3
maintenance = withSystemDatabaseOperationLock(() => pool.connect(), async () => {
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, afterDetached)
})
void maintenance.catch(() => {})
await waitFor(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, before + 2)
child.send("finish-detached")
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [finishDetachedSql])
assert.ok((await pool.query(pendingExclusiveLock, [SYSTEM_DATABASE_REQUEST_LOCK])).rowCount)
await blocker.query("ROLLBACK")
await maintenance
console.log("Independent PostgreSQL replica: detached work held admission after HTTP 202 through its final committed UPDATE")
process.env.DATABASE_RESTORE_QUARANTINE = "true"
await blocker.query("BEGIN")
await blocker.query("SELECT pg_catalog.pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
await verifyRipeCacheWrites("partition lock held")
await blocker.query("ROLLBACK")
snapshot = await pool.connect()
await snapshot.query("BEGIN ISOLATION LEVEL REPEATABLE READ")
assert.equal((await snapshot.query(readQuarantineId)).rows[0].operation_id, null)
let writerCalls = 0
const writePartition = () => withPartitionWrite(async ({ client }) => {
writerCalls++
return (await client.query<{ audit_retention_days: number }>(lockedRead)).rows[0].audit_retention_days
}, { signal })
await blocker.query("BEGIN")
await blocker.query(lockedRead)
partitionWriter = writePartition()
void partitionWriter.catch(() => {})
await waitFor("SELECT 1 FROM pg_stat_activity WHERE query=$1 AND wait_event_type='Lock'", [lockedRead])
const operationId = randomUUID(), otherId = randomUUID()
let restoreTargetReached = false
quarantine = withSystemDatabaseOperationLock(() => pool.connect(), async owner => {
await owner.query("BEGIN", [])
await owner.query("SET LOCAL synchronous_commit=on", [])
await beginDatabaseRestoreQuarantine(owner, operationId)
await owner.query("COMMIT", [])
await owner.query("BEGIN", [])
try {
await owner.query("SET LOCAL lock_timeout='5s'", [])
await owner.query("SELECT pg_catalog.pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
await owner.query(readSql, [])
restoreTargetReached = true
} finally {
await owner.query("ROLLBACK", [])
}
})
void quarantine.catch(() => {})
await waitFor(pendingExclusiveLock + " AND EXISTS (SELECT 1 FROM pg_stat_activity AS writer WHERE writer.pid=ANY(pg_blocking_pids(pg_locks.pid)) AND writer.query=$2)", [PARTITION_WRITER_ADVISORY_KEY, lockedRead])
assert.equal(restoreTargetReached, false)
await blocker.query("ROLLBACK")
assert.equal(await partitionWriter, afterDetached)
await quarantine
assert.equal(restoreTargetReached, true)
await refused(quarantineCode)
await refused(quarantineCode, detachedAdmissionEndpoint)
await assert.rejects(beginDatabaseRestoreQuarantine(pool, otherId), { code: quarantineCode })
await assert.rejects(completeDatabaseRestoreQuarantine(pool, otherId), { code: quarantineCode })
await refused(quarantineCode)
assert.equal((await pool.query(readQuarantineId)).rows[0].operation_id, operationId)
assert.equal((await snapshot.query(readQuarantineId)).rows[0].operation_id, null)
await assert.rejects(admitPartitionWriter(snapshot), { code: quarantineCode })
await assert.rejects(writePartition(), error => {
assert(error instanceof PartitionWriteError)
assert.equal(error.code, "PARTITION_WRITE_UNAVAILABLE")
assert.equal(partitionWriteFailureDiagnostic(error)?.stage, "admission")
return true
})
assert.equal(writerCalls, 1)
const flowBatchId = randomUUID()
await assert.rejects(commitFlowWrite(
{ id: flowBatchId, kind: "ingest", sourceProof: "a".repeat(64) },
async () => assert.fail("Quarantined NetFlow batch entered its write callback"),
), { code: quarantineCode })
assert.equal((await pool.query("SELECT 1 FROM flow_write_receipts WHERE batch_id=$1", [flowBatchId])).rowCount, 0)
await assert.rejects(deleteTrafficFlowData(
() => pool.connect(), async () => assert.fail("Quarantined NetFlow purge entered its delete callback"),
), { code: quarantineCode })
await verifyRipeCacheWrites("quarantined")
assert.equal((await pool.query(readSql)).rows[0].audit_retention_days, afterDetached)
await withSystemDatabaseOperationLock(() => pool.connect(), async owner => {
await owner.query("BEGIN", [])
await owner.query("SET LOCAL synchronous_commit=on", [])
await completeDatabaseRestoreQuarantine(owner, operationId)
await owner.query("COMMIT", [])
})
await verifyRipeCacheWrites("reopened")
console.log("Independent PostgreSQL replica: RIPE prefix and ASN writes resumed only after exclusive partition locking and durable quarantine were cleared")
assert.equal(await writePartition(), afterDetached)
assert.equal(writerCalls, 2)
const recovered = await post()
assert.equal(recovered.status, 200)
assert.deepEqual(await recovered.json(), { audit_retention_days: afterDetached + 1 })
console.log("Independent PostgreSQL replica: the exclusive restore lock drained an admitted partition writer before target SQL; durable quarantine refused requests, detached work and partition writes after lock release; exact resolution reopened admission")
} finally {
if (child.connected) child.send("finish-detached")
try { await blocker.query("ROLLBACK") } finally { blocker.release() }
try { await snapshot?.query("ROLLBACK") } finally { snapshot?.release() }
await Promise.allSettled([first, maintenance, partitionWriter, quarantine])
if (previousQuarantine === undefined) delete process.env.DATABASE_RESTORE_QUARANTINE
else process.env.DATABASE_RESTORE_QUARANTINE = previousQuarantine
if (child.connected) child.send("stop")
try {
const [code] = await Promise.race([exited, delay(2500, undefined, { ref: false }).then(() => assert.fail("Replica did not shut down"))])
assert.equal(code, 0)
} catch (error) { child.kill(); throw error }
}
}