import assert from "node:assert/strict" import { randomUUID, createHash, createHmac } from "node:crypto" import { readFile, writeFile } from "node:fs/promises" import path from "node:path" import type { PoolClient } from "pg" import { setTimeout as delay } from "node:timers/promises" import { pool, closePool } from "../../db/index.js" import { mcpConnectionCreateSchema } from "@mmapp/contracts/mcp" import { type AuthUser } from "../../lib/permissions.js" import { accessGeneration, mcpStateDirectory } from "./security.js" import { createMcpConnection, authenticateMcpToken, validatePrincipal, updateMcpSettings } from "./store.js" import { invokeMcpTool, approvePlan, applyPlan } from "./runtime.js" import { uploadRestoreArchive, readRestoreReceipt, executeMcpDatabaseRestore } from "./database-restore.js" import { exportSystemDatabaseBackup, restoreSystemDatabaseBackup } from "../../services/system-database-backup.js" import { initializeAutomaticSchedulerRoot } from "../../services/scheduler-automatic-root.js" import { sharedDatabaseGate } from "../../services/shared-database-gate.js" import { stopSchedulerForDatabaseOperation } from "../../services/scheduler.js" import { stopTrafficFlowListenerForDatabaseOperation } from "../../services/traffic-flow-ingest.js" import { acceptPortalRevocation, assertPortalClaimsCurrent } from "./portal-revocation.js" // This destructive test is only invoked by the private temporary-cluster runner. const url = new URL(process.env.DATABASE_URL ?? "postgresql://invalid") assert.equal(process.env.MMAPP_OWN_TERMINAL_RESTORE, "true") assert.equal(url.hostname, "127.0.0.1") assert.equal(url.pathname.slice(1), process.env.MMAPP_OWN_SQL_DB) assert.match(mcpStateDirectory(), /mm-mcp-test-/) // Standalone owned runner has no buildApp bootstrap; initialize before actor/request scopes. initializeAutomaticSchedulerRoot() const actor: AuthUser = { id: "restore-owner", email: "restore@example.invalid", name: "Restore acceptance", apps: ["mm"], permissions: ["mm:settings:admin"], expiresAt: Date.now() + 3600000 } const verifyData = process.env.MMAPP_OWN_TERMINAL_DATA_EQUIVALENCE === "true" let dataExpectation: Awaited> | undefined const originalConnect = pool.connect let migrationCaptures = 0 async function waitForDatabaseRetirement() { let timer: ReturnType | undefined try { await Promise.race([sharedDatabaseGate.whenOpen(), new Promise((_, reject) => { timer = setTimeout(() => reject(Error("Terminal restore did not retire its database admission")), 10000) })]) } finally { clearTimeout(timer) } } try { // Every owned restore profile archives inert domain collectors, not only the // data-equivalence profile. Otherwise restart can launch unrelated downloads // or router polling and race the final scheduler drain. Lifecycle SQL remains // real; production scheduler configuration and drain checks are unchanged. for (const table of ["traffic_settings", "uptime_settings", "traffic_flow_settings", "servers_api_ping_settings", "certificate_renew_settings", "backup_schedule_settings", "geoip_settings", "internet_path_settings", "router_log_settings"]) await pool.query(`UPDATE ${table} SET enabled=false WHERE id=1`) await pool.query("UPDATE uptime_settings SET resources_enabled=false,ping_enabled=false,speed_enabled=false WHERE id=1") await pool.query("DELETE FROM mcp_protections") await updateMcpSettings({ enabled: true }, actor) await pool.query("CREATE TABLE mcp_restore_acceptance_marker (value text NOT NULL)") await pool.query("INSERT INTO mcp_restore_acceptance_marker VALUES('before')") const serverIds = (await pool.query("SELECT id FROM servers")).rows.map(row => Number(row.id)) // Combined owned profiles retain their evidence/disabled synthetic servers. // The human restore fixture grants this exact catalog instead of inheriting // the ordinary five-server default; production policy guards stay unchanged. assert(serverIds.length <= 1000) const issue = (owner = actor) => createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Database restore acceptance", mode: "approval", expiresAt: new Date(Date.now() + 1800000).toISOString(), permissions: ["mm:settings:admin"], scope: { allowGlobal: true, serverIds, operationIds: ["database.restore"] }, limits: { serversPerPlan: Math.max(1, serverIds.length) } }), owner) const cutoff = Date.now() - 1000 const revokedOwner = { ...actor, id: "restore-revocation-after-backup", issuedAt: cutoff - 1 } const oldAccess = await issue(revokedOwner), oldPrincipal = await authenticateMcpToken(oldAccess.token) const readHistoricalAccess = async () => (await pool.query<{ status: string; credential_current: boolean; generation: string }>( "SELECT c.policy->>'status' AS status, k.revoked_at IS NULL AND k.expires_at>now() AS credential_current, c.generation FROM mcp_connections c JOIN mcp_credentials k ON k.connection_id=c.id WHERE c.id=$1 AND k.id=$2", [oldPrincipal.connectionId, oldPrincipal.credentialId], )).rows[0] const beforeDumpAccess = await readHistoricalAccess() assert.deepEqual(beforeDumpAccess, { status: "active", credential_current: true, generation: await accessGeneration() }) const dump = await exportSystemDatabaseBackup() assert.equal(dump.buffer.subarray(0, 5).toString(), "PGDMP") if (verifyData) dataExpectation = await (await import("./database-restore-data-equivalence.sql-helper.js")).prepareOwnedRestoreDataExpectation(dump.buffer) const portalSecret = "restore-acceptance-only-portal-secret-32", stamp = String(Math.floor(Date.now() / 1000)) const eventId = randomUUID() const portalEvent = Buffer.from(JSON.stringify({ version: 2, eventId, userId: revokedOwner.id, type: "permissions.changed", eventOccurredAt: cutoff })) assert.deepEqual(await acceptPortalRevocation(portalEvent, stamp, "sha256=" + createHmac("sha256", portalSecret).update(stamp + ".").update(portalEvent).digest("hex"), portalSecret), { ok: true, duplicate: false }) const cutoffFile = path.join(mcpStateDirectory(), "owner-revocations", createHash("sha256").update(revokedOwner.id).digest("hex") + ".json") const cutoffBytes = await readFile(cutoffFile) assert.deepEqual(JSON.parse(cutoffBytes.toString("utf8")), { invalidBefore: cutoff }) assert.deepEqual(await readHistoricalAccess(), { ...beforeDumpAccess, status: "revoked", credential_current: false }) await assert.rejects(validatePrincipal(oldPrincipal), { code: "OWNER_CLAIMS_REVOKED" }) const source = await uploadRestoreArchive(dump.buffer, actor) const encrypted = await readFile(path.join(mcpStateDirectory(), "restore-vault", source.archiveRef + ".archive.bin")) assert.ok(!encrypted.equals(dump.buffer)) assert.ok(!encrypted.subarray(0, 5).equals(Buffer.from("PGDMP"))) await assert.rejects(uploadRestoreArchive(dump.buffer, { ...actor, permissions: ["mm:mcp:admin"] })) await assert.rejects(uploadRestoreArchive(Buffer.from("invalid"), actor)) const issued = await issue(), principal = await authenticateMcpToken(issued.token) const other = await issue({ ...actor, id: "another-owner" }), otherPrincipal = await authenticateMcpToken(other.token) await assert.rejects(invokeMcpTool(otherPrincipal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: source.sha256 })) await assert.rejects(invokeMcpTool(principal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: "0".repeat(64) })) const prepared = await invokeMcpTool(principal, "mm_database_restore_plan", { archiveRef: source.archiveRef, expectedSha256: source.sha256 }) as { planId: string; state: string } assert.equal(prepared.state, "awaiting-approval") await assert.rejects(applyPlan(principal, prepared.planId, "restore-acceptance")) await approvePlan(prepared.planId, actor, true) // Legacy direct restore must respect protected objects in every domain. const protection = randomUUID() await pool.query("INSERT INTO mcp_protections(id,object_ref,reason,created_by) VALUES($1,$2::jsonb,'fixture',$3)", [protection, JSON.stringify({ kind: "user", id: "protected-user", protection: "deny" }), actor.id]) await assert.rejects(restoreSystemDatabaseBackup(dump.buffer)) await pool.query("DELETE FROM mcp_protections WHERE id=$1", [protection]) await pool.query("UPDATE mcp_restore_acceptance_marker SET value='after'") const generation = await accessGeneration() // Emulates the admitted HTTP apply request: runner must wait, never deadlock it. const restoreStartedAt = Date.now() if (dataExpectation) { // Owned-test observation only: capture after migrations release their // advisory lease, before the production restore resumes consumers. pool.connect = function (...args: unknown[]) { if (args.length) return Reflect.apply(originalConnect, pool, args) return (Reflect.apply(originalConnect, pool, []) as Promise).then(client => { let migrationClient = false return new Proxy(client, { get(target, property) { if (property === "query") return (...queryArgs: unknown[]) => { const sql = queryArgs[0] if (typeof sql === "string" && sql.includes("CREATE TABLE IF NOT EXISTS schema_migrations")) migrationClient = true const result = Reflect.apply(target.query, target, queryArgs) if (migrationClient && sql === "RESET statement_timeout" && queryArgs.length === 1) { return Promise.resolve(result).then(async value => { assert.equal(++migrationCaptures, 1) await dataExpectation!.compareBeforeConsumers() return value }) } return result } const value = Reflect.get(target, property, target) return typeof value === "function" ? value.bind(target) : value }, }) }) } as typeof pool.connect } const operation = await sharedDatabaseGate.request(() => applyPlan(principal, prepared.planId, "restore-acceptance")) let receipt: Record = {} for (let i = 0; i < 600; i++) { receipt = await readRestoreReceipt(operation.id, actor).catch(() => ({})) if (["applied", "unknown", "blocked"].includes(String(receipt.state))) break await delay(100) } assert.equal(receipt.state, "applied", JSON.stringify(receipt)) assert.equal(receipt.accessRevoked, true) pool.connect = originalConnect if (dataExpectation) { await stopSchedulerForDatabaseOperation() await stopTrafficFlowListenerForDatabaseOperation() await (await import("../../services/database-background-work.js")).databaseBackgroundWork.drain() const dataInput = { operationId: operation.id, receipt, actorId: actor.id, startedAt: restoreStartedAt } await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, actorId: "foreign-owner" })) await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, operationId: randomUUID() })) await assert.rejects(dataExpectation.compareAfterRestore({ ...dataInput, receipt: { ...receipt, accessRevoked: false } })) console.log("Owned terminal restore data equivalence:", JSON.stringify(await dataExpectation.compareAfterRestore(dataInput))) assert.equal(migrationCaptures, 1) assert.equal((await dataExpectation.compareAfterRestore(dataInput)).auditRecordsObserved, 1) } assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "before") assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_operations WHERE id=$1", [operation.id])).rows[0].n, 0) assert.notEqual(await accessGeneration(), generation) assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_owner_revocations WHERE owner_id=$1", [revokedOwner.id])).rows[0].n, 0) assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_portal_events WHERE id=$1", [eventId])).rows[0].n, 0) assert.deepEqual(await readHistoricalAccess(), beforeDumpAccess) assert.deepEqual(await readFile(cutoffFile), cutoffBytes) // Both paths must report the external owner cutoff, independently of the // blanket generation change that also invalidates older connections. await assert.rejects(assertPortalClaimsCurrent(revokedOwner), { code: "OWNER_CLAIMS_REVOKED" }) await assert.rejects(authenticateMcpToken(oldAccess.token), { code: "OWNER_CLAIMS_REVOKED" }) await assert.rejects(validatePrincipal(oldPrincipal), { code: "OWNER_CLAIMS_REVOKED" }) await assert.rejects(authenticateMcpToken(issued.token)) await assert.rejects(readRestoreReceipt(operation.id, { ...actor, permissions: ["mm:mcp:read"] })) // The applied receipt is durable before the terminal callback releases its // last SQL lease/exclusive admission. Await that existing owner, rather than // racing its cleanup with the stale-callback/no-replay assertion below. await waitForDatabaseRetirement() await assert.rejects(issue(revokedOwner), { code: "OWNER_CLAIMS_REVOKED" }) const refreshedOwnerAccess = await issue({ ...revokedOwner, issuedAt: cutoff + 1 }) const refreshedOwner = await validatePrincipal(await authenticateMcpToken(refreshedOwnerAccess.token)) assert.equal(refreshedOwner.owner_id, revokedOwner.id) assert.equal(refreshedOwner.generation, await accessGeneration()) assert.notEqual(refreshedOwner.generation, beforeDumpAccess.generation) // A stale callback or process replay cannot invoke pg_restore a second time. await pool.query("UPDATE mcp_restore_acceptance_marker SET value='after-receipt'") await executeMcpDatabaseRestore(operation.id, principal) assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "after-receipt") assert.equal((await readRestoreReceipt(operation.id, actor)).state, "applied") if (dataExpectation) { console.log("Owned restore data negative cases: foreign owner, wrong operation and unconfirmed receipt refused; repeat validation writes no audit; full equality observed before consumers resume") } console.log("✓ Terminal restore: real pg_dump/pg_restore transaction, encrypted human upload, approval, global protection, request drain, external receipt, restored live credential refused by external Portal cutoff, post-cutoff claims admitted and no replay") // Archive authentication is checked before a terminal attempt can replace data. await updateMcpSettings({ enabled: true }, actor) const next = await issue(), nextPrincipal = await authenticateMcpToken(next.token), tampered = await uploadRestoreArchive(dump.buffer, actor) const badPlan = await invokeMcpTool(nextPrincipal, "mm_database_restore_plan", { archiveRef: tampered.archiveRef, expectedSha256: tampered.sha256 }) as { planId: string } await approvePlan(badPlan.planId, actor, true) await writeFile(path.join(mcpStateDirectory(), "restore-vault", tampered.archiveRef + ".archive.bin"), Buffer.from("corrupted")) const generationBeforeCorruption = await accessGeneration() const bad = await applyPlan(nextPrincipal, badPlan.planId, "corrupted-archive") // Admission, durable receipt writes and authority checks are asynchronous. // Use the same terminal-state budget as the successful restore above; never // interpret a still-running receipt as a rejection or dispatch apply again. for (let i = 0; i < 600; i++) { receipt = await readRestoreReceipt(bad.id, actor).catch(() => ({})) if (["blocked", "applied", "unknown"].includes(String(receipt.state))) break await delay(100) } assert.equal(receipt.state, "blocked", JSON.stringify(receipt)) await waitForDatabaseRetirement() assert.equal(await accessGeneration(), generationBeforeCorruption) assert.equal((await pool.query("SELECT value FROM mcp_restore_acceptance_marker")).rows[0].value, "after-receipt") console.log("✓ Tampered archive is blocked before data replacement") } finally { pool.connect = originalConnect await stopSchedulerForDatabaseOperation() await stopTrafficFlowListenerForDatabaseOperation() await dataExpectation?.close() await closePool() }