Files
MikrotikManager/backend/src/services/bandwidth-external-peer-metadata.sql-helper.ts
T

226 lines
18 KiB
TypeScript

/** Owned integration: genuine operator/window/source guards, real SQL owner and
* localhost license HTTP. Docker Q and configured catalog attestation are explicit
* synthetic fixture boundaries; this does not accept production quiescence. */
import assert from "node:assert/strict"
import { createCipheriv, createHash, generateKeyPairSync, randomBytes, randomUUID, sign } from "node:crypto"
import { copyFile, mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises"
import { dirname, resolve } from "node:path"
import { performance } from "node:perf_hooks"
import { setImmediate as immediate } from "node:timers/promises"
import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js"
import { canonicalBandwidthNodeValue as canonical } from "./bandwidth-node-bootstrap.js"
import { bandwidthOperatorEnrollmentGrantSchema, createBandwidthOperatorEnrollmentPrerequisite } from "./bandwidth-operator-enrollment.js"
import { createBandwidthOperatorOriginalAuthorityReader, createBandwidthOperatorExternalFencer, createBandwidthOperatorWriterWindows, type BandwidthOperatorWriterWindow } from "./bandwidth-operator-writer-window.js"
import { createProbeStorageConnectionConnector } from "./server-probe-storage-connection.js"
import { createConfiguredExternalBandwidthPeerSource } from "./bandwidth-external-peer-source.js"
import { createConfiguredManagedBandwidthPeerSource } from "./bandwidth-managed-peer-source.js"
import type { createOwnedExternalRegistryHarness } from "./bandwidth-external-node-registry.sql-helper.js"
type Harness = Awaited<ReturnType<typeof createOwnedExternalRegistryHarness>>
const sha = (value: Buffer | string) => createHash("sha256").update(value).digest("hex")
export async function acceptOwnedExternalPeerMetadata(f: Harness): Promise<readonly string[]> {
assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY, "true")
assert.equal(process.env.MMAPP_OWN_EXTERNAL_PEER_METADATA, "true")
const parent = resolve(process.env.MCP_STATE_DIR!), folder = await mkdtemp(resolve(parent, "external-peer-metadata-"))
assert.equal(dirname(folder), parent)
const authorityFolder = resolve(folder, "authority"), sourceFolder = resolve(folder, "source")
await mkdir(authorityFolder); await mkdir(sourceFolder)
const trustKeyFile = resolve(authorityFolder, "trust.pem"), grantFile = resolve(authorityFolder, "grant.json")
const profileFile = resolve(sourceFolder, "profile.json"), keyFile = resolve(sourceFolder, "management.key")
const keys = generateKeyPairSync("ed25519"), key = randomBytes(32), keySha256 = sha(key)
const publicBytes = keys.publicKey.export({ type: "spki", format: "pem" })
await writeFile(trustKeyFile, publicBytes, { flag: "wx", mode: 0o600 })
await writeFile(keyFile, key, { flag: "wx", mode: 0o600 })
const vector = { "system-id": "OWNED-EXTERNAL-PUBLIC-METADATA", "software-id": "OWNED-EXTERNAL-METADATA-SW" }
const enrollmentId = f.externalEndpoint(vector), peerRef = Object.freeze({ kind: "external" as const, enrollmentId })
const setup = await f.readPeerSnapshot(peerRef, f.scope()); assert(setup)
const endpoint = setup.endpoint
const header = { format: "mmapp-bandwidth-external-management-v1", installationId: f.root.installationId, enrollmentId,
generation: randomUUID(), enabled: true, routingEndpointEpoch: endpoint.routingEndpointEpoch, keySha256 }
const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv)
cipher.setAAD(Buffer.from(canonical(header)))
const encrypted = Buffer.concat([cipher.update(Buffer.from(canonical({ host: endpoint.host, port: endpoint.port, useSsl: endpoint.useSsl,
verifySsl: endpoint.verifySsl, username: endpoint.username, password: endpoint.password }))), cipher.final()])
const profile = { ...header, ciphertext: Buffer.concat([iv, cipher.getAuthTag(), encrypted]).toString("base64url") }
const profileBytes = Buffer.from(canonical(profile)), sourceProof = sha(profileBytes)
await writeFile(profileFile, profileBytes, { flag: "wx", mode: 0o600 }); key.fill(0)
const managedId = await f.managedEndpoint(vector), [managedRow] = await f.applicationRows([managedId])
assert(managedRow)
const managedConfig = { serverId: managedId, sourceProof: sha(randomBytes(32)), authorityGeneration: randomUUID(), routingEndpointEpoch: String(managedRow.routing_endpoint_epoch) }
const checks: string[] = [], coreTables = ["bandwidth_attempts", "bandwidth_node_occupancy", "bandwidth_attempt_events", "bandwidth_nodes", "bandwidth_node_aliases", "bandwidth_external_node_aliases"]
const baseline = await f.digest(coreTables), posts = f.nativePosts()
const connector = createProbeStorageConnectionConnector(async () => f.owner!.connect())
const gone = async (pid: number) => {
const end = performance.now() + 2000
for (;;) {
await f.owner!.query("SELECT pg_catalog.pg_stat_clear_snapshot()")
if ((await f.owner!.query("SELECT count(*)::int AS n FROM pg_catalog.pg_stat_activity WHERE pid=$1", [pid])).rows[0].n === 0) return
assert(performance.now() < end); await immediate()
}
}
try {
for (const mode of ["positive", "generation", "ciphertext", "directory-clone", "grant", "copies", "managed-positive", "physical-owner-loss", "physical-owner-during-Q"] as const) {
console.log("Owned external peer metadata stage:", mode)
await writeFile(profileFile, profileBytes)
const currentPeer = mode === "managed-positive" ? Object.freeze({ kind: "server" as const, serverId: managedId }) : peerRef
const binding = mode === "managed-positive" ? managedConfig : { sourceProof, authorityGeneration: header.generation, routingEndpointEpoch: header.routingEndpointEpoch }
const now = Date.now(), grant = bandwidthOperatorEnrollmentGrantSchema.parse({ format: "mmapp-bandwidth-operator-enrollment-v1", grantId: randomUUID(),
operatorId: "owned-external-metadata-operator", ...f.root, executorInventorySha256: sha("EXPLICIT OWNED SYNTHETIC DOCKER Q"), executorsStoppedAt: now - 1000,
issuedAt: now - 1000, expiresAt: now + 120000, peerRef: currentPeer, sourceProof: binding.sourceProof, authorityGeneration: binding.authorityGeneration,
routingEndpointEpoch: binding.routingEndpointEpoch, claims: Object.entries(vector).map(([kind, value]) => ({ kind, value })) })
const signed = Buffer.from(canonical(grant)), grantBytes = Buffer.from(JSON.stringify({ format: "mmapp-bandwidth-operator-enrollment-envelope-v1",
payload: signed.toString("base64url"), signature: sign(null, signed, keys.privateKey).toString("base64url") }))
await writeFile(grantFile, grantBytes, { mode: 0o600 })
const config = { trustKeyFile, grantFile, trustKeySha256: sha(keys.publicKey.export({ type: "spki", format: "der" })), ...f.root }
const originals = createBandwidthOperatorOriginalAuthorityReader(config)
const scope = Object.freeze({ deadlineAt: Date.now() + 5000 })
const original = await originals.acquire(scope)
const caseGets = f.nativeGets()
let q = 0, sql = 0, commits = 0, pid = 0, metadataPassed = false, negativeObserved = false
let lifetimeSignal: AbortSignal | undefined, armPostHeld = false, heldAcknowledged = false, injected = false
let injection: Promise<void> | undefined
let callbackStarted = false, completeCallback!: () => void
const callbackDone = new Promise<void>(resolve => { completeCallback = resolve })
const joinCallback = async () => {
if (!callbackStarted) return
let timer: ReturnType<typeof setTimeout> | undefined
try { await Promise.race([callbackDone, new Promise<never>((_resolve, reject) => { timer = setTimeout(() => reject(Error("Owned callback cleanup did not finish")), 3500) })]) }
finally { clearTimeout(timer) }
}
const terminateOwner = async () => {
assert(Number.isSafeInteger(pid) && pid > 0)
const terminated = await f.admin!.query("SELECT pg_catalog.pg_terminate_backend(pid) AS terminated FROM pg_catalog.pg_stat_activity WHERE pid=$1 AND datname=$2 AND usename=$3", [pid, f.databaseName, f.ownerRole])
assert.equal(terminated.command, "SELECT"); assert.equal(terminated.rowCount, 1)
assert.equal(terminated.rows.length, 1); assert.equal(terminated.rows[0].terminated, true)
}
const external = createBandwidthOperatorExternalFencer({ async acquire() { return { async assertCurrent() {
q++
if (mode === "physical-owner-during-Q" && armPostHeld && heldAcknowledged && !injection) {
injection = (async () => {
await terminateOwner(); await gone(pid)
assert(lifetimeSignal)
const signalEnd = Math.min(performance.now() + 1000, performance.now() + Math.max(0, scope.deadlineAt - Date.now()))
while (!lifetimeSignal.aborted) { assert(performance.now() < signalEnd); await immediate() }
assert(Date.now() < scope.deadlineAt, "transport must revoke before original deadline, not because it expired")
assert.equal(lifetimeSignal.aborted, true)
injected = true
})()
// Return this Q normally only after the actual transport lifetime is
// revoked. A preceding successful HELD row must no longer authorize.
await injection
}
}, close() {} } } })
let window: BandwidthOperatorWriterWindow | undefined
const windows = createBandwidthOperatorWriterWindows({ originals, external, source: {
async open(current, guard) {
await guard(); const raw = await connector(current)
lifetimeSignal = raw.lifetimeSignal
let closed = false
const close = () => { if (!closed) { closed = true; raw.release(true) } }
return Object.freeze({
connection: Object.freeze({ release: close, lifetimeSignal: raw.lifetimeSignal, async query(text: string, values?: unknown[], options?: { signal?: AbortSignal; timeoutMs: number }) {
await guard(); sql++; if (text === "COMMIT") commits++
const result = await raw.query(text, values, options)
if (armPostHeld && text === "SELECT pg_catalog.pg_backend_pid() AS backend_pid, EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() AND mode='ExclusiveLock' AND classid=0::pg_catalog.oid AND objid=8723116::pg_catalog.oid AND objsubid=1 AND granted) AS held") {
assert("command" in result); assert.equal(result.command, "SELECT")
assert.equal(result.rows.length, 1); assert.equal(result.rows[0].backend_pid, pid); assert.equal(result.rows[0].held, true)
heldAcknowledged = true
}
await guard()
if (text === "SELECT pg_catalog.pg_backend_pid() AS backend_pid") pid = Number(result.rows[0].backend_pid)
return result
} }), binding: async () => { await guard(); return Object.freeze({ ...f.root }) },
keyedProof: async () => { throw Error("Owned metadata fixture does not provide HMAC authority") }, close,
})
},
}, registry: { async register(_original, borrowed, current) {
callbackStarted = true
assert(window)
const guard = windows.createPeerSourceGuard(window, original)
const source = mode === "managed-positive" ? createConfiguredManagedBandwidthPeerSource(managedConfig, guard)
: createConfiguredExternalBandwidthPeerSource({ profileFile, keyFile, installationId: f.root.installationId, enrollmentId, keySha256 }, guard)
const coordinator = createBandwidthOperatorEnrollmentPrerequisite(config, source, { async verify(_grant, requested) { await windows.verify(window!, original, requested) } })
try {
if (mode === "physical-owner-loss") {
await terminateOwner()
const postCount = f.nativePosts()
await assert.rejects(coordinator.acquire(currentPeer, current))
// verifyRead awaits the genuine held-owner check before key read /
// decrypt. A socket error may retire the window even earlier.
assert.equal(commits, 0); assert.equal(f.nativeGets(), caseGets); assert.equal(f.nativePosts(), postCount)
negativeObserved = true
throw Error("Owned expected physical owner loss: no secret/native effect or COMMIT")
}
if (mode === "physical-owner-during-Q") {
armPostHeld = true
await assert.rejects(coordinator.acquire(currentPeer, current))
await injection; assert(heldAcknowledged && injected)
assert.equal(commits, 0); assert.equal(f.nativeGets(), caseGets); assert.equal(f.nativePosts(), posts)
negativeObserved = true
throw Error("Owned expected post-HELD owner loss: Q returns normally after transport revocation")
}
const prerequisite = await coordinator.acquire(currentPeer, current)
assert.equal(f.nativeGets(), caseGets + 1)
const ownership = await f.owner!.query("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE pid=$1 AND locktype='advisory' AND classid=0::pg_catalog.oid AND objid=8723116::pg_catalog.oid AND objsubid=1 AND mode='ExclusiveLock' AND granted) AS held", [pid])
assert.equal(ownership.rows[0].held, true)
const qBefore = q, sqlBefore = sql, getsBefore = f.nativeGets()
if (mode === "positive" || mode === "managed-positive") {
if (mode === "positive") await rename(keyFile, keyFile + ".held")
try {
const metadata = await coordinator.inspectMetadata(prerequisite, current)
assert.equal(metadata.format, "mmapp-bandwidth-operator-enrollment-metadata-v1")
assert.equal(metadata.metadata.evidence.peerRef.kind, currentPeer.kind)
assert.equal(JSON.stringify(metadata).includes('"endpoint"'), false)
assert.equal(JSON.stringify(metadata).includes(endpoint.password), false)
assert.equal(JSON.stringify(metadata).includes(endpoint.username), false)
assert.equal(q, qBefore); assert.equal(sql, sqlBefore); assert.equal(f.nativeGets(), getsBefore)
metadataPassed = true
} finally { if (mode === "positive") await rename(keyFile + ".held", keyFile) }
const began = await borrowed.connection.query("BEGIN", undefined, { timeoutMs: 1000 })
assert("command" in began); assert.equal(began.command, "BEGIN")
const committed = await borrowed.connection.query("COMMIT", undefined, { timeoutMs: 1000 })
assert("command" in committed); assert.equal(committed.command, "COMMIT")
borrowed.connection.release(true)
return "known-commit"
}
const inspect = () => coordinator.inspectMetadata(prerequisite, current)
if (mode === "generation") await writeFile(profileFile, canonical({ ...profile, generation: randomUUID() }))
if (mode === "ciphertext") await writeFile(profileFile, canonical({ ...profile, ciphertext: Buffer.concat([iv, randomBytes(16), encrypted]).toString("base64url") }))
if (mode === "grant") await writeFile(grantFile, "{}")
if (mode === "directory-clone") {
await rename(sourceFolder, sourceFolder + ".held"); await mkdir(sourceFolder)
try {
await copyFile(resolve(sourceFolder + ".held", "profile.json"), profileFile)
await copyFile(resolve(sourceFolder + ".held", "management.key"), keyFile)
await assert.rejects(inspect)
} finally { assert.equal(dirname(sourceFolder), folder); await rm(sourceFolder, { recursive: true }); await rename(sourceFolder + ".held", sourceFolder) }
} else if (mode === "copies") {
await assert.rejects(coordinator.inspectMetadata({ ...prerequisite }, current))
await assert.rejects(coordinator.inspectMetadata(JSON.parse(JSON.stringify(prerequisite)), current))
assert.throws(() => createConfiguredExternalBandwidthPeerSource({ profileFile, keyFile, installationId: f.root.installationId, enrollmentId, keySha256 }, { ...guard }))
} else await assert.rejects(inspect)
negativeObserved = true
assert.equal(sql, sqlBefore); assert.equal(q, qBefore); assert.equal(f.nativeGets(), getsBefore)
throw Error("Owned expected negative: no COMMIT or automatic replay")
} finally { try { coordinator.close(); source.close() } finally { completeCallback() } }
} } })
try {
window = await windows.open(original, scope)
if (mode === "positive" || mode === "managed-positive") { assert.equal(await windows.register(window), "known-commit"); assert(metadataPassed) }
else { await assert.rejects(windows.register(window)); await joinCallback(); assert(negativeObserved) }
} finally { windows.close(); originals.close(); await writeFile(grantFile, grantBytes); await writeFile(profileFile, profileBytes) }
assert(pid > 0); await gone(pid)
if (mode === "physical-owner-loss" || mode === "physical-owner-during-Q") assert.equal(commits, 0)
checks.push(mode === "positive" ? "genuine signed original/window guard, actual held PostgreSQL PID/session lock and localhost license GET; encrypted external metadata succeeds with key absent and zero Q/control-SQL/native/projection"
: mode === "managed-positive" ? "genuine managed guarded credentials use actual application row epoch; actual localhost GET and same physical owner; metadata fixed application boolean comparison adds no Q/control-SQL/native or credential projection"
: mode === "physical-owner-loss" ? "only exact owned database/owner/PID terminated; genuine window/source guard refuses before initial native GET and COMMIT; ordered read guard precedes key/decrypt; physical PID disappears"
: mode === "physical-owner-during-Q" ? "actual HELD ACK followed by pinned-owner termination during awaited Q; raw lifetime signal revoked before normal Q return; no key/decrypt admission, native GET or COMMIT"
: "owned external metadata refuses " + mode + " without new Q/SQL/native calls or automatic replay; physical owner retires")
}
assert.equal(await f.digest(coreTables), baseline); assert.equal(f.nativePosts(), posts)
return Object.freeze(checks)
} finally {
assert.equal(dirname(folder), parent); await rm(folder, { recursive: true })
}
}