/** Owned integration: genuine operator/window/source guards, real SQL owner and * localhost license HTTP. Docker Q and configured catalog attestation are explicit * synthetic fixture boundaries; this does not accept production quiescence. */ import assert from "node:assert/strict" import { createCipheriv, createHash, generateKeyPairSync, randomBytes, randomUUID, sign } from "node:crypto" import { copyFile, mkdir, mkdtemp, rename, rm, writeFile } from "node:fs/promises" import { dirname, resolve } from "node:path" import { performance } from "node:perf_hooks" import { setImmediate as immediate } from "node:timers/promises" import { assertOwnedBandwidthSql } from "./bandwidth-admission-process.sql-helper.js" import { canonicalBandwidthNodeValue as canonical } from "./bandwidth-node-bootstrap.js" import { bandwidthOperatorEnrollmentGrantSchema, createBandwidthOperatorEnrollmentPrerequisite } from "./bandwidth-operator-enrollment.js" import { createBandwidthOperatorOriginalAuthorityReader, createBandwidthOperatorExternalFencer, createBandwidthOperatorWriterWindows, type BandwidthOperatorWriterWindow } from "./bandwidth-operator-writer-window.js" import { createProbeStorageConnectionConnector } from "./server-probe-storage-connection.js" import { createConfiguredExternalBandwidthPeerSource } from "./bandwidth-external-peer-source.js" import { createConfiguredManagedBandwidthPeerSource } from "./bandwidth-managed-peer-source.js" import type { createOwnedExternalRegistryHarness } from "./bandwidth-external-node-registry.sql-helper.js" type Harness = Awaited> const sha = (value: Buffer | string) => createHash("sha256").update(value).digest("hex") export async function acceptOwnedExternalPeerMetadata(f: Harness): Promise { assertOwnedBandwidthSql(); assert.equal(process.env.MMAPP_OWN_BANDWIDTH_EXTERNAL_REGISTRY, "true") assert.equal(process.env.MMAPP_OWN_EXTERNAL_PEER_METADATA, "true") const parent = resolve(process.env.MCP_STATE_DIR!), folder = await mkdtemp(resolve(parent, "external-peer-metadata-")) assert.equal(dirname(folder), parent) const authorityFolder = resolve(folder, "authority"), sourceFolder = resolve(folder, "source") await mkdir(authorityFolder); await mkdir(sourceFolder) const trustKeyFile = resolve(authorityFolder, "trust.pem"), grantFile = resolve(authorityFolder, "grant.json") const profileFile = resolve(sourceFolder, "profile.json"), keyFile = resolve(sourceFolder, "management.key") const keys = generateKeyPairSync("ed25519"), key = randomBytes(32), keySha256 = sha(key) const publicBytes = keys.publicKey.export({ type: "spki", format: "pem" }) await writeFile(trustKeyFile, publicBytes, { flag: "wx", mode: 0o600 }) await writeFile(keyFile, key, { flag: "wx", mode: 0o600 }) const vector = { "system-id": "OWNED-EXTERNAL-PUBLIC-METADATA", "software-id": "OWNED-EXTERNAL-METADATA-SW" } const enrollmentId = f.externalEndpoint(vector), peerRef = Object.freeze({ kind: "external" as const, enrollmentId }) const setup = await f.readPeerSnapshot(peerRef, f.scope()); assert(setup) const endpoint = setup.endpoint const header = { format: "mmapp-bandwidth-external-management-v1", installationId: f.root.installationId, enrollmentId, generation: randomUUID(), enabled: true, routingEndpointEpoch: endpoint.routingEndpointEpoch, keySha256 } const iv = randomBytes(12), cipher = createCipheriv("aes-256-gcm", key, iv) cipher.setAAD(Buffer.from(canonical(header))) const encrypted = Buffer.concat([cipher.update(Buffer.from(canonical({ host: endpoint.host, port: endpoint.port, useSsl: endpoint.useSsl, verifySsl: endpoint.verifySsl, username: endpoint.username, password: endpoint.password }))), cipher.final()]) const profile = { ...header, ciphertext: Buffer.concat([iv, cipher.getAuthTag(), encrypted]).toString("base64url") } const profileBytes = Buffer.from(canonical(profile)), sourceProof = sha(profileBytes) await writeFile(profileFile, profileBytes, { flag: "wx", mode: 0o600 }); key.fill(0) const managedId = await f.managedEndpoint(vector), [managedRow] = await f.applicationRows([managedId]) assert(managedRow) const managedConfig = { serverId: managedId, sourceProof: sha(randomBytes(32)), authorityGeneration: randomUUID(), routingEndpointEpoch: String(managedRow.routing_endpoint_epoch) } const checks: string[] = [], coreTables = ["bandwidth_attempts", "bandwidth_node_occupancy", "bandwidth_attempt_events", "bandwidth_nodes", "bandwidth_node_aliases", "bandwidth_external_node_aliases"] const baseline = await f.digest(coreTables), posts = f.nativePosts() const connector = createProbeStorageConnectionConnector(async () => f.owner!.connect()) const gone = async (pid: number) => { const end = performance.now() + 2000 for (;;) { await f.owner!.query("SELECT pg_catalog.pg_stat_clear_snapshot()") if ((await f.owner!.query("SELECT count(*)::int AS n FROM pg_catalog.pg_stat_activity WHERE pid=$1", [pid])).rows[0].n === 0) return assert(performance.now() < end); await immediate() } } try { for (const mode of ["positive", "generation", "ciphertext", "directory-clone", "grant", "copies", "managed-positive", "physical-owner-loss", "physical-owner-during-Q"] as const) { console.log("Owned external peer metadata stage:", mode) await writeFile(profileFile, profileBytes) const currentPeer = mode === "managed-positive" ? Object.freeze({ kind: "server" as const, serverId: managedId }) : peerRef const binding = mode === "managed-positive" ? managedConfig : { sourceProof, authorityGeneration: header.generation, routingEndpointEpoch: header.routingEndpointEpoch } const now = Date.now(), grant = bandwidthOperatorEnrollmentGrantSchema.parse({ format: "mmapp-bandwidth-operator-enrollment-v1", grantId: randomUUID(), operatorId: "owned-external-metadata-operator", ...f.root, executorInventorySha256: sha("EXPLICIT OWNED SYNTHETIC DOCKER Q"), executorsStoppedAt: now - 1000, issuedAt: now - 1000, expiresAt: now + 120000, peerRef: currentPeer, sourceProof: binding.sourceProof, authorityGeneration: binding.authorityGeneration, routingEndpointEpoch: binding.routingEndpointEpoch, claims: Object.entries(vector).map(([kind, value]) => ({ kind, value })) }) const signed = Buffer.from(canonical(grant)), grantBytes = Buffer.from(JSON.stringify({ format: "mmapp-bandwidth-operator-enrollment-envelope-v1", payload: signed.toString("base64url"), signature: sign(null, signed, keys.privateKey).toString("base64url") })) await writeFile(grantFile, grantBytes, { mode: 0o600 }) const config = { trustKeyFile, grantFile, trustKeySha256: sha(keys.publicKey.export({ type: "spki", format: "der" })), ...f.root } const originals = createBandwidthOperatorOriginalAuthorityReader(config) const scope = Object.freeze({ deadlineAt: Date.now() + 5000 }) const original = await originals.acquire(scope) const caseGets = f.nativeGets() let q = 0, sql = 0, commits = 0, pid = 0, metadataPassed = false, negativeObserved = false let lifetimeSignal: AbortSignal | undefined, armPostHeld = false, heldAcknowledged = false, injected = false let injection: Promise | undefined let callbackStarted = false, completeCallback!: () => void const callbackDone = new Promise(resolve => { completeCallback = resolve }) const joinCallback = async () => { if (!callbackStarted) return let timer: ReturnType | undefined try { await Promise.race([callbackDone, new Promise((_resolve, reject) => { timer = setTimeout(() => reject(Error("Owned callback cleanup did not finish")), 3500) })]) } finally { clearTimeout(timer) } } const terminateOwner = async () => { assert(Number.isSafeInteger(pid) && pid > 0) const terminated = await f.admin!.query("SELECT pg_catalog.pg_terminate_backend(pid) AS terminated FROM pg_catalog.pg_stat_activity WHERE pid=$1 AND datname=$2 AND usename=$3", [pid, f.databaseName, f.ownerRole]) assert.equal(terminated.command, "SELECT"); assert.equal(terminated.rowCount, 1) assert.equal(terminated.rows.length, 1); assert.equal(terminated.rows[0].terminated, true) } const external = createBandwidthOperatorExternalFencer({ async acquire() { return { async assertCurrent() { q++ if (mode === "physical-owner-during-Q" && armPostHeld && heldAcknowledged && !injection) { injection = (async () => { await terminateOwner(); await gone(pid) assert(lifetimeSignal) const signalEnd = Math.min(performance.now() + 1000, performance.now() + Math.max(0, scope.deadlineAt - Date.now())) while (!lifetimeSignal.aborted) { assert(performance.now() < signalEnd); await immediate() } assert(Date.now() < scope.deadlineAt, "transport must revoke before original deadline, not because it expired") assert.equal(lifetimeSignal.aborted, true) injected = true })() // Return this Q normally only after the actual transport lifetime is // revoked. A preceding successful HELD row must no longer authorize. await injection } }, close() {} } } }) let window: BandwidthOperatorWriterWindow | undefined const windows = createBandwidthOperatorWriterWindows({ originals, external, source: { async open(current, guard) { await guard(); const raw = await connector(current) lifetimeSignal = raw.lifetimeSignal let closed = false const close = () => { if (!closed) { closed = true; raw.release(true) } } return Object.freeze({ connection: Object.freeze({ release: close, lifetimeSignal: raw.lifetimeSignal, async query(text: string, values?: unknown[], options?: { signal?: AbortSignal; timeoutMs: number }) { await guard(); sql++; if (text === "COMMIT") commits++ const result = await raw.query(text, values, options) if (armPostHeld && text === "SELECT pg_catalog.pg_backend_pid() AS backend_pid, EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE locktype='advisory' AND pid=pg_catalog.pg_backend_pid() AND mode='ExclusiveLock' AND classid=0::pg_catalog.oid AND objid=8723116::pg_catalog.oid AND objsubid=1 AND granted) AS held") { assert("command" in result); assert.equal(result.command, "SELECT") assert.equal(result.rows.length, 1); assert.equal(result.rows[0].backend_pid, pid); assert.equal(result.rows[0].held, true) heldAcknowledged = true } await guard() if (text === "SELECT pg_catalog.pg_backend_pid() AS backend_pid") pid = Number(result.rows[0].backend_pid) return result } }), binding: async () => { await guard(); return Object.freeze({ ...f.root }) }, keyedProof: async () => { throw Error("Owned metadata fixture does not provide HMAC authority") }, close, }) }, }, registry: { async register(_original, borrowed, current) { callbackStarted = true assert(window) const guard = windows.createPeerSourceGuard(window, original) const source = mode === "managed-positive" ? createConfiguredManagedBandwidthPeerSource(managedConfig, guard) : createConfiguredExternalBandwidthPeerSource({ profileFile, keyFile, installationId: f.root.installationId, enrollmentId, keySha256 }, guard) const coordinator = createBandwidthOperatorEnrollmentPrerequisite(config, source, { async verify(_grant, requested) { await windows.verify(window!, original, requested) } }) try { if (mode === "physical-owner-loss") { await terminateOwner() const postCount = f.nativePosts() await assert.rejects(coordinator.acquire(currentPeer, current)) // verifyRead awaits the genuine held-owner check before key read / // decrypt. A socket error may retire the window even earlier. assert.equal(commits, 0); assert.equal(f.nativeGets(), caseGets); assert.equal(f.nativePosts(), postCount) negativeObserved = true throw Error("Owned expected physical owner loss: no secret/native effect or COMMIT") } if (mode === "physical-owner-during-Q") { armPostHeld = true await assert.rejects(coordinator.acquire(currentPeer, current)) await injection; assert(heldAcknowledged && injected) assert.equal(commits, 0); assert.equal(f.nativeGets(), caseGets); assert.equal(f.nativePosts(), posts) negativeObserved = true throw Error("Owned expected post-HELD owner loss: Q returns normally after transport revocation") } const prerequisite = await coordinator.acquire(currentPeer, current) assert.equal(f.nativeGets(), caseGets + 1) const ownership = await f.owner!.query("SELECT EXISTS(SELECT 1 FROM pg_catalog.pg_locks WHERE pid=$1 AND locktype='advisory' AND classid=0::pg_catalog.oid AND objid=8723116::pg_catalog.oid AND objsubid=1 AND mode='ExclusiveLock' AND granted) AS held", [pid]) assert.equal(ownership.rows[0].held, true) const qBefore = q, sqlBefore = sql, getsBefore = f.nativeGets() if (mode === "positive" || mode === "managed-positive") { if (mode === "positive") await rename(keyFile, keyFile + ".held") try { const metadata = await coordinator.inspectMetadata(prerequisite, current) assert.equal(metadata.format, "mmapp-bandwidth-operator-enrollment-metadata-v1") assert.equal(metadata.metadata.evidence.peerRef.kind, currentPeer.kind) assert.equal(JSON.stringify(metadata).includes('"endpoint"'), false) assert.equal(JSON.stringify(metadata).includes(endpoint.password), false) assert.equal(JSON.stringify(metadata).includes(endpoint.username), false) assert.equal(q, qBefore); assert.equal(sql, sqlBefore); assert.equal(f.nativeGets(), getsBefore) metadataPassed = true } finally { if (mode === "positive") await rename(keyFile + ".held", keyFile) } const began = await borrowed.connection.query("BEGIN", undefined, { timeoutMs: 1000 }) assert("command" in began); assert.equal(began.command, "BEGIN") const committed = await borrowed.connection.query("COMMIT", undefined, { timeoutMs: 1000 }) assert("command" in committed); assert.equal(committed.command, "COMMIT") borrowed.connection.release(true) return "known-commit" } const inspect = () => coordinator.inspectMetadata(prerequisite, current) if (mode === "generation") await writeFile(profileFile, canonical({ ...profile, generation: randomUUID() })) if (mode === "ciphertext") await writeFile(profileFile, canonical({ ...profile, ciphertext: Buffer.concat([iv, randomBytes(16), encrypted]).toString("base64url") })) if (mode === "grant") await writeFile(grantFile, "{}") if (mode === "directory-clone") { await rename(sourceFolder, sourceFolder + ".held"); await mkdir(sourceFolder) try { await copyFile(resolve(sourceFolder + ".held", "profile.json"), profileFile) await copyFile(resolve(sourceFolder + ".held", "management.key"), keyFile) await assert.rejects(inspect) } finally { assert.equal(dirname(sourceFolder), folder); await rm(sourceFolder, { recursive: true }); await rename(sourceFolder + ".held", sourceFolder) } } else if (mode === "copies") { await assert.rejects(coordinator.inspectMetadata({ ...prerequisite }, current)) await assert.rejects(coordinator.inspectMetadata(JSON.parse(JSON.stringify(prerequisite)), current)) assert.throws(() => createConfiguredExternalBandwidthPeerSource({ profileFile, keyFile, installationId: f.root.installationId, enrollmentId, keySha256 }, { ...guard })) } else await assert.rejects(inspect) negativeObserved = true assert.equal(sql, sqlBefore); assert.equal(q, qBefore); assert.equal(f.nativeGets(), getsBefore) throw Error("Owned expected negative: no COMMIT or automatic replay") } finally { try { coordinator.close(); source.close() } finally { completeCallback() } } } } }) try { window = await windows.open(original, scope) if (mode === "positive" || mode === "managed-positive") { assert.equal(await windows.register(window), "known-commit"); assert(metadataPassed) } else { await assert.rejects(windows.register(window)); await joinCallback(); assert(negativeObserved) } } finally { windows.close(); originals.close(); await writeFile(grantFile, grantBytes); await writeFile(profileFile, profileBytes) } assert(pid > 0); await gone(pid) if (mode === "physical-owner-loss" || mode === "physical-owner-during-Q") assert.equal(commits, 0) checks.push(mode === "positive" ? "genuine signed original/window guard, actual held PostgreSQL PID/session lock and localhost license GET; encrypted external metadata succeeds with key absent and zero Q/control-SQL/native/projection" : mode === "managed-positive" ? "genuine managed guarded credentials use actual application row epoch; actual localhost GET and same physical owner; metadata fixed application boolean comparison adds no Q/control-SQL/native or credential projection" : mode === "physical-owner-loss" ? "only exact owned database/owner/PID terminated; genuine window/source guard refuses before initial native GET and COMMIT; ordered read guard precedes key/decrypt; physical PID disappears" : mode === "physical-owner-during-Q" ? "actual HELD ACK followed by pinned-owner termination during awaited Q; raw lifetime signal revoked before normal Q return; no key/decrypt admission, native GET or COMMIT" : "owned external metadata refuses " + mode + " without new Q/SQL/native calls or automatic replay; physical owner retires") } assert.equal(await f.digest(coreTables), baseline); assert.equal(f.nativePosts(), posts) return Object.freeze(checks) } finally { assert.equal(dirname(folder), parent); await rm(folder, { recursive: true }) } }