/** Real owned PostgreSQL -> configured sources -> registry -> broker -> HTTP. * Operator grants, license claims and remote-address approval are explicitly * synthetic fixture authority. This is not production enrollment/readiness. */ import assert from "node:assert/strict" import { createHmac, randomBytes, randomUUID, createHash } from "node:crypto" import { createServer, type ServerResponse } from "node:http" import { once } from "node:events" import { setTimeout as delay } from "node:timers/promises" import type { Pool } from "pg" import type { createConfiguredBandwidthControlSource } from "./bandwidth-control-source.js" import type { createConfiguredBandwidthNodeSource } from "./bandwidth-node-source.js" import type { BandwidthNodeSourceSnapshot } from "./bandwidth-node-observation.js" import type { BandwidthExecutionIntent } from "./bandwidth-execution-host.js" import { assertOwnedBandwidthSql, scope } from "./bandwidth-admission-process.sql-helper.js" type Core = ReturnType type Nodes = ReturnType type Row = { id: number; host: string; port: number; use_ssl: boolean; verify_ssl: boolean; username: string; password: string; routing_endpoint_epoch: string; enabled: boolean } type Grant = { intent: BandwidthExecutionIntent; rows: readonly Row[]; allowed: Set; remoteAllowed: boolean; retireRemote: boolean } const tableNames = ["bandwidth_attempts", "bandwidth_node_occupancy", "bandwidth_attempt_events", "bandwidth_source_identity", "bandwidth_node_authority", "bandwidth_nodes", "bandwidth_node_aliases", "bandwidth_node_events"] as const export async function acceptBandwidthExecutionHost(core: Core, nodeSource: Nodes, owner: Pool) { assertOwnedBandwidthSql() // Imports that allocate/use the default application pool follow the guard. const { pool: admin } = await import("../db/index.js") const { applySqlMigrations } = await import("../db/migrate.js") const { createBandwidthNodeObservationReader } = await import("./bandwidth-node-observation.js") const { createBandwidthNodeRegistry } = await import("./bandwidth-node-registry.js") const { createBandwidthExecutionHost, BandwidthExecutionHostError } = await import("./bandwidth-execution-host.js") await applySqlMigrations(admin) const key = randomBytes(32), ownIds = new Set(), claims = new Map(), grants = new WeakMap() const mac = (purpose: string, value: unknown) => createHmac("sha256", key).update("owned-broker-" + purpose + ":" + JSON.stringify(value)).digest("hex") const checks: string[] = [] let posts = 0, gets = 0, credentialReads = 0, credentialId = -1 let mode: "terminal" | "drop" | "nonterminal" = "terminal", pendingResponse: ServerResponse | undefined let notifyPost: (() => void) | undefined, lastBody: unknown, postAt = 0 const licenseReads=new Map() let changedLicense: {id:number;at:number}|undefined const server = createServer(async (req, res) => { try { const auth = Buffer.from((req.headers.authorization ?? "").slice(6), "base64").toString() const match = /^broker-(\d+):/.exec(auth), id = Number(match?.[1]) assert(ownIds.has(id)); assert.equal(auth, `broker-${id}:owned-localhost-secret`) res.setHeader("content-type", "application/json") if (req.method === "GET") { assert.equal(req.url, "/rest/system/license"); assert(claims.has(id)); assert.notEqual(id, credentialId) gets++;const count=(licenseReads.get(id)??0)+1;licenseReads.set(id,count) const claim=claims.get(id)+(changedLicense?.id===id&&count>=changedLicense.at?"-CHANGED-AFTER-ORIGINAL-PHASE":"") res.end(JSON.stringify({ "system-id": claim })); return } assert.equal(req.method, "POST"); assert.equal(req.url, "/rest/tool/bandwidth-test") let text = ""; for await (const chunk of req) { text += String(chunk); assert(Buffer.byteLength(text) <= 65536) } lastBody = JSON.parse(text); posts++; postAt = Date.now() if (mode === "drop") { res.destroy(); notifyPost?.(); return } if (mode === "nonterminal") { res.end(JSON.stringify([{ status: "running" }])); notifyPost?.(); return } pendingResponse = res; notifyPost?.() } catch { res.destroy() } }) server.listen(0, "127.0.0.1"); await once(server, "listening") const address = server.address(); assert(address && typeof address !== "string"); const port = address.port const live = (current: { deadlineAt: number; signal?: AbortSignal }) => { current.signal?.throwIfAborted(); assert(Date.now() < current.deadlineAt) } const readRow = async (id: number): Promise => { assert(ownIds.has(id)) if (id === credentialId) credentialReads++ const result = await admin.query("SELECT id,host,port,use_ssl,verify_ssl,username,password,routing_endpoint_epoch::text,enabled FROM public.servers WHERE id=$1", [id]) assert.equal(result.rows.length, 1) return result.rows[0] as Row } const reader = createBandwidthNodeObservationReader({ async read(id, current) { live(current); assert(ownIds.has(id)); assert.notEqual(id, credentialId) const row = await readRow(id); live(current) const value: BandwidthNodeSourceSnapshot = { serverId: id, enabled: row.enabled, sourceProof: mac("source", row), authorityGeneration: "owned-broker-actor-v1", endpoint: { host: row.host, port: row.port, useSsl: row.use_ssl, verifySsl: row.verify_ssl, username: row.username, password: row.password, routingEndpointEpoch: row.routing_endpoint_epoch } } return value } }) const snapshot = async () => { const result = [] for (const table of tableNames) result.push((await owner.query(`SELECT to_jsonb(t) AS row FROM mmapp_control.${table} t ORDER BY to_jsonb(t)::text`)).rows) return createHash("sha256").update(JSON.stringify(result)).digest("hex") } const allocate = async (claim?: string) => { // Two owned application fixture databases have independent serial // sequences but share this canonical registry. Keep their test aliases // disjoint; changing a claim on an existing alias must still be refused. const externalMax=Number((await owner.query("SELECT coalesce(max(server_id),0)::pg_catalog.text AS last FROM mmapp_control.bandwidth_node_aliases")).rows[0].last) const applicationMax=Number((await admin.query("SELECT coalesce(max(id),0)::pg_catalog.text AS last FROM public.servers")).rows[0].last) const nextId=Math.max(externalMax,applicationMax)+100000 assert(Number.isSafeInteger(nextId)&&nextId>0) const row = (await admin.query("INSERT INTO public.servers(id,name,host,port,use_ssl,verify_ssl,password,enabled) VALUES($1,'owned-bandwidth-broker','127.0.0.1',$2,false,true,'owned-localhost-secret',true) RETURNING id", [nextId,port])).rows[0] const id = Number(row.id); ownIds.add(id) await admin.query("UPDATE public.servers SET username=$2 WHERE id=$1", [id, "broker-" + id]) if (claim) claims.set(id, claim) return id } const enroll = async (id: number) => { const current = scope(10000), token = await reader.observe(id, current), observed = await reader.inspect(token, current) const guard = async () => { assert.deepEqual(await reader.inspect(token, current), observed); live(current) } const session = await nodeSource.open(current, guard) try { const binding = await session.binding(), expiry = Date.now() + 30000 const registry = createBandwidthNodeRegistry({ now: Date.now, connect: async () => session.connection, inspect: (value, window) => reader.inspect(value, window), keyedProof: (purpose, value) => session.keyedProof(purpose, value), keyedProofs: inputs => session.keyedProofs(inputs), authorize: async (action, serverId, window, evidence) => { assert.equal(action, "register"); assert.equal(serverId, id); assert.deepEqual(evidence, observed) await guard(); live(window); assert(Date.now() < expiry) return { installationId: binding.installationId, issuerKeySha256: binding.issuerKeySha256, quiescenceProof: binding.quiescenceProof, authorityProof: mac("SYNTHETIC-operator-enrollment", { id, claims: observed.claims }), authorityGeneration: "synthetic-owned-operator-v1", expiresAt: expiry } } }) assert.equal((await registry.register(token, current)).status, "active") } finally { session.close() } } const retire = async (id: number) => { const current = scope(10000), session = await nodeSource.open(current) try { const binding = await session.binding(), expiry = Date.now() + 10000 const registry = createBandwidthNodeRegistry({ now: Date.now, connect: async () => session.connection, inspect: (value, window) => reader.inspect(value, window), keyedProof: (purpose, value) => session.keyedProof(purpose, value), keyedProofs: inputs => session.keyedProofs(inputs), authorize: async (action, serverId, window, evidence) => { assert.equal(action, "tombstone"); assert.equal(serverId, id); assert.equal(evidence, undefined); live(window) return { installationId: binding.installationId, issuerKeySha256: binding.issuerKeySha256, quiescenceProof: binding.quiescenceProof, authorityProof: mac("SYNTHETIC-operator-retirement", { id }), authorityGeneration: "synthetic-owned-operator-v1", expiresAt: expiry } } }) assert.equal((await registry.retire(id, "tombstone", current)).status, "tombstoned") } finally { session.close() } } const context = async (sourceServerId: number, actualRemoteServerId: number) => { const rows = await Promise.all([sourceServerId, actualRemoteServerId, credentialId].map(readRow)), expiresAt = Date.now() + 60000 const intent: BandwidthExecutionIntent = { attemptId: randomUUID(), operationId: randomUUID(), callerKind: "rest-diagnostic", sourceServerId, actualRemoteServerId, credentialServerId: credentialId, request: { fixture: "synthetic-owned-broker", address: "198.51.100.42@fixture-vrf" }, durationSec: 2147483647, socketTimeoutMs: 30000, nativeBody: { address: "198.51.100.42@fixture-vrf", user: rows[2].username, password: rows[2].password, protocol: "tcp", direction: "both", duration: "2147483647s" }, authorityProof: mac("sealed-original", { sourceServerId, actualRemoteServerId, credentialId, rows, expiresAt }), authorityGeneration: "owned-broker-actor-v1", expiresAt } const opaque = Object.freeze(Object.create(null)) as object const grant: Grant = { intent, rows, allowed: new Set([sourceServerId, actualRemoteServerId, credentialId]), remoteAllowed: true, retireRemote: false } grants.set(opaque, grant); return { opaque, grant } } const host = createBandwidthExecutionHost({ coreSource: core, nodeSource, observer: reader, readOriginal: async (opaque, current) => { const grant = opaque && typeof opaque === "object" ? grants.get(opaque) : undefined assert(grant); live(current); assert(Date.now() < grant.intent.expiresAt) // Authorize every ID before reading ANY credentials, especially the third. for (const id of [grant.intent.sourceServerId, grant.intent.actualRemoteServerId, grant.intent.credentialServerId]) assert(grant.allowed.has(id)) // One bounded, scoped snapshot for all three already-authorized IDs. // It reads the same original fields without three separate pool checkouts. const ids=grant.rows.map(row=>row.id) credentialReads++ const selected=(await admin.query("SELECT id,host,port,use_ssl,verify_ssl,username,password,routing_endpoint_epoch::text,enabled FROM public.servers WHERE id=ANY($1::bigint[])",[ids])).rows as Row[] assert.equal(selected.length,ids.length) const byId=new Map(selected.map(row=>[Number(row.id),row])) const fresh=ids.map(id=>byId.get(id));live(current) assert.deepEqual(fresh, grant.rows) return structuredClone(grant.intent) }, verifyRemote: async (opaque, intent, evidence, current) => { const grant = opaque && typeof opaque === "object" ? grants.get(opaque) : undefined assert(grant); live(current); assert(grant.remoteAllowed); assert.deepEqual(intent, grant.intent) assert.equal(intent.nativeBody.address, "198.51.100.42@fixture-vrf") assert.equal(evidence.remoteReceipt.serverId, intent.actualRemoteServerId); assert.notEqual(intent.actualRemoteServerId, intent.credentialServerId) assert.equal(evidence.remoteEvidence.serverId, intent.actualRemoteServerId) if (grant.retireRemote) { grant.retireRemote = false; await retire(intent.actualRemoteServerId) } live(current) return { proof: mac("SYNTHETIC-exact-address-VRF-policy", { address: intent.nativeBody.address, actualRemote: evidence.remoteReceipt, credentialId: intent.credentialServerId }), expiresAt: intent.expiresAt } } }) const execute = async(value: { opaque: object }) => { const began=Date.now() try{return await host.execute(value.opaque,scope(60000))} catch(error){console.error("Owned broker refusal timing:",JSON.stringify({elapsedMs:Date.now()-began,phase:error instanceof BandwidthExecutionHostError?error.phase:"dependency",posts}));throw error} } const unknown = async (attemptId: string) => { assert.equal((await owner.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [attemptId])).rows[0]?.state, "unknown_busy") assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", [attemptId])).rows[0].n), 2) } try { const sourceId = await allocate("BROKER-SOURCE-" + randomUUID()), remoteId = await allocate("BROKER-REMOTE-" + randomUUID()) credentialId = await allocate() const {acceptBandwidthServiceContextSql}=await import("../modules/mcp/bandwidth-service-context.sql-helper.js") const serviceContextAcceptance=await acceptBandwidthServiceContextSql([sourceId,remoteId,credentialId]) await enroll(sourceId); await enroll(remoteId) assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_aliases WHERE server_id=$1", [credentialId])).rows[0].n), 0) const denied = await context(sourceId, remoteId), baseline = await snapshot(), beforeReads = credentialReads denied.grant.allowed.delete(credentialId) await assert.rejects(execute(denied), error => error instanceof Error && "code" in error && error.code === "REFUSED") assert.equal(credentialReads, beforeReads); assert.equal(posts, 0); assert.equal(await snapshot(), baseline) checks.push("third credential-provider ID denied before any credential fetch, native POST or control mutation") const policyDenied = await context(sourceId, remoteId); policyDenied.grant.remoteAllowed = false await assert.rejects(execute(policyDenied), error => error instanceof Error && "code" in error && error.code === "REFUSED") assert.equal(posts, 0); assert.equal(await snapshot(), baseline) checks.push("exact address/VRF remote policy denial preserves control ledger and sends zero POST") const changedSubject=await context(sourceId,remoteId) changedLicense={id:sourceId,at:(licenseReads.get(sourceId)??0)+2} try{ await assert.rejects(execute(changedSubject),error=>error instanceof BandwidthExecutionHostError&&error.code==="REFUSED"&&error.phase==="source-resolution") assert.equal(posts,0);assert.equal(await snapshot(),baseline) }finally{changedLicense=undefined} checks.push("a NEW genuine license observation with changed original claims is refused between phases before reserve/native; no token renewal, alias adoption or control mutation") const success = await context(sourceId, remoteId) const entered = new Promise(resolve => { notifyPost = resolve }) const running = execute(success); void running.catch(() => {}) await Promise.race([entered, running.then(() => assert.fail("Completed before native response"))]) // Every admission lease must be returned before awaiting a long native call. const borrowed = await Promise.allSettled(Array.from({ length: 4 }, () => core.connect(scope(5000)))) try { assert(borrowed.every(result => result.status === "fulfilled")) await Promise.all(borrowed.map(result => result.status === "fulfilled" ? result.value.query("SELECT 1") : Promise.reject(Error("Lease unavailable")))) } finally { for (const result of borrowed) if (result.status === "fulfilled") result.value.release(true) } await delay(Math.max(0, 5400 - (Date.now() - postAt))) assert(pendingResponse); pendingResponse.end(JSON.stringify([{ status: "done testing", "tx-total-average": "1000" }])) const completed = await running assert.equal(completed.remoteCompletionConfirmed, true); assert(Date.now() - postAt >= 5300) assert.deepEqual(lastBody, success.grant.intent.nativeBody); assert.equal(posts, 1) assert.equal((await owner.query("SELECT state,resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [success.grant.intent.attemptId])).rows[0].state, "released") assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", [success.grant.intent.attemptId])).rows[0].n), 0) const audit = (await admin.query("SELECT action FROM public.mcp_audit WHERE payload->>'serverId'=$1", [String(sourceId)])).rows.map(row => row.action) assert(audit.includes("configuration.intent")); assert(audit.includes("configuration.acknowledged")) checks.push("real configured broker terminal response beyond 5.3s observation TTL releases both owners without grant renewal; exact max duration/body and actual configuration intent/acknowledged audit; four core leases available during native wait") mode = "drop"; const dropped = await context(sourceId, remoteId) await assert.rejects(execute(dropped)); assert.equal(posts, 2); await unknown(dropped.grant.intent.attemptId) const again = await context(sourceId, remoteId) await assert.rejects(execute(again)); assert.equal(posts, 2); await unknown(dropped.grant.intent.attemptId) checks.push("one dropped physical POST leaves both exact owners unknown-busy; fresh attempt on same pair sends zero additional POST and never retries") const otherSource = await allocate("BROKER-OTHER-SOURCE-" + randomUUID()), otherRemote = await allocate("BROKER-OTHER-REMOTE-" + randomUUID()) await enroll(otherSource); await enroll(otherRemote) const retired = await context(otherSource, otherRemote); retired.grant.retireRemote = true await assert.rejects(execute(retired)); assert.equal(posts, 2) assert.equal((await owner.query("SELECT status FROM mmapp_control.bandwidth_node_aliases WHERE server_id=$1", [otherRemote])).rows[0].status, "tombstoned") assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [retired.grant.intent.attemptId])).rows[0].n), 0) await unknown(dropped.grant.intent.attemptId) checks.push("genuine operator tombstone after resolve is caught by same-physical alias fence before reservation/native dispatch; unrelated unknown owners preserved") return { checks, scenarios: 6, serviceContextAcceptance, actualLocalhostBandwidthPosts: posts, actualLocalhostLicenseGets: gets, productionActivation: false, operatorAuthoritySynthetic: true, remotePolicySynthetic: true, realChrCalls: 0 } } finally { reader.close(); server.closeAllConnections() await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())) key.fill(0) if (ownIds.size) await admin.query("DELETE FROM public.servers WHERE id=ANY($1::bigint[])", [[...ownIds]]) } }