Files
MikrotikManager/backend/src/services/bandwidth-execution-host.sql-helper.ts
T

239 lines
19 KiB
TypeScript

/** Real owned PostgreSQL -> configured sources -> registry -> broker -> HTTP.
* Operator grants, license claims and remote-address approval are explicitly
* synthetic fixture authority. This is not production enrollment/readiness. */
import assert from "node:assert/strict"
import { createHmac, randomBytes, randomUUID, createHash } from "node:crypto"
import { createServer, type ServerResponse } from "node:http"
import { once } from "node:events"
import { setTimeout as delay } from "node:timers/promises"
import type { Pool } from "pg"
import type { createConfiguredBandwidthControlSource } from "./bandwidth-control-source.js"
import type { createConfiguredBandwidthNodeSource } from "./bandwidth-node-source.js"
import type { BandwidthNodeSourceSnapshot } from "./bandwidth-node-observation.js"
import type { BandwidthExecutionIntent } from "./bandwidth-execution-host.js"
import { assertOwnedBandwidthSql, scope } from "./bandwidth-admission-process.sql-helper.js"
type Core = ReturnType<typeof createConfiguredBandwidthControlSource>
type Nodes = ReturnType<typeof createConfiguredBandwidthNodeSource>
type Row = { id: number; host: string; port: number; use_ssl: boolean; verify_ssl: boolean; username: string; password: string; routing_endpoint_epoch: string; enabled: boolean }
type Grant = { intent: BandwidthExecutionIntent; rows: readonly Row[]; allowed: Set<number>; remoteAllowed: boolean; retireRemote: boolean }
const tableNames = ["bandwidth_attempts", "bandwidth_node_occupancy", "bandwidth_attempt_events", "bandwidth_source_identity", "bandwidth_node_authority", "bandwidth_nodes", "bandwidth_node_aliases", "bandwidth_node_events"] as const
export async function acceptBandwidthExecutionHost(core: Core, nodeSource: Nodes, owner: Pool) {
assertOwnedBandwidthSql()
// Imports that allocate/use the default application pool follow the guard.
const { pool: admin } = await import("../db/index.js")
const { applySqlMigrations } = await import("../db/migrate.js")
const { createBandwidthNodeObservationReader } = await import("./bandwidth-node-observation.js")
const { createBandwidthNodeRegistry } = await import("./bandwidth-node-registry.js")
const { createBandwidthExecutionHost, BandwidthExecutionHostError } = await import("./bandwidth-execution-host.js")
await applySqlMigrations(admin)
const key = randomBytes(32), ownIds = new Set<number>(), claims = new Map<number, string>(), grants = new WeakMap<object, Grant>()
const mac = (purpose: string, value: unknown) => createHmac("sha256", key).update("owned-broker-" + purpose + ":" + JSON.stringify(value)).digest("hex")
const checks: string[] = []
let posts = 0, gets = 0, credentialReads = 0, credentialId = -1
let mode: "terminal" | "drop" | "nonterminal" = "terminal", pendingResponse: ServerResponse | undefined
let notifyPost: (() => void) | undefined, lastBody: unknown, postAt = 0
const licenseReads=new Map<number,number>()
let changedLicense: {id:number;at:number}|undefined
const server = createServer(async (req, res) => {
try {
const auth = Buffer.from((req.headers.authorization ?? "").slice(6), "base64").toString()
const match = /^broker-(\d+):/.exec(auth), id = Number(match?.[1])
assert(ownIds.has(id)); assert.equal(auth, `broker-${id}:owned-localhost-secret`)
res.setHeader("content-type", "application/json")
if (req.method === "GET") {
assert.equal(req.url, "/rest/system/license"); assert(claims.has(id)); assert.notEqual(id, credentialId)
gets++;const count=(licenseReads.get(id)??0)+1;licenseReads.set(id,count)
const claim=claims.get(id)+(changedLicense?.id===id&&count>=changedLicense.at?"-CHANGED-AFTER-ORIGINAL-PHASE":"")
res.end(JSON.stringify({ "system-id": claim })); return
}
assert.equal(req.method, "POST"); assert.equal(req.url, "/rest/tool/bandwidth-test")
let text = ""; for await (const chunk of req) { text += String(chunk); assert(Buffer.byteLength(text) <= 65536) }
lastBody = JSON.parse(text); posts++; postAt = Date.now()
if (mode === "drop") { res.destroy(); notifyPost?.(); return }
if (mode === "nonterminal") { res.end(JSON.stringify([{ status: "running" }])); notifyPost?.(); return }
pendingResponse = res; notifyPost?.()
} catch { res.destroy() }
})
server.listen(0, "127.0.0.1"); await once(server, "listening")
const address = server.address(); assert(address && typeof address !== "string"); const port = address.port
const live = (current: { deadlineAt: number; signal?: AbortSignal }) => { current.signal?.throwIfAborted(); assert(Date.now() < current.deadlineAt) }
const readRow = async (id: number): Promise<Row> => {
assert(ownIds.has(id))
if (id === credentialId) credentialReads++
const result = await admin.query("SELECT id,host,port,use_ssl,verify_ssl,username,password,routing_endpoint_epoch::text,enabled FROM public.servers WHERE id=$1", [id])
assert.equal(result.rows.length, 1)
return result.rows[0] as Row
}
const reader = createBandwidthNodeObservationReader({ async read(id, current) {
live(current); assert(ownIds.has(id)); assert.notEqual(id, credentialId)
const row = await readRow(id); live(current)
const value: BandwidthNodeSourceSnapshot = { serverId: id, enabled: row.enabled, sourceProof: mac("source", row), authorityGeneration: "owned-broker-actor-v1",
endpoint: { host: row.host, port: row.port, useSsl: row.use_ssl, verifySsl: row.verify_ssl, username: row.username, password: row.password, routingEndpointEpoch: row.routing_endpoint_epoch } }
return value
} })
const snapshot = async () => {
const result = []
for (const table of tableNames) result.push((await owner.query(`SELECT to_jsonb(t) AS row FROM mmapp_control.${table} t ORDER BY to_jsonb(t)::text`)).rows)
return createHash("sha256").update(JSON.stringify(result)).digest("hex")
}
const allocate = async (claim?: string) => {
// Two owned application fixture databases have independent serial
// sequences but share this canonical registry. Keep their test aliases
// disjoint; changing a claim on an existing alias must still be refused.
const externalMax=Number((await owner.query("SELECT coalesce(max(server_id),0)::pg_catalog.text AS last FROM mmapp_control.bandwidth_node_aliases")).rows[0].last)
const applicationMax=Number((await admin.query("SELECT coalesce(max(id),0)::pg_catalog.text AS last FROM public.servers")).rows[0].last)
const nextId=Math.max(externalMax,applicationMax)+100000
assert(Number.isSafeInteger(nextId)&&nextId>0)
const row = (await admin.query("INSERT INTO public.servers(id,name,host,port,use_ssl,verify_ssl,password,enabled) VALUES($1,'owned-bandwidth-broker','127.0.0.1',$2,false,true,'owned-localhost-secret',true) RETURNING id", [nextId,port])).rows[0]
const id = Number(row.id); ownIds.add(id)
await admin.query("UPDATE public.servers SET username=$2 WHERE id=$1", [id, "broker-" + id])
if (claim) claims.set(id, claim)
return id
}
const enroll = async (id: number) => {
const current = scope(10000), token = await reader.observe(id, current), observed = await reader.inspect(token, current)
const guard = async () => { assert.deepEqual(await reader.inspect(token, current), observed); live(current) }
const session = await nodeSource.open(current, guard)
try {
const binding = await session.binding(), expiry = Date.now() + 30000
const registry = createBandwidthNodeRegistry({ now: Date.now, connect: async () => session.connection,
inspect: (value, window) => reader.inspect(value, window), keyedProof: (purpose, value) => session.keyedProof(purpose, value), keyedProofs: inputs => session.keyedProofs(inputs),
authorize: async (action, serverId, window, evidence) => {
assert.equal(action, "register"); assert.equal(serverId, id); assert.deepEqual(evidence, observed)
await guard(); live(window); assert(Date.now() < expiry)
return { installationId: binding.installationId, issuerKeySha256: binding.issuerKeySha256, quiescenceProof: binding.quiescenceProof,
authorityProof: mac("SYNTHETIC-operator-enrollment", { id, claims: observed.claims }), authorityGeneration: "synthetic-owned-operator-v1", expiresAt: expiry }
} })
assert.equal((await registry.register(token, current)).status, "active")
} finally { session.close() }
}
const retire = async (id: number) => {
const current = scope(10000), session = await nodeSource.open(current)
try {
const binding = await session.binding(), expiry = Date.now() + 10000
const registry = createBandwidthNodeRegistry({ now: Date.now, connect: async () => session.connection,
inspect: (value, window) => reader.inspect(value, window), keyedProof: (purpose, value) => session.keyedProof(purpose, value), keyedProofs: inputs => session.keyedProofs(inputs),
authorize: async (action, serverId, window, evidence) => {
assert.equal(action, "tombstone"); assert.equal(serverId, id); assert.equal(evidence, undefined); live(window)
return { installationId: binding.installationId, issuerKeySha256: binding.issuerKeySha256, quiescenceProof: binding.quiescenceProof,
authorityProof: mac("SYNTHETIC-operator-retirement", { id }), authorityGeneration: "synthetic-owned-operator-v1", expiresAt: expiry }
} })
assert.equal((await registry.retire(id, "tombstone", current)).status, "tombstoned")
} finally { session.close() }
}
const context = async (sourceServerId: number, actualRemoteServerId: number) => {
const rows = await Promise.all([sourceServerId, actualRemoteServerId, credentialId].map(readRow)), expiresAt = Date.now() + 60000
const intent: BandwidthExecutionIntent = { attemptId: randomUUID(), operationId: randomUUID(), callerKind: "rest-diagnostic", sourceServerId, actualRemoteServerId, credentialServerId: credentialId,
request: { fixture: "synthetic-owned-broker", address: "198.51.100.42@fixture-vrf" }, durationSec: 2147483647, socketTimeoutMs: 30000,
nativeBody: { address: "198.51.100.42@fixture-vrf", user: rows[2].username, password: rows[2].password, protocol: "tcp", direction: "both", duration: "2147483647s" },
authorityProof: mac("sealed-original", { sourceServerId, actualRemoteServerId, credentialId, rows, expiresAt }), authorityGeneration: "owned-broker-actor-v1", expiresAt }
const opaque = Object.freeze(Object.create(null)) as object
const grant: Grant = { intent, rows, allowed: new Set([sourceServerId, actualRemoteServerId, credentialId]), remoteAllowed: true, retireRemote: false }
grants.set(opaque, grant); return { opaque, grant }
}
const host = createBandwidthExecutionHost({ coreSource: core, nodeSource, observer: reader,
readOriginal: async (opaque, current) => {
const grant = opaque && typeof opaque === "object" ? grants.get(opaque) : undefined
assert(grant); live(current); assert(Date.now() < grant.intent.expiresAt)
// Authorize every ID before reading ANY credentials, especially the third.
for (const id of [grant.intent.sourceServerId, grant.intent.actualRemoteServerId, grant.intent.credentialServerId]) assert(grant.allowed.has(id))
// One bounded, scoped snapshot for all three already-authorized IDs.
// It reads the same original fields without three separate pool checkouts.
const ids=grant.rows.map(row=>row.id)
credentialReads++
const selected=(await admin.query("SELECT id,host,port,use_ssl,verify_ssl,username,password,routing_endpoint_epoch::text,enabled FROM public.servers WHERE id=ANY($1::bigint[])",[ids])).rows as Row[]
assert.equal(selected.length,ids.length)
const byId=new Map(selected.map(row=>[Number(row.id),row]))
const fresh=ids.map(id=>byId.get(id));live(current)
assert.deepEqual(fresh, grant.rows)
return structuredClone(grant.intent)
},
verifyRemote: async (opaque, intent, evidence, current) => {
const grant = opaque && typeof opaque === "object" ? grants.get(opaque) : undefined
assert(grant); live(current); assert(grant.remoteAllowed); assert.deepEqual(intent, grant.intent)
assert.equal(intent.nativeBody.address, "198.51.100.42@fixture-vrf")
assert.equal(evidence.remoteReceipt.serverId, intent.actualRemoteServerId); assert.notEqual(intent.actualRemoteServerId, intent.credentialServerId)
assert.equal(evidence.remoteEvidence.serverId, intent.actualRemoteServerId)
if (grant.retireRemote) { grant.retireRemote = false; await retire(intent.actualRemoteServerId) }
live(current)
return { proof: mac("SYNTHETIC-exact-address-VRF-policy", { address: intent.nativeBody.address, actualRemote: evidence.remoteReceipt, credentialId: intent.credentialServerId }), expiresAt: intent.expiresAt }
} })
const execute = async(value: { opaque: object }) => {
const began=Date.now()
try{return await host.execute(value.opaque,scope(60000))}
catch(error){console.error("Owned broker refusal timing:",JSON.stringify({elapsedMs:Date.now()-began,phase:error instanceof BandwidthExecutionHostError?error.phase:"dependency",posts}));throw error}
}
const unknown = async (attemptId: string) => {
assert.equal((await owner.query("SELECT state FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [attemptId])).rows[0]?.state, "unknown_busy")
assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", [attemptId])).rows[0].n), 2)
}
try {
const sourceId = await allocate("BROKER-SOURCE-" + randomUUID()), remoteId = await allocate("BROKER-REMOTE-" + randomUUID())
credentialId = await allocate()
const {acceptBandwidthServiceContextSql}=await import("../modules/mcp/bandwidth-service-context.sql-helper.js")
const serviceContextAcceptance=await acceptBandwidthServiceContextSql([sourceId,remoteId,credentialId])
await enroll(sourceId); await enroll(remoteId)
assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_aliases WHERE server_id=$1", [credentialId])).rows[0].n), 0)
const denied = await context(sourceId, remoteId), baseline = await snapshot(), beforeReads = credentialReads
denied.grant.allowed.delete(credentialId)
await assert.rejects(execute(denied), error => error instanceof Error && "code" in error && error.code === "REFUSED")
assert.equal(credentialReads, beforeReads); assert.equal(posts, 0); assert.equal(await snapshot(), baseline)
checks.push("third credential-provider ID denied before any credential fetch, native POST or control mutation")
const policyDenied = await context(sourceId, remoteId); policyDenied.grant.remoteAllowed = false
await assert.rejects(execute(policyDenied), error => error instanceof Error && "code" in error && error.code === "REFUSED")
assert.equal(posts, 0); assert.equal(await snapshot(), baseline)
checks.push("exact address/VRF remote policy denial preserves control ledger and sends zero POST")
const changedSubject=await context(sourceId,remoteId)
changedLicense={id:sourceId,at:(licenseReads.get(sourceId)??0)+2}
try{
await assert.rejects(execute(changedSubject),error=>error instanceof BandwidthExecutionHostError&&error.code==="REFUSED"&&error.phase==="source-resolution")
assert.equal(posts,0);assert.equal(await snapshot(),baseline)
}finally{changedLicense=undefined}
checks.push("a NEW genuine license observation with changed original claims is refused between phases before reserve/native; no token renewal, alias adoption or control mutation")
const success = await context(sourceId, remoteId)
const entered = new Promise<void>(resolve => { notifyPost = resolve })
const running = execute(success); void running.catch(() => {})
await Promise.race([entered, running.then(() => assert.fail("Completed before native response"))])
// Every admission lease must be returned before awaiting a long native call.
const borrowed = await Promise.allSettled(Array.from({ length: 4 }, () => core.connect(scope(5000))))
try {
assert(borrowed.every(result => result.status === "fulfilled"))
await Promise.all(borrowed.map(result => result.status === "fulfilled" ? result.value.query("SELECT 1") : Promise.reject(Error("Lease unavailable"))))
} finally { for (const result of borrowed) if (result.status === "fulfilled") result.value.release(true) }
await delay(Math.max(0, 5400 - (Date.now() - postAt)))
assert(pendingResponse); pendingResponse.end(JSON.stringify([{ status: "done testing", "tx-total-average": "1000" }]))
const completed = await running
assert.equal(completed.remoteCompletionConfirmed, true); assert(Date.now() - postAt >= 5300)
assert.deepEqual(lastBody, success.grant.intent.nativeBody); assert.equal(posts, 1)
assert.equal((await owner.query("SELECT state,resolution FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [success.grant.intent.attemptId])).rows[0].state, "released")
assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_node_occupancy WHERE attempt_id=$1", [success.grant.intent.attemptId])).rows[0].n), 0)
const audit = (await admin.query("SELECT action FROM public.mcp_audit WHERE payload->>'serverId'=$1", [String(sourceId)])).rows.map(row => row.action)
assert(audit.includes("configuration.intent")); assert(audit.includes("configuration.acknowledged"))
checks.push("real configured broker terminal response beyond 5.3s observation TTL releases both owners without grant renewal; exact max duration/body and actual configuration intent/acknowledged audit; four core leases available during native wait")
mode = "drop"; const dropped = await context(sourceId, remoteId)
await assert.rejects(execute(dropped)); assert.equal(posts, 2); await unknown(dropped.grant.intent.attemptId)
const again = await context(sourceId, remoteId)
await assert.rejects(execute(again)); assert.equal(posts, 2); await unknown(dropped.grant.intent.attemptId)
checks.push("one dropped physical POST leaves both exact owners unknown-busy; fresh attempt on same pair sends zero additional POST and never retries")
const otherSource = await allocate("BROKER-OTHER-SOURCE-" + randomUUID()), otherRemote = await allocate("BROKER-OTHER-REMOTE-" + randomUUID())
await enroll(otherSource); await enroll(otherRemote)
const retired = await context(otherSource, otherRemote); retired.grant.retireRemote = true
await assert.rejects(execute(retired)); assert.equal(posts, 2)
assert.equal((await owner.query("SELECT status FROM mmapp_control.bandwidth_node_aliases WHERE server_id=$1", [otherRemote])).rows[0].status, "tombstoned")
assert.equal(Number((await owner.query("SELECT count(*) AS n FROM mmapp_control.bandwidth_attempts WHERE attempt_id=$1", [retired.grant.intent.attemptId])).rows[0].n), 0)
await unknown(dropped.grant.intent.attemptId)
checks.push("genuine operator tombstone after resolve is caught by same-physical alias fence before reservation/native dispatch; unrelated unknown owners preserved")
return { checks, scenarios: 6, serviceContextAcceptance, actualLocalhostBandwidthPosts: posts, actualLocalhostLicenseGets: gets, productionActivation: false, operatorAuthoritySynthetic: true, remotePolicySynthetic: true, realChrCalls: 0 }
} finally {
reader.close(); server.closeAllConnections()
await new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve()))
key.fill(0)
if (ownIds.size) await admin.query("DELETE FROM public.servers WHERE id=ANY($1::bigint[])", [[...ownIds]])
}
}