import assert from "node:assert/strict" import { createHmac, randomUUID } from "node:crypto" import { createServerProbeExecutor, ServerProbeExecutionError, type ServerProbeAuthority, type ServerProbeEndpoint, type ServerProbeExecutionDependencies, type ServerProbeReceipt } from "./server-probe-execution.js" import { planServerProbeExecution, projectServerProbeRoute } from "./server-probe-execution-model.js" import { McpError } from "../modules/mcp/security.js" function fixture() { let time = 1000000, sends = 0, dnsCalls = 0, beforeGet: ((id: number) => void) | undefined, beforeSend: (() => void) | undefined, afterSend: (() => void) | undefined let providerFailure = false, sourceDnsFailure = false, repeatedFence = false, parallelFence = false, waitDns = false let claimLost = false, intentLost = false, receiptLost = false, receivedLost = false let raw: unknown = [{ seq: "0", ttl: "64", time: "1ms" }], metadataRef = 2 const source: ServerProbeEndpoint = { id: 1, enabled: true, host: "192.0.2.1", port: 80, username: "NEVER_SOURCE_USER", password: "NEVER_SOURCE_PASSWORD", useSsl: false, verifySsl: true, routingEndpointEpoch: randomUUID() } const destination: ServerProbeEndpoint = { ...source, id: 2, host: "192.0.2.2", username: "NEVER_DEST_USER", password: "NEVER_DEST_PASSWORD", routingEndpointEpoch: randomUUID() } const authority: ServerProbeAuthority = { ownerId: "owner", connectionId: randomUUID(), credentialId: randomUUID(), expiresAt: 2000000, generation: "generation", policyProof: "a".repeat(64), ownerProof: "b".repeat(64), active: true, operationAllowed: true, scopeServerIds: [1, 2], ownerPermissions: ["mm:servers:write"], policyPermissions: ["mm:servers:write"] } const events: Array<{ kind: string; value?: unknown }> = [], bindings = new Map(), claims = new Map(), receipts = new Map(), getIds: number[] = [] let expectedSignal: AbortSignal | undefined const deps: ServerProbeExecutionDependencies = { async getServer(id) { getIds.push(id); beforeGet?.(id); return id === 1 ? structuredClone(source) : id === 2 ? structuredClone(destination) : undefined }, async findDestination(_remote, explicit, allowedIds) { events.push({ kind: "metadata", value: [...allowedIds] }); return explicit !== undefined && !allowedIds.includes(explicit) ? undefined : { id: metadataRef } }, async readAuthority() { return structuredClone(authority) }, async assertProtections(ids) { assert(ids.every(id => authority.scopeServerIds.includes(id))); events.push({ kind: "protection", value: ids }) }, async privateProof(domain, value) { return createHmac("sha256", "server-private-test-key").update(domain).update(JSON.stringify(value)).digest("hex") }, async seal(value) { const id = randomUUID(); bindings.set(id, value); return id }, async open(value) { return bindings.get(value)! }, now: () => time, async post(_server, path, body, timeout, signal, fence) { expectedSignal = signal; beforeSend?.(); if (parallelFence) await Promise.all([fence(), fence()]); else await fence(); sends++; events.push({ kind: "native", value: { path, body: { ...body }, timeout } }) if (repeatedFence) await fence() if (providerFailure) throw Error("NEVER_PROVIDER_PASSWORD") afterSend?.(); return structuredClone(raw) }, async routes(_server, timeout, signal, fence) { expectedSignal = signal; beforeSend?.(); await fence(); sends++; events.push({ kind: "route", value: timeout }); if (providerFailure) throw Error("NEVER_ROUTE_PROVIDER"); afterSend?.(); return structuredClone(raw) }, async dns(name, type, signal, fence) { expectedSignal = signal; await fence(); dnsCalls++; events.push({ kind: "dns", value: { name, type } }) if (waitDns) return new Promise(() => {}) if (sourceDnsFailure && name === "router.example") throw Error("NEVER_DNS_PROVIDER") if (providerFailure) throw Error("NEVER_DNS_PROVIDER") afterSend?.(); return structuredClone(raw) }, journal: { async claim(value) { if (claims.has(value.receiptId)) return "exists"; claims.set(value.receiptId, value.bindingProof); events.push({ kind: "claim", value }); if (claimLost) throw Error("lost claim ACK"); return "claimed" }, async intent(value) { assert(claims.has(value.receiptId)); events.push({ kind: "intent", value }); if (intentLost) throw Error("lost intent ACK") }, async received(value) { events.push({ kind: "received", value }); if (receivedLost) throw Error("lost received ACK") }, async finish(value) { receipts.set(value.receiptId, structuredClone(value)); events.push({ kind: "finish", value }); if (receiptLost) throw Error("lost receipt ACK") }, async quarantine(value) { claims.set(value.receiptId, value.bindingProof); receipts.set(value.receiptId, { receiptId: value.receiptId, tool: value.tool as ServerProbeReceipt["tool"], serverId: 1, state: "unknown", httpStatus: 503, errorCode: value.errorCode, attemptedStages: value.attemptedStages, completedStages: value.completedStages, automaticReplay: false, forwardingVerified: false }); events.push({ kind: "quarantine", value }) }, async read(id, bindingProof) { assert.equal(claims.get(id), bindingProof); return receipts.get(id) }, }, } return { executor: createServerProbeExecutor(deps), deps, source, destination, authority, events, receipts, getIds, raw(value: unknown) { raw = value }, beforeGet(hook: (id: number) => void) { beforeGet = hook }, beforeSend(hook: () => void) { beforeSend = hook }, afterSend(hook: () => void) { afterSend = hook }, failure() { providerFailure = true }, sourceFailure() { sourceDnsFailure = true }, duplicateFence() { repeatedFence = true }, concurrentFence() { parallelFence = true }, pendingDns() { waitDns = true }, metadata(id: number) { metadataRef = id }, lost(kind: "claim" | "intent" | "receipt" | "received") { if (kind === "claim") claimLost = true; if (kind === "intent") intentLost = true; if (kind === "receipt") receiptLost = true; if (kind === "received") receivedLost = true }, advance(ms: number) { time += ms }, counts() { return { sends, dnsCalls } }, signal() { return expectedSignal }, } } let cases = 0 // A completed network read must survive bounded journal/authority overhead; // its wire timeout and original private expiry must not be extended. for (const [metadataMs, authorityMs, success] of [[11000, 1000000, true], [41000, 1000000, false], [13000, 12000, false]] as const) { const f = fixture() f.authority.expiresAt = 1000000 + authorityMs f.raw([{ active: "true", "dst-address": "0.0.0.0/0", gateway: "192.0.2.254", distance: "1" }]) const acknowledge = f.deps.journal!.received.bind(f.deps.journal!) f.deps.journal!.received = async value => { await acknowledge(value); f.advance(metadataMs) } const prepared = await f.executor.prepare(1, { tool: "route", target: "192.0.2.2" }) if (success) assert.equal((await f.executor.execute(prepared)).state, "completed") else await assert.rejects(f.executor.execute(prepared), value => value instanceof ServerProbeExecutionError && value.code === "CANCELLED") assert.equal(f.counts().sends, 1) assert.equal(f.events.find(event => event.kind === "route")?.value, 10000) cases++ } const request = (tool: string, extra: Record = {}) => ({ tool, target: "192.0.2.2", bwRemoteAddress: "192.0.2.2", ...extra }) async function failure(work: Promise, code?: string) { await assert.rejects(work, value => value instanceof ServerProbeExecutionError && value.automaticReplay === false && !value.message.includes("NEVER_") && (code === undefined || value.code === code)); cases++ } for (const tool of ["ping", "traceroute", "bandwidth", "dns", "route", "mtu"]) { const f = fixture() if (tool === "traceroute") f.raw([{ address: "192.0.2.2", loss: "0%", last: "1ms", avg: "1ms", "opaque-password": "NEVER_RAW" }]) if (tool === "bandwidth") f.raw([{ "tx-current": "0", "rx-current": "", "opaque-password": "NEVER_RAW" }]) if (tool === "dns") f.raw(["192.0.2.2"]) if (tool === "route") f.raw([{ ".id": "*1", "dst-address": "192.0.2.0/24", active: "true", gateway: "ether1", distance: "1", "opaque-password": "NEVER_RAW" }]) const prepared = await f.executor.prepare(1, request(tool)), result = await f.executor.execute(prepared) assert.equal(result.state, "completed"); assert.equal(result.httpStatus, 200); assert.equal(result.forwardingVerified, false) assert.ok(result.output); assert.ok(result.data); assert(!JSON.stringify(result).includes("NEVER_")); assert.equal(f.receipts.get(prepared.receiptId)?.state, "completed") assert.equal(f.events.filter(event => event.kind === "intent").length, tool === "dns" ? f.counts().dnsCalls : f.counts().sends) assert(f.signal() instanceof AbortSignal) const sends = f.counts(); assert.deepEqual(await f.executor.execute(prepared), result); assert.deepEqual(f.counts(), sends) if (tool === "bandwidth") { const native = f.events.find(event => event.kind === "native")!.value as { body: Record }; assert.equal(native.body.user, f.destination.username); assert.equal(native.body.password, f.destination.password); assert.equal(native.body.direction, "both") } cases++ } assert.deepEqual(planServerProbeExecution(request("ping", { pingCount: 999, pingSize: 1, pingTtl: 999 })).nativeBody, { address: "192.0.2.2", count: "100", size: "28", ttl: "255", interval: "0.2s" }); cases++ const trace = planServerProbeExecution(request("traceroute", { traceHopTimeout: "3s", traceProbeCount: 3, traceMaxHops: 64 })) assert.equal(trace.nativeBody!["max-hops"], "5"); assert.equal(trace.nativeBody!.timeout, "00:00:03"); assert.equal(planServerProbeExecution(request("traceroute", { traceHopTimeout: "800ms" })).nativeBody!.timeout, "00:00:00.800"); cases++ assert.equal(planServerProbeExecution(request("bandwidth", { bwDuration: 999 })).nativeBody!.duration, "120s"); assert.equal(planServerProbeExecution(request("bandwidth")).wireTimeoutMs, 30000); cases++ for (const malformed of [{ tool: "unknown" }, request("ping", { pingCount: "4" }), request("route", { target: "example.com" }), request("traceroute", { traceHopTimeout: "00:99:00" }), request("dns", { dnsType: "SOA" }), request("ping", { password: "caller-secret" })]) { assert.throws(() => planServerProbeExecution(malformed)); cases++ } for (const type of ["A", "AAAA", "MX", "NS", "TXT", "CNAME", "PTR"]) { const f = fixture(), rows: Record = { A: ["192.0.2.2"], AAAA: ["2001:db8::1"], MX: [{ priority: 10, exchange: "mail.example" }], NS: ["ns.example"], TXT: [["part1", "part2"]], CNAME: ["canonical.example"], PTR: ["ptr.example"] } f.raw(rows[type]); const result = await f.executor.execute(await f.executor.prepare(1, request("dns", { target: "example.com", dnsType: type }))) assert.equal(result.data!.resolver, "manager"); assert.match(result.output!, new RegExp(type)); assert.equal(f.counts().sends, 0); cases++ } { const f = fixture(); f.raw([{ seq: "0", status: "timeout" }]); const result = await f.executor.execute(await f.executor.prepare(1, request("mtu"))) assert.equal(f.counts().sends, 10); assert.match(result.output!, /could not determine/); assert.doesNotMatch(result.output!, /fragment needed|✓ ok/); cases++ } { const f = fixture(); f.raw([]); await failure(f.executor.execute(await f.executor.prepare(1, request("mtu"))), "MEASUREMENT_UNKNOWN"); assert.equal(f.counts().sends, 1) } { const f = fixture(); f.source.host = "router.example"; f.sourceFailure() const prepared = await f.executor.prepare(1, request("ping")), result = await f.executor.execute(prepared) assert.equal(result.state, "completed"); assert.equal(f.counts().dnsCalls, 1); assert.equal(f.counts().sends, 1) assert.equal((f.events.find(event => event.kind === "native")!.value as { body: Record }).body["src-address"], undefined) assert.equal(f.events.filter(event => event.kind === "received").length, 2); cases++ } { const f = fixture(); f.authority.scopeServerIds = [1]; f.metadata(99) await failure(f.executor.prepare(1, request("bandwidth")), "PERMISSION_DENIED") assert(!f.getIds.includes(99)); assert(!f.getIds.includes(2)); assert.deepEqual(f.events.find(event => event.kind === "metadata")?.value, [1]) } { const f = fixture(); f.beforeGet(() => { f.authority.active = false }) await failure(f.executor.prepare(1, request("bandwidth")), "PERMISSION_DENIED") assert.deepEqual(f.getIds, [1]); assert(!f.events.some(event => event.kind === "metadata")) } for (const mutate of [ (f: ReturnType) => { f.authority.policyProof = "c".repeat(64) }, (f: ReturnType) => { f.authority.ownerProof = "c".repeat(64) }, (f: ReturnType) => { f.authority.credentialId = randomUUID() }, (f: ReturnType) => { f.authority.generation = "changed" }, (f: ReturnType) => { f.source.routingEndpointEpoch = randomUUID() }, (f: ReturnType) => { f.destination.password = "rotated" }, (f: ReturnType) => { f.destination.enabled = false }, ]) { const f = fixture(), prepared = await f.executor.prepare(1, request("bandwidth")); mutate(f); await failure(f.executor.execute(prepared)); assert.equal(f.counts().sends, 0); assert(!f.events.some(event => event.kind === "claim")) } for (const kind of ["claim", "intent", "receipt", "received"] as const) { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); f.lost(kind) await failure(f.executor.execute(prepared), "JOURNAL_UNKNOWN") assert.equal(f.receipts.get(prepared.receiptId)?.state, "unknown"); assert.equal(f.counts().sends, ["claim", "intent"].includes(kind) ? 0 : 1) const sends = f.counts(); await failure(f.executor.execute(prepared), "REPLAY_FORBIDDEN"); assert.deepEqual(f.counts(), sends) } { const f = fixture(); f.source.host = "router.example"; f.raw(["192.0.2.1"]) const prepared = await f.executor.prepare(1, request("ping")); f.lost("received") await failure(f.executor.execute(prepared), "JOURNAL_UNKNOWN"); assert.equal(f.counts().dnsCalls, 1); assert.equal(f.counts().sends, 0); assert.equal(f.events.filter(event => event.kind === "received").length, 1) } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); f.duplicateFence() await failure(f.executor.execute(prepared), "REPLAY_FORBIDDEN"); assert.equal(f.counts().sends, 1); assert.equal(f.receipts.get(prepared.receiptId)?.state, "unknown") } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); f.concurrentFence() await failure(f.executor.execute(prepared), "REPLAY_FORBIDDEN"); assert.equal(f.counts().sends, 0) } for (const mutate of [ (value: ServerProbeReceipt) => { value.receiptId = randomUUID() }, (value: ServerProbeReceipt) => { value.tool = "dns" }, (value: ServerProbeReceipt) => { value.serverId = 99 }, (value: ServerProbeReceipt) => { value.destinationServerId = 99 }, (value: ServerProbeReceipt) => { value.completedStages = -1 }, (value: ServerProbeReceipt) => { value.attemptedStages = Number.NaN }, (value: ServerProbeReceipt) => { value.attemptedStages = 1.5 }, (value: ServerProbeReceipt) => { value.completedStages = 2 }, (value: ServerProbeReceipt) => { value.attemptedStages = value.completedStages = 3 }, (value: ServerProbeReceipt) => { value.data = { kind: "foreign" } }, ]) { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); await f.executor.execute(prepared) mutate(f.receipts.get(prepared.receiptId)!) await failure(f.executor.execute(prepared), "RECEIPT_INVALID"); assert.equal(f.counts().sends, 1) } { const f = fixture(); f.raw([{ "tx-current": `native user=${f.destination.username} secret=${f.destination.password}`, "rx-current": "0" }]) const prepared = await f.executor.prepare(1, request("bandwidth")), result = await f.executor.execute(prepared) assert(!JSON.stringify(result).includes("NEVER_")); assert(!JSON.stringify(f.receipts.get(prepared.receiptId)).includes("NEVER_")); assert.equal(result.receiptId, prepared.receiptId); assert.equal(result.state, "completed"); cases++ } { const f = fixture(); f.source.username = "a"; const prepared = await f.executor.prepare(1, request("ping")), result = await f.executor.execute(prepared) assert.equal(result.receiptId, prepared.receiptId); assert.equal(result.state, "completed"); assert.equal(result.data!.kind, "ping"); assert.equal(result.attemptedStages, 1) assert.equal((await f.executor.execute(prepared)).receiptId, prepared.receiptId); cases++ } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); f.beforeSend(() => { f.authority.active = false }) await failure(f.executor.execute(prepared), "PERMISSION_DENIED"); assert.equal(f.counts().sends, 0) } { const f = fixture(), prepared = await f.executor.prepare(1, request("mtu")); f.raw([{ status: "timeout" }]); f.afterSend(() => { f.source.password = "rotated" }) await failure(f.executor.execute(prepared), "STATE_CONFLICT"); assert.equal(f.counts().sends, 1); assert.equal(f.receipts.get(prepared.receiptId)?.state, "unknown") } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")); f.beforeSend(() => f.advance(300001)) await failure(f.executor.execute(prepared), "CANCELLED"); assert.equal(f.counts().sends, 0) } { const f = fixture(), controller = new AbortController(); f.pendingDns(); const prepared = await f.executor.prepare(1, request("dns")) const pending = f.executor.execute(prepared, controller.signal) while (!f.counts().dnsCalls) await new Promise(resolve => setTimeout(resolve, 1)) controller.abort(); await failure(pending, "CANCELLED"); assert.equal(f.signal()?.aborted, true); assert.equal(f.receipts.get(prepared.receiptId)?.state, "unknown") } { const f = fixture(); f.failure(); const prepared = await f.executor.prepare(1, request("ping")) await failure(f.executor.execute(prepared), "UPSTREAM_UNAVAILABLE"); const sends = f.counts(); await failure(f.executor.execute(prepared), "REPLAY_FORBIDDEN"); assert.deepEqual(f.counts(), sends) } { const view = projectServerProbeRoute([{ "dst-address": "0.0.0.0/0", active: "true", gateway: "default", distance: "1" }, { "dst-address": "192.0.2.0/24", active: "true", gateway: "scoped", distance: "10" }, { "dst-address": "192.0.2.2/32", active: "true", "routing-table": "foreign", gateway: "private" }, { "dst-address": "192.0.2.2/32", active: "unknown", gateway: "unconfirmed" }], "192.0.2.2") assert.match(view.output, /gateway: scoped/); assert.equal(view.data.unknown, 1); cases++ } { const f = fixture() Object.assign(f.authority, { transport: "rest", requestId: "real-http-request-id", issuedAt: 999000 }) delete f.authority.connectionId; delete f.authority.credentialId const prepared = await f.executor.prepare(1, request("ping")) await f.executor.execute(prepared) const claim = f.events.find(event => event.kind === "claim")!.value as Record assert.equal(claim.transport, "rest"); assert.equal(claim.requestId, "real-http-request-id") assert.equal(Object.hasOwn(claim, "connectionId"), false, "REST must not fabricate an MCP connection") assert.equal(claim.issuedAt, 999000); cases++ } for (const change of [{ connectionId: "fabricated" }, { issuedAt: 1000001 }, { requestId: "" }, { credentialId: "fabricated" }]) { const f = fixture() Object.assign(f.authority, { transport: "rest", requestId: "real-http-request-id", issuedAt: 999000 }) delete f.authority.connectionId; delete f.authority.credentialId Object.assign(f.authority, change) await failure(f.executor.prepare(1, request("ping")), "PERMISSION_DENIED") assert.equal(f.getIds.length, 0, "Malformed REST identity refuses before privileged reads") } { const f = fixture(), prepared = await f.executor.prepare(1, request("bandwidth")) const before = await f.executor.preflight(prepared) assert.deepEqual(before.serverIds, [1, 2]); assert.equal(before.receiptId, prepared.receiptId) assert.equal(f.events.some(event => ["claim", "native", "dns", "finish"].includes(event.kind)), false) assert.deepEqual(await f.executor.preflight(prepared), before); cases++ f.destination.password = "changed" await failure(f.executor.preflight(prepared), "STATE_CONFLICT") } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")), cancel = new AbortController() cancel.abort(); await failure(f.executor.preflight(prepared, cancel.signal), "CANCELLED") await failure(f.executor.execute({ ...prepared, expiresAt: new Date(2000001).toISOString() }), "INVALID_BINDING") assert.deepEqual(f.counts(), { sends: 0, dnsCalls: 0 }) } for (const code of ["OWNER_CLAIMS_REVOKED", "PROTECTED_OBJECT", "GRANT_INACTIVE", "UNAUTHORIZED", "MCP_DISABLED", "CONNECTION_REVOKED"]) { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")) f.beforeSend(() => { throw new McpError(code, "NEVER_PRIVATE_AUTHORITY_DIAGNOSTIC", 403) }) await failure(f.executor.execute(prepared), "PERMISSION_DENIED") assert.deepEqual(f.counts(), { sends: 0, dnsCalls: 0 }); assert.equal(f.receipts.get(prepared.receiptId)?.errorCode, "PERMISSION_DENIED") } { const f = fixture(), prepared = await f.executor.prepare(1, request("ping")) f.afterSend(() => { throw new McpError("OWNER_CLAIMS_REVOKED", "NEVER_PRIVATE_AUTHORITY_DIAGNOSTIC", 403) }) await failure(f.executor.execute(prepared), "PERMISSION_DENIED") assert.equal(f.counts().sends, 1); assert.equal(f.receipts.get(prepared.receiptId)?.state, "unknown") } { const f = fixture(), executor = createServerProbeExecutor({ ...f.deps, journal: undefined }), prepared = await executor.prepare(1, request("ping")) await executor.preflight(prepared) const reads = f.getIds.length await failure(executor.execute(prepared), "JOURNAL_UNAVAILABLE") assert.equal(f.getIds.length, reads); assert.deepEqual(f.counts(), { sends: 0, dnsCalls: 0 }) assert.equal(f.events.some(event => event.kind === "claim"), false) } { const f = fixture(), executor = createServerProbeExecutor({ ...f.deps, findDestination: async (_remote, _explicit, allowed) => { assert.deepEqual(allowed, [1, 2]); return { id: 2 } } }), prepared = await executor.prepare(1, request("bandwidth", { dstServerId: 999 })) assert.equal(prepared.destinationServerId, 2, "Scoped metadata resolver may return the documented host fallback") await executor.execute(prepared) assert.equal(f.receipts.get(prepared.receiptId)?.destinationServerId, 2); cases++ } console.log(`Server probe execution: ${cases} cases passed; all six tools, manager DNS7, private scoped BW credentials, source/authority fences, durable injected journal, loss/cancel/no replay; zero real I/O`)