/** Owned disposable PG + loopback HTTP acceptance of the real REST assembly. * Existing routes, production schema and MCP registration are untouched. */ import assert from "node:assert/strict" import { randomUUID } from "node:crypto" import { createServer } from "node:http" import { basename, dirname, resolve } from "node:path" import { realpath } from "node:fs/promises" import type { AuthUser } from "../lib/permissions.js" export async function acceptServerProbeStorageRestSql() { assert.equal(process.env.MMAPP_OWN_PROBE_STORAGE_JOURNAL, "true") const url = new URL(process.env.DATABASE_URL ?? "postgresql://invalid") assert(["postgres:", "postgresql:"].includes(url.protocol)); assert.equal(url.hostname, "127.0.0.1") assert.equal(decodeURIComponent(url.username), "mcp_test"); assert.equal(url.pathname, "/postgres") assert.equal(process.env.MMAPP_OWN_SQL_DB, "postgres"); assert.equal(url.search, ""); assert.equal(url.hash, "") assert(process.env.MCP_STATE_DIR) const state = resolve(process.env.MCP_STATE_DIR) assert.equal(basename(state), "state"); assert.match(basename(dirname(state)), /^mm-mcp-test-[A-Za-z0-9-]+$/) assert.equal(resolve(await realpath(state)).toLowerCase(), state.toLowerCase()) const { pool } = await import("../db/index.js") const { createStorageServerProbeRestExecution } = await import("./server-probe-storage-rest.js") const { PROBE_STORAGE_LIMITS } = await import("../modules/mcp/diagnostic-cancellation-storage.js") const { decryptSecret } = await import("../modules/mcp/security.js") const { buildApp } = await import("../index.js") const { SignJWT } = await import("jose") const { env } = await import("../config.js") assert.equal(env.authRequired, true) assert.equal((await pool.query("SELECT to_regclass('server_probe_storage_admissions') IS NOT NULL AS ready")).rows[0].ready, true) const priorSettings = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0]; assert(priorSettings) const actor: AuthUser = { id: "owned-rest-storage-" + randomUUID(), email: "fixture@example.test", name: "Owned REST", apps: ["mm"], permissions: ["mm:servers:write"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 3600000 } const sourceUser = "NEVER_REST_SOURCE_USER", sourcePassword = "NEVER_REST_SOURCE_PASSWORD" const peerUser = "NEVER_REST_PEER_USER", peerPassword = "NEVER_REST_PEER_PASSWORD" const privateValues = [sourceUser, sourcePassword, peerUser, peerPassword] const hidden = (value: unknown) => assert.equal(privateValues.some(secret => JSON.stringify(value).includes(secret)), false, "Private REST diagnostic material leaked") const serverIds: number[] = [], peerHost = "owned-peer-" + randomUUID() + ".invalid" let nativeCalls = 0, protocolErrors = 0, mode: "normal" | "error" | "hold" = "normal", activeAbort: AbortController | undefined let heldStarted!: () => void, heldClosed!: () => void, releaseHeld!: () => void let started!: Promise, closed!: Promise, heldGate!: Promise const resetHeld = () => { started = new Promise(yes => { heldStarted = yes }); closed = new Promise(yes => { heldClosed = yes }) heldGate = new Promise(yes => { releaseHeld = yes }) } resetHeld() let httpApp: import("fastify").FastifyInstance | undefined const barrier = async (pending: Promise, message: string) => { let timer: ReturnType | undefined try { await Promise.race([pending, new Promise((_yes, no) => { timer = setTimeout(() => no(Error(message)), 15000) })]) } finally { if (timer) clearTimeout(timer) } } const router = createServer(async (request, response) => { response.setHeader("content-type", "application/json") try { nativeCalls++ const selectedMode = mode assert.equal(request.socket.remoteAddress, "127.0.0.1"); assert.equal(request.method, "POST") assert.equal(request.headers.authorization === "Basic " + Buffer.from(`${sourceUser}:${sourcePassword}`).toString("base64"), true) let raw = ""; for await (const chunk of request) raw += chunk const body = JSON.parse(raw) as Record if (request.url === "/rest/tool/bandwidth-test") { assert.equal(body.address, peerHost); assert.equal(body.user === peerUser && body.password === peerPassword, true) assert.equal(body.duration, "3s"); assert.equal(body.protocol, "tcp"); assert.equal(body.direction, "both") response.end(JSON.stringify([{ "tx-current": "0", "rx-current": "42", "test-duration": "3s" }])); return } assert.equal(request.url, "/rest/tool/ping"); assert.equal(body.address, "127.0.0.1"); assert.equal(body.count, "1") if (selectedMode === "error") { response.statusCode = 500; response.end(JSON.stringify({ error: `${sourceUser}:${sourcePassword}:${peerUser}:${peerPassword}` })); return } if (selectedMode === "hold") { response.once("close", heldClosed); heldStarted(); await heldGate; if (response.destroyed) return } response.end(JSON.stringify([{ seq: "0", ttl: "64", time: "1ms", status: `${sourceUser}:${sourcePassword}` }, { sent: "1", received: "1", "packet-loss": "0%" }])) } catch { protocolErrors++; response.statusCode = 502; response.end('{"error":"Owned REST loopback protocol mismatch"}') } }) await new Promise((yes, no) => { router.once("error", no); router.listen(0, "127.0.0.1", yes) }) const execution = createStorageServerProbeRestExecution() const context = (kind: string, selectedActor: AuthUser | undefined = actor, authRequired = true, signal?: AbortSignal) => ({ actor: selectedActor, authRequired, signal, requestId: "owned-rest-" + kind + "-" + randomUUID(), acceptedAt: Date.now() }) const ping = { tool: "ping", target: "127.0.0.1", pingCount: 1 } const usage = async (owner: string) => { const row = (await pool.query("SELECT admissions,reserved_files,reserved_bytes FROM server_probe_storage_usage WHERE scope=$1", ["owner:" + owner])).rows[0] return { admissions: Number(row?.admissions ?? 0), files: Number(row?.reserved_files ?? 0), bytes: Number(row?.reserved_bytes ?? 0) } } const archive = async (requestId: string) => (await pool.query(`SELECT j.*,to_jsonb(j) ? 'receipt_ciphertext' AS anchor_has_ciphertext, a.owner_id,a.claim_token,a.claim_attempted_at,p.receipt_ciphertext,p.receipt_proof AS payload_proof, (SELECT count(*)::int FROM server_probe_execution_quarantines q WHERE q.receipt_id=j.receipt_id AND q.execution_id=j.execution_id) AS quarantines FROM server_probe_executions j JOIN server_probe_storage_admissions a USING(receipt_id) LEFT JOIN server_probe_execution_payloads p USING(receipt_id) WHERE j.request_id=$1`, [requestId])).rows const completed = async (requestId: string, ownerId: string, authentication: "jwt" | "local") => { const rows = await archive(requestId); assert.equal(rows.length, 1) const row = rows[0]; assert.equal(row.identity_kind, "rest"); assert.equal(row.authentication, authentication) assert.equal(row.actor_id, ownerId); assert.equal(row.owner_id, ownerId); assert.equal(row.mcp_connection_id, null); assert.equal(row.mcp_credential_id, null) assert.equal(row.state, "completed"); assert.equal(row.anchor_has_ciphertext, false); assert.equal(row.claim_token, null); assert(row.claim_attempted_at) assert.equal(row.quarantines, 0); assert.equal(typeof row.receipt_ciphertext, "string"); assert.equal(row.payload_proof === row.receipt_proof, true) hidden(JSON.parse(await decryptSecret(row.receipt_ciphertext))) return row } const success = (reply: Awaited>) => { hidden(reply); assert.equal(reply.statusCode, 200); assert.deepEqual(Object.keys(reply.body), ["output"]) } try { await pool.query("UPDATE mcp_settings SET enabled=false WHERE id=1") const sourceId = Number((await pool.query("INSERT INTO servers(name,host,port,use_ssl,verify_ssl,enabled,username,password) VALUES('owned-rest-source','127.0.0.1',$1,false,false,true,$2,$3) RETURNING id", [(router.address() as { port: number }).port, sourceUser, sourcePassword])).rows[0].id) serverIds.push(sourceId) const peerId = Number((await pool.query("INSERT INTO servers(name,host,enabled,username,password) VALUES('owned-rest-peer',$1,false,$2,$3) RETURNING id", [peerHost, peerUser, peerPassword])).rows[0].id) serverIds.push(peerId) const inactive = (await pool.query("INSERT INTO servers(name,host,enabled) SELECT $1||n::text,'198.51.100.254',false FROM generate_series(1,1001) n RETURNING id", ["owned-rest-inactive-" + randomUUID() + "-"])).rows serverIds.push(...inactive.map(row => Number(row.id))); assert.equal(inactive.length, 1001) const missingId = Number((await pool.query("SELECT max(id)+10000 AS id FROM servers")).rows[0].id) const firstContext = context("write"), first = await execution(String(sourceId), ping, firstContext) success(first); assert.equal((first.body as { output: string }).output, "PING 127.0.0.1\n seq=0 ttl=64 time=1ms status=[redacted]:[redacted]\n sent=1 received=1 packet-loss=0%") await completed(firstContext.requestId, actor.id, "jwt"); assert.equal(nativeCalls, 1) assert.equal((await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled, false) assert.deepEqual(await usage(actor.id), { admissions: 1, files: 1, bytes: 294912 }) const missing = await execution(String(missingId), ping, context("missing")); assert.equal(missing.statusCode, 404); assert.deepEqual(missing.body, { error: "Server not found" }) const readonly = await execution(String(missingId), ping, context("readonly", { ...actor, permissions: ["mm:servers:read"] })) assert.equal(readonly.statusCode, 403) const deniedContext = { ...context("unauth"), actor: undefined } const denied = await execution(String(sourceId), { ...ping, authRequired: false, ownerId: actor.id, kind: "local" }, deniedContext) assert.equal(denied.statusCode, 403); hidden(denied); assert.equal(nativeCalls, 1) const localBefore = await usage("local-operator"), forgedExecution = randomUUID() const localContext = { ...context("local", undefined, false), actor: undefined } const local = await execution(String(sourceId), { ...ping, ownerId: "forged-owner", kind: "mcp", authRequired: true, executionId: forgedExecution, requestId: "forged-request", connectionId: randomUUID(), credentialId: randomUUID() }, localContext) success(local) const localRow = await completed(localContext.requestId, "local-operator", "local") assert.equal(localRow.execution_id === forgedExecution, false) const localAfter = await usage("local-operator") assert.deepEqual(localAfter, { admissions: localBefore.admissions + 1, files: localBefore.files + 1, bytes: localBefore.bytes + 294912 }) assert.equal(nativeCalls, 2) const bandwidth = { tool: "bandwidth", bwRemoteAddress: peerHost, dstServerId: peerId, bwDuration: 3 } const disabled = await execution(String(sourceId), bandwidth, context("disabled-peer")) assert.equal(disabled.statusCode, 409); hidden(disabled); assert.equal(nativeCalls, 2) await pool.query("UPDATE servers SET enabled=true WHERE id=$1", [peerId]) const bwContext = context("bw-fallback"), bw = await execution(String(sourceId), { ...bandwidth, dstServerId: missingId }, bwContext) success(bw); assert.equal((bw.body as { output: string }).output, "bandwidth-test:\n tx=0 rx=42 t=3s") const bwRow = await completed(bwContext.requestId, actor.id, "jwt"); assert.equal(bwRow.destination_server_id, peerId); assert.equal(nativeCalls, 3) mode = "error" const failureContext = context("provider-error"), failure = await execution(String(sourceId), ping, failureContext) hidden(failure); assert.equal(failure.statusCode, 502); assert.deepEqual(Object.keys(failure.body), ["error"]) const failed = await archive(failureContext.requestId); assert.equal(failed.length, 1); assert.equal(failed[0].quarantines, 1); assert.equal(failed[0].receipt_ciphertext, null) assert.equal(nativeCalls, 4) mode = "hold"; activeAbort = new AbortController() const abortContext = context("abort", actor, true, activeAbort.signal), pending = execution(String(sourceId), ping, abortContext) await barrier(started, "Owned REST native did not reach held response barrier") activeAbort.abort() const aborted = await pending; hidden(aborted); assert.equal(aborted.statusCode, 502); assert.deepEqual(Object.keys(aborted.body), ["error"]) await barrier(closed, "REST abort did not close the owned native socket") const stopped = await archive(abortContext.requestId); assert.equal(stopped.length, 1) assert.equal(stopped[0].state, "claimed"); assert.equal(stopped[0].quarantines, 1); assert.equal(stopped[0].receipt_ciphertext, null) assert.equal(stopped[0].claim_token, null); assert(stopped[0].claim_attempted_at) assert.equal(nativeCalls, 5); releaseHeld(); mode = "normal" assert.deepEqual(await usage(actor.id), { admissions: 4, files: 4, bytes: 4 * 294912 }) const quotaActor = { ...actor, id: "owned-rest-quota-" + randomUUID() }, quotaLimits = { ...PROBE_STORAGE_LIMITS, ownerBytes: 294912, ownerFiles: 1, ownerAdmissions: 1 } const quotaRun = createStorageServerProbeRestExecution(quotaLimits), quotaContext = context("quota-first", quotaActor) success(await quotaRun(String(sourceId), ping, quotaContext)); await completed(quotaContext.requestId, quotaActor.id, "jwt") const beforeQuota = nativeCalls, anotherInstance = createStorageServerProbeRestExecution(quotaLimits) const exhausted = await anotherInstance(String(sourceId), ping, context("quota-exhausted", quotaActor)) hidden(exhausted); assert.equal(exhausted.statusCode, 503); assert.equal(nativeCalls, beforeQuota) assert.deepEqual(await usage(quotaActor.id), { admissions: 1, files: 1, bytes: 294912 }); assert.equal(protocolErrors, 0) // Real HTTP bridge: the actual auth plugin and route create trusted context. // No identity, auth mode or request ID comes from body/forwarded headers. httpApp = await buildApp({ logger: false, startScheduler: false }) const base = await httpApp.listen({ host: "127.0.0.1", port: 0 }), endpoint = `${base}/api/servers/${sourceId}/probes/run` const token = (owner: string, permission: string) => new SignJWT({ sub: owner, apps: ["mm"], permissions: [permission] }) .setProtectedHeader({ alg: "HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("1h") .sign(new TextEncoder().encode(env.jwtSecret)) const httpOwner = "owned-rest-http-" + randomUUID(), abortOwner = "owned-rest-http-abort-" + randomUUID() const forgedRequestId = randomUUID(), forgedBody = { ...ping, kind: "mcp", authRequired: false, ownerId: "forged-http-owner", connectionId: randomUUID(), credentialId: randomUUID(), requestId: forgedRequestId } const postHttp = (jwt?: string, signal?: AbortSignal) => fetch(endpoint, { method: "POST", signal, headers: { "content-type": "application/json", "x-request-id": forgedRequestId, ...(jwt ? { authorization: "Bearer " + jwt } : {}) }, body: JSON.stringify(forgedBody) }) const unauthHttp = await postHttp(); assert.equal(unauthHttp.status, 401); hidden(await unauthHttp.json()) const readonlyHttp = await postHttp(await token(httpOwner, "mm:servers:read")); assert.equal(readonlyHttp.status, 403); hidden(await readonlyHttp.json()) assert.equal(nativeCalls, 6) const writeHttp = await postHttp(await token(httpOwner, "mm:servers:write")), writeBody: unknown = await writeHttp.json() assert.equal(writeHttp.status, 200); hidden(writeBody) assert.deepEqual(writeBody, first.body); assert.equal(nativeCalls, 7) const httpRows = (await pool.query("SELECT request_id FROM server_probe_executions WHERE actor_id=$1 AND source_server_id=$2", [httpOwner, sourceId])).rows assert.equal(httpRows.length, 1); assert.match(httpRows[0].request_id, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i) assert.notEqual(httpRows[0].request_id, forgedRequestId); await completed(httpRows[0].request_id, httpOwner, "jwt") resetHeld(); mode = "hold"; activeAbort = new AbortController() const disconnect = postHttp(await token(abortOwner, "mm:servers:write"), activeAbort.signal).then( async response => { await response.body?.cancel(); return false }, () => true) await barrier(started, "HTTP bridge native did not reach held response barrier") activeAbort.abort(); assert.equal(await disconnect, true) await barrier(closed, "HTTP client disconnect did not close the owned native socket") const until = Date.now() + 15000 let disconnectedRows: Awaited> = [] while (Date.now() < until) { const identities = (await pool.query("SELECT request_id FROM server_probe_executions WHERE actor_id=$1 AND source_server_id=$2", [abortOwner, sourceId])).rows assert(identities.length <= 1) disconnectedRows = identities.length ? await archive(identities[0].request_id) : [] if (disconnectedRows.length === 1 && disconnectedRows[0].quarantines === 1) break await new Promise(yes => setTimeout(yes, 25)) } assert.equal(disconnectedRows.length, 1) const disconnected = disconnectedRows[0] assert.equal(disconnected.identity_kind, "rest"); assert.equal(disconnected.authentication, "jwt") assert.equal(disconnected.actor_id, abortOwner); assert.equal(disconnected.owner_id, abortOwner) assert.equal(disconnected.mcp_connection_id, null); assert.equal(disconnected.mcp_credential_id, null) assert.notEqual(disconnected.request_id, forgedRequestId); assert.equal(disconnected.quarantines, 1) assert.equal(disconnected.receipt_ciphertext, null); assert.equal(disconnected.claim_token, null); assert(disconnected.claim_attempted_at) assert.deepEqual(await usage(abortOwner), { admissions: 1, files: 1, bytes: 294912 }) assert.equal(nativeCalls, 8); assert.equal(protocolErrors, 0); releaseHeld(); mode = "normal" return { actualProductionRestAssembly: true, productionAuthorityAndCrypto: true, mcpDisabledRestWorks: true, unrelatedInactiveServers: inactive.length, boundedParticipantsWithoutFleetEnumeration: true, authenticatedMissingServer404: true, readOnlyAndUnauthenticatedDenied: true, fixedLocalRestIdentityIgnoresBodyCapabilities: true, bandwidthDisabledAndFallback: true, safeProviderFailure: true, abortStatus: aborted.statusCode, abortSocketClosedAndQuarantined: true, sharedQuotaAcrossFactoryInstances: true, separateEncryptedPayload: true, nativeLoopbackHttpCalls: nativeCalls, actualHttpRouteSignedJwt: true, clientDisconnect: true, actualApplicationBuildRegistration: true, routesRegisteredInFixtureOnly: false, nativeOrManagerDnsCalls: 0, allSixToolsClaimed: false, routesRegistered: true, productionDdlApplied: false } } finally { activeAbort?.abort(); releaseHeld() try { if (serverIds.length) await pool.query("DELETE FROM servers WHERE id=ANY($1::int[])", [serverIds]) } finally { try { await pool.query("UPDATE mcp_settings SET enabled=$1 WHERE id=1", [priorSettings.enabled]) } finally { try { await httpApp?.close() } finally { router.closeAllConnections(); await new Promise(yes => router.close(() => yes())) } } } } }