228 lines
19 KiB
TypeScript
228 lines
19 KiB
TypeScript
/** Owned disposable PG + loopback HTTP acceptance of the real REST assembly.
|
|
* Existing routes, production schema and MCP registration are untouched. */
|
|
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { createServer } from "node:http"
|
|
import { basename, dirname, resolve } from "node:path"
|
|
import { realpath } from "node:fs/promises"
|
|
import type { AuthUser } from "../lib/permissions.js"
|
|
|
|
export async function acceptServerProbeStorageRestSql() {
|
|
assert.equal(process.env.MMAPP_OWN_PROBE_STORAGE_JOURNAL, "true")
|
|
const url = new URL(process.env.DATABASE_URL ?? "postgresql://invalid")
|
|
assert(["postgres:", "postgresql:"].includes(url.protocol)); assert.equal(url.hostname, "127.0.0.1")
|
|
assert.equal(decodeURIComponent(url.username), "mcp_test"); assert.equal(url.pathname, "/postgres")
|
|
assert.equal(process.env.MMAPP_OWN_SQL_DB, "postgres"); assert.equal(url.search, ""); assert.equal(url.hash, "")
|
|
assert(process.env.MCP_STATE_DIR)
|
|
const state = resolve(process.env.MCP_STATE_DIR)
|
|
assert.equal(basename(state), "state"); assert.match(basename(dirname(state)), /^mm-mcp-test-[A-Za-z0-9-]+$/)
|
|
assert.equal(resolve(await realpath(state)).toLowerCase(), state.toLowerCase())
|
|
const { pool } = await import("../db/index.js")
|
|
const { createStorageServerProbeRestExecution } = await import("./server-probe-storage-rest.js")
|
|
const { PROBE_STORAGE_LIMITS } = await import("../modules/mcp/diagnostic-cancellation-storage.js")
|
|
const { decryptSecret } = await import("../modules/mcp/security.js")
|
|
const { buildApp } = await import("../index.js")
|
|
const { SignJWT } = await import("jose")
|
|
const { env } = await import("../config.js")
|
|
assert.equal(env.authRequired, true)
|
|
assert.equal((await pool.query("SELECT to_regclass('server_probe_storage_admissions') IS NOT NULL AS ready")).rows[0].ready, true)
|
|
const priorSettings = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0]; assert(priorSettings)
|
|
const actor: AuthUser = { id: "owned-rest-storage-" + randomUUID(), email: "[email protected]", name: "Owned REST", apps: ["mm"],
|
|
permissions: ["mm:servers:write"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 3600000 }
|
|
const sourceUser = "NEVER_REST_SOURCE_USER", sourcePassword = "NEVER_REST_SOURCE_PASSWORD"
|
|
const peerUser = "NEVER_REST_PEER_USER", peerPassword = "NEVER_REST_PEER_PASSWORD"
|
|
const privateValues = [sourceUser, sourcePassword, peerUser, peerPassword]
|
|
const hidden = (value: unknown) => assert.equal(privateValues.some(secret => JSON.stringify(value).includes(secret)), false, "Private REST diagnostic material leaked")
|
|
const serverIds: number[] = [], peerHost = "owned-peer-" + randomUUID() + ".invalid"
|
|
let nativeCalls = 0, protocolErrors = 0, mode: "normal" | "error" | "hold" = "normal", activeAbort: AbortController | undefined
|
|
let heldStarted!: () => void, heldClosed!: () => void, releaseHeld!: () => void
|
|
let started!: Promise<void>, closed!: Promise<void>, heldGate!: Promise<void>
|
|
const resetHeld = () => {
|
|
started = new Promise<void>(yes => { heldStarted = yes }); closed = new Promise<void>(yes => { heldClosed = yes })
|
|
heldGate = new Promise<void>(yes => { releaseHeld = yes })
|
|
}
|
|
resetHeld()
|
|
let httpApp: import("fastify").FastifyInstance | undefined
|
|
const barrier = async (pending: Promise<void>, message: string) => {
|
|
let timer: ReturnType<typeof setTimeout> | undefined
|
|
try { await Promise.race([pending, new Promise<never>((_yes, no) => { timer = setTimeout(() => no(Error(message)), 15000) })]) }
|
|
finally { if (timer) clearTimeout(timer) }
|
|
}
|
|
const router = createServer(async (request, response) => {
|
|
response.setHeader("content-type", "application/json")
|
|
try {
|
|
nativeCalls++
|
|
const selectedMode = mode
|
|
assert.equal(request.socket.remoteAddress, "127.0.0.1"); assert.equal(request.method, "POST")
|
|
assert.equal(request.headers.authorization === "Basic " + Buffer.from(`${sourceUser}:${sourcePassword}`).toString("base64"), true)
|
|
let raw = ""; for await (const chunk of request) raw += chunk
|
|
const body = JSON.parse(raw) as Record<string, unknown>
|
|
if (request.url === "/rest/tool/bandwidth-test") {
|
|
assert.equal(body.address, peerHost); assert.equal(body.user === peerUser && body.password === peerPassword, true)
|
|
assert.equal(body.duration, "3s"); assert.equal(body.protocol, "tcp"); assert.equal(body.direction, "both")
|
|
response.end(JSON.stringify([{ "tx-current": "0", "rx-current": "42", "test-duration": "3s" }])); return
|
|
}
|
|
assert.equal(request.url, "/rest/tool/ping"); assert.equal(body.address, "127.0.0.1"); assert.equal(body.count, "1")
|
|
if (selectedMode === "error") {
|
|
response.statusCode = 500; response.end(JSON.stringify({ error: `${sourceUser}:${sourcePassword}:${peerUser}:${peerPassword}` })); return
|
|
}
|
|
if (selectedMode === "hold") { response.once("close", heldClosed); heldStarted(); await heldGate; if (response.destroyed) return }
|
|
response.end(JSON.stringify([{ seq: "0", ttl: "64", time: "1ms", status: `${sourceUser}:${sourcePassword}` }, { sent: "1", received: "1", "packet-loss": "0%" }]))
|
|
} catch { protocolErrors++; response.statusCode = 502; response.end('{"error":"Owned REST loopback protocol mismatch"}') }
|
|
})
|
|
await new Promise<void>((yes, no) => { router.once("error", no); router.listen(0, "127.0.0.1", yes) })
|
|
const execution = createStorageServerProbeRestExecution()
|
|
const context = (kind: string, selectedActor: AuthUser | undefined = actor, authRequired = true, signal?: AbortSignal) => ({ actor: selectedActor, authRequired, signal, requestId: "owned-rest-" + kind + "-" + randomUUID(), acceptedAt: Date.now() })
|
|
const ping = { tool: "ping", target: "127.0.0.1", pingCount: 1 }
|
|
const usage = async (owner: string) => {
|
|
const row = (await pool.query("SELECT admissions,reserved_files,reserved_bytes FROM server_probe_storage_usage WHERE scope=$1", ["owner:" + owner])).rows[0]
|
|
return { admissions: Number(row?.admissions ?? 0), files: Number(row?.reserved_files ?? 0), bytes: Number(row?.reserved_bytes ?? 0) }
|
|
}
|
|
const archive = async (requestId: string) => (await pool.query(`SELECT j.*,to_jsonb(j) ? 'receipt_ciphertext' AS anchor_has_ciphertext,
|
|
a.owner_id,a.claim_token,a.claim_attempted_at,p.receipt_ciphertext,p.receipt_proof AS payload_proof,
|
|
(SELECT count(*)::int FROM server_probe_execution_quarantines q WHERE q.receipt_id=j.receipt_id AND q.execution_id=j.execution_id) AS quarantines
|
|
FROM server_probe_executions j JOIN server_probe_storage_admissions a USING(receipt_id)
|
|
LEFT JOIN server_probe_execution_payloads p USING(receipt_id) WHERE j.request_id=$1`, [requestId])).rows
|
|
const completed = async (requestId: string, ownerId: string, authentication: "jwt" | "local") => {
|
|
const rows = await archive(requestId); assert.equal(rows.length, 1)
|
|
const row = rows[0]; assert.equal(row.identity_kind, "rest"); assert.equal(row.authentication, authentication)
|
|
assert.equal(row.actor_id, ownerId); assert.equal(row.owner_id, ownerId); assert.equal(row.mcp_connection_id, null); assert.equal(row.mcp_credential_id, null)
|
|
assert.equal(row.state, "completed"); assert.equal(row.anchor_has_ciphertext, false); assert.equal(row.claim_token, null); assert(row.claim_attempted_at)
|
|
assert.equal(row.quarantines, 0); assert.equal(typeof row.receipt_ciphertext, "string"); assert.equal(row.payload_proof === row.receipt_proof, true)
|
|
hidden(JSON.parse(await decryptSecret(row.receipt_ciphertext)))
|
|
return row
|
|
}
|
|
const success = (reply: Awaited<ReturnType<typeof execution>>) => { hidden(reply); assert.equal(reply.statusCode, 200); assert.deepEqual(Object.keys(reply.body), ["output"]) }
|
|
try {
|
|
await pool.query("UPDATE mcp_settings SET enabled=false WHERE id=1")
|
|
const sourceId = Number((await pool.query("INSERT INTO servers(name,host,port,use_ssl,verify_ssl,enabled,username,password) VALUES('owned-rest-source','127.0.0.1',$1,false,false,true,$2,$3) RETURNING id", [(router.address() as { port: number }).port, sourceUser, sourcePassword])).rows[0].id)
|
|
serverIds.push(sourceId)
|
|
const peerId = Number((await pool.query("INSERT INTO servers(name,host,enabled,username,password) VALUES('owned-rest-peer',$1,false,$2,$3) RETURNING id", [peerHost, peerUser, peerPassword])).rows[0].id)
|
|
serverIds.push(peerId)
|
|
const inactive = (await pool.query("INSERT INTO servers(name,host,enabled) SELECT $1||n::text,'198.51.100.254',false FROM generate_series(1,1001) n RETURNING id", ["owned-rest-inactive-" + randomUUID() + "-"])).rows
|
|
serverIds.push(...inactive.map(row => Number(row.id))); assert.equal(inactive.length, 1001)
|
|
const missingId = Number((await pool.query("SELECT max(id)+10000 AS id FROM servers")).rows[0].id)
|
|
const firstContext = context("write"), first = await execution(String(sourceId), ping, firstContext)
|
|
success(first); assert.equal((first.body as { output: string }).output, "PING 127.0.0.1\n seq=0 ttl=64 time=1ms status=[redacted]:[redacted]\n sent=1 received=1 packet-loss=0%")
|
|
await completed(firstContext.requestId, actor.id, "jwt"); assert.equal(nativeCalls, 1)
|
|
assert.equal((await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0].enabled, false)
|
|
assert.deepEqual(await usage(actor.id), { admissions: 1, files: 1, bytes: 294912 })
|
|
const missing = await execution(String(missingId), ping, context("missing")); assert.equal(missing.statusCode, 404); assert.deepEqual(missing.body, { error: "Server not found" })
|
|
const readonly = await execution(String(missingId), ping, context("readonly", { ...actor, permissions: ["mm:servers:read"] }))
|
|
assert.equal(readonly.statusCode, 403)
|
|
const deniedContext = { ...context("unauth"), actor: undefined }
|
|
const denied = await execution(String(sourceId), { ...ping, authRequired: false, ownerId: actor.id, kind: "local" }, deniedContext)
|
|
assert.equal(denied.statusCode, 403); hidden(denied); assert.equal(nativeCalls, 1)
|
|
|
|
const localBefore = await usage("local-operator"), forgedExecution = randomUUID()
|
|
const localContext = { ...context("local", undefined, false), actor: undefined }
|
|
const local = await execution(String(sourceId), { ...ping, ownerId: "forged-owner", kind: "mcp", authRequired: true,
|
|
executionId: forgedExecution, requestId: "forged-request", connectionId: randomUUID(), credentialId: randomUUID() }, localContext)
|
|
success(local)
|
|
const localRow = await completed(localContext.requestId, "local-operator", "local")
|
|
assert.equal(localRow.execution_id === forgedExecution, false)
|
|
const localAfter = await usage("local-operator")
|
|
assert.deepEqual(localAfter, { admissions: localBefore.admissions + 1, files: localBefore.files + 1, bytes: localBefore.bytes + 294912 })
|
|
assert.equal(nativeCalls, 2)
|
|
|
|
const bandwidth = { tool: "bandwidth", bwRemoteAddress: peerHost, dstServerId: peerId, bwDuration: 3 }
|
|
const disabled = await execution(String(sourceId), bandwidth, context("disabled-peer"))
|
|
assert.equal(disabled.statusCode, 409); hidden(disabled); assert.equal(nativeCalls, 2)
|
|
await pool.query("UPDATE servers SET enabled=true WHERE id=$1", [peerId])
|
|
const bwContext = context("bw-fallback"), bw = await execution(String(sourceId), { ...bandwidth, dstServerId: missingId }, bwContext)
|
|
success(bw); assert.equal((bw.body as { output: string }).output, "bandwidth-test:\n tx=0 rx=42 t=3s")
|
|
const bwRow = await completed(bwContext.requestId, actor.id, "jwt"); assert.equal(bwRow.destination_server_id, peerId); assert.equal(nativeCalls, 3)
|
|
|
|
mode = "error"
|
|
const failureContext = context("provider-error"), failure = await execution(String(sourceId), ping, failureContext)
|
|
hidden(failure); assert.equal(failure.statusCode, 502); assert.deepEqual(Object.keys(failure.body), ["error"])
|
|
const failed = await archive(failureContext.requestId); assert.equal(failed.length, 1); assert.equal(failed[0].quarantines, 1); assert.equal(failed[0].receipt_ciphertext, null)
|
|
assert.equal(nativeCalls, 4)
|
|
|
|
mode = "hold"; activeAbort = new AbortController()
|
|
const abortContext = context("abort", actor, true, activeAbort.signal), pending = execution(String(sourceId), ping, abortContext)
|
|
await barrier(started, "Owned REST native did not reach held response barrier")
|
|
activeAbort.abort()
|
|
const aborted = await pending; hidden(aborted); assert.equal(aborted.statusCode, 502); assert.deepEqual(Object.keys(aborted.body), ["error"])
|
|
await barrier(closed, "REST abort did not close the owned native socket")
|
|
const stopped = await archive(abortContext.requestId); assert.equal(stopped.length, 1)
|
|
assert.equal(stopped[0].state, "claimed"); assert.equal(stopped[0].quarantines, 1); assert.equal(stopped[0].receipt_ciphertext, null)
|
|
assert.equal(stopped[0].claim_token, null); assert(stopped[0].claim_attempted_at)
|
|
assert.equal(nativeCalls, 5); releaseHeld(); mode = "normal"
|
|
assert.deepEqual(await usage(actor.id), { admissions: 4, files: 4, bytes: 4 * 294912 })
|
|
|
|
const quotaActor = { ...actor, id: "owned-rest-quota-" + randomUUID() }, quotaLimits = { ...PROBE_STORAGE_LIMITS, ownerBytes: 294912, ownerFiles: 1, ownerAdmissions: 1 }
|
|
const quotaRun = createStorageServerProbeRestExecution(quotaLimits), quotaContext = context("quota-first", quotaActor)
|
|
success(await quotaRun(String(sourceId), ping, quotaContext)); await completed(quotaContext.requestId, quotaActor.id, "jwt")
|
|
const beforeQuota = nativeCalls, anotherInstance = createStorageServerProbeRestExecution(quotaLimits)
|
|
const exhausted = await anotherInstance(String(sourceId), ping, context("quota-exhausted", quotaActor))
|
|
hidden(exhausted); assert.equal(exhausted.statusCode, 503); assert.equal(nativeCalls, beforeQuota)
|
|
assert.deepEqual(await usage(quotaActor.id), { admissions: 1, files: 1, bytes: 294912 }); assert.equal(protocolErrors, 0)
|
|
|
|
// Real HTTP bridge: the actual auth plugin and route create trusted context.
|
|
// No identity, auth mode or request ID comes from body/forwarded headers.
|
|
httpApp = await buildApp({ logger: false, startScheduler: false })
|
|
const base = await httpApp.listen({ host: "127.0.0.1", port: 0 }), endpoint = `${base}/api/servers/${sourceId}/probes/run`
|
|
const token = (owner: string, permission: string) => new SignJWT({ sub: owner, apps: ["mm"], permissions: [permission] })
|
|
.setProtectedHeader({ alg: "HS256" }).setIssuer(env.authIssuer).setIssuedAt().setExpirationTime("1h")
|
|
.sign(new TextEncoder().encode(env.jwtSecret))
|
|
const httpOwner = "owned-rest-http-" + randomUUID(), abortOwner = "owned-rest-http-abort-" + randomUUID()
|
|
const forgedRequestId = randomUUID(), forgedBody = { ...ping, kind: "mcp", authRequired: false,
|
|
ownerId: "forged-http-owner", connectionId: randomUUID(), credentialId: randomUUID(), requestId: forgedRequestId }
|
|
const postHttp = (jwt?: string, signal?: AbortSignal) => fetch(endpoint, { method: "POST", signal,
|
|
headers: { "content-type": "application/json", "x-request-id": forgedRequestId, ...(jwt ? { authorization: "Bearer " + jwt } : {}) },
|
|
body: JSON.stringify(forgedBody) })
|
|
const unauthHttp = await postHttp(); assert.equal(unauthHttp.status, 401); hidden(await unauthHttp.json())
|
|
const readonlyHttp = await postHttp(await token(httpOwner, "mm:servers:read")); assert.equal(readonlyHttp.status, 403); hidden(await readonlyHttp.json())
|
|
assert.equal(nativeCalls, 6)
|
|
const writeHttp = await postHttp(await token(httpOwner, "mm:servers:write")), writeBody: unknown = await writeHttp.json()
|
|
assert.equal(writeHttp.status, 200); hidden(writeBody)
|
|
assert.deepEqual(writeBody, first.body); assert.equal(nativeCalls, 7)
|
|
const httpRows = (await pool.query("SELECT request_id FROM server_probe_executions WHERE actor_id=$1 AND source_server_id=$2", [httpOwner, sourceId])).rows
|
|
assert.equal(httpRows.length, 1); assert.match(httpRows[0].request_id, /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i)
|
|
assert.notEqual(httpRows[0].request_id, forgedRequestId); await completed(httpRows[0].request_id, httpOwner, "jwt")
|
|
|
|
resetHeld(); mode = "hold"; activeAbort = new AbortController()
|
|
const disconnect = postHttp(await token(abortOwner, "mm:servers:write"), activeAbort.signal).then(
|
|
async response => { await response.body?.cancel(); return false }, () => true)
|
|
await barrier(started, "HTTP bridge native did not reach held response barrier")
|
|
activeAbort.abort(); assert.equal(await disconnect, true)
|
|
await barrier(closed, "HTTP client disconnect did not close the owned native socket")
|
|
const until = Date.now() + 15000
|
|
let disconnectedRows: Awaited<ReturnType<typeof archive>> = []
|
|
while (Date.now() < until) {
|
|
const identities = (await pool.query("SELECT request_id FROM server_probe_executions WHERE actor_id=$1 AND source_server_id=$2", [abortOwner, sourceId])).rows
|
|
assert(identities.length <= 1)
|
|
disconnectedRows = identities.length ? await archive(identities[0].request_id) : []
|
|
if (disconnectedRows.length === 1 && disconnectedRows[0].quarantines === 1) break
|
|
await new Promise<void>(yes => setTimeout(yes, 25))
|
|
}
|
|
assert.equal(disconnectedRows.length, 1)
|
|
const disconnected = disconnectedRows[0]
|
|
assert.equal(disconnected.identity_kind, "rest"); assert.equal(disconnected.authentication, "jwt")
|
|
assert.equal(disconnected.actor_id, abortOwner); assert.equal(disconnected.owner_id, abortOwner)
|
|
assert.equal(disconnected.mcp_connection_id, null); assert.equal(disconnected.mcp_credential_id, null)
|
|
assert.notEqual(disconnected.request_id, forgedRequestId); assert.equal(disconnected.quarantines, 1)
|
|
assert.equal(disconnected.receipt_ciphertext, null); assert.equal(disconnected.claim_token, null); assert(disconnected.claim_attempted_at)
|
|
assert.deepEqual(await usage(abortOwner), { admissions: 1, files: 1, bytes: 294912 })
|
|
assert.equal(nativeCalls, 8); assert.equal(protocolErrors, 0); releaseHeld(); mode = "normal"
|
|
return { actualProductionRestAssembly: true, productionAuthorityAndCrypto: true, mcpDisabledRestWorks: true,
|
|
unrelatedInactiveServers: inactive.length, boundedParticipantsWithoutFleetEnumeration: true, authenticatedMissingServer404: true,
|
|
readOnlyAndUnauthenticatedDenied: true, fixedLocalRestIdentityIgnoresBodyCapabilities: true, bandwidthDisabledAndFallback: true,
|
|
safeProviderFailure: true, abortStatus: aborted.statusCode, abortSocketClosedAndQuarantined: true,
|
|
sharedQuotaAcrossFactoryInstances: true, separateEncryptedPayload: true, nativeLoopbackHttpCalls: nativeCalls,
|
|
actualHttpRouteSignedJwt: true, clientDisconnect: true, actualApplicationBuildRegistration: true, routesRegisteredInFixtureOnly: false,
|
|
nativeOrManagerDnsCalls: 0, allSixToolsClaimed: false, routesRegistered: true, productionDdlApplied: false }
|
|
} finally {
|
|
activeAbort?.abort(); releaseHeld()
|
|
try { if (serverIds.length) await pool.query("DELETE FROM servers WHERE id=ANY($1::int[])", [serverIds]) }
|
|
finally {
|
|
try { await pool.query("UPDATE mcp_settings SET enabled=$1 WHERE id=1", [priorSettings.enabled]) }
|
|
finally {
|
|
try { await httpApp?.close() }
|
|
finally { router.closeAllConnections(); await new Promise<void>(yes => router.close(() => yes())) }
|
|
}
|
|
}
|
|
}
|
|
}
|