239 lines
20 KiB
TypeScript
239 lines
20 KiB
TypeScript
/** Owned disposable-PG/loopback acceptance of the public production registry and
|
|
* storage host. No executor/authority overrides or real router. */
|
|
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { createServer } from "node:http"
|
|
import { basename, dirname, resolve } from "node:path"
|
|
import { realpath } from "node:fs/promises"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import type { PoolClient } from "pg"
|
|
import type { AuthUser } from "../../lib/permissions.js"
|
|
|
|
export async function acceptServerProbeStorageHostSql() {
|
|
// Guard BEFORE loading configured pools, crypto, runtime, or the host.
|
|
assert.equal(process.env.MMAPP_OWN_PROBE_STORAGE_JOURNAL, "true")
|
|
const url = new URL(process.env.DATABASE_URL ?? "postgresql://invalid")
|
|
assert(["postgres:", "postgresql:"].includes(url.protocol)); assert.equal(url.hostname, "127.0.0.1")
|
|
assert.equal(decodeURIComponent(url.username), "mcp_test"); assert.equal(url.pathname, "/postgres")
|
|
assert.equal(process.env.MMAPP_OWN_SQL_DB, "postgres"); assert.equal(url.search, ""); assert.equal(url.hash, "")
|
|
assert(process.env.MCP_STATE_DIR)
|
|
const state = resolve(process.env.MCP_STATE_DIR)
|
|
assert.equal(basename(state), "state"); assert.match(basename(dirname(state)), /^mm-mcp-test-[A-Za-z0-9-]+$/)
|
|
assert.equal(resolve(await realpath(state)).toLowerCase(), state.toLowerCase())
|
|
const { pool } = await import("../../db/index.js")
|
|
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
|
const { createStorageServerProbeMcpAdapter } = await import("./server-probe-storage-host.js")
|
|
const { findAction, findActionByOperation } = await import("./actions.js")
|
|
const { findMcpService } = await import("./service-registry.js")
|
|
const { SERVER_PROBE_CANCEL_OPERATION } = await import("./server-probe-registration.js")
|
|
const { SERVER_PROBE_OPERATION, SERVER_PROBE_RECEIPT_READ_OPERATION } = await import("../../services/server-probe-authority.js")
|
|
const { createMcpConnection, authenticateMcpToken, getConnection, rotateConnectionToken } = await import("./store.js")
|
|
const { invokeMcpTool, approvePlan, applyPlan } = await import("./runtime.js")
|
|
const { decryptSecret } = await import("./security.js")
|
|
const { McpError } = await import("./security.js")
|
|
const { connectProbeStorage } = await import("../../services/server-probe-storage-connection.js")
|
|
assert.equal((await pool.query("SELECT to_regclass('server_probe_storage_admissions') IS NOT NULL AS ready")).rows[0].ready, true)
|
|
assert(findMcpService(SERVER_PROBE_OPERATION))
|
|
assert.equal(findActionByOperation(SERVER_PROBE_OPERATION).operation.toolName, "mm_server_probe_plan")
|
|
assert.equal(findAction("mm_server_probe_receipt_get").operation.id, SERVER_PROBE_RECEIPT_READ_OPERATION)
|
|
assert.equal(findAction("mm_server_probe_cancel").operation.id, SERVER_PROBE_CANCEL_OPERATION)
|
|
const host = createStorageServerProbeMcpAdapter()
|
|
const actor: AuthUser = { id: "owned-probe-host-" + randomUUID(), email: "[email protected]", name: "Owned host", apps: ["mm"],
|
|
permissions: ["mm:servers:write", "mm:servers:read", "mm:mcp:approve"], issuedAt: Date.now() - 1000, expiresAt: Date.now() + 3600000 }
|
|
const username = "NEVER_STORAGE_SOURCE_USER", password = "NEVER_STORAGE_SOURCE_PASSWORD"
|
|
const privateValues = [username, password]
|
|
const hidden = (value: unknown) => { const text = JSON.stringify(value); assert.equal(privateValues.some(secret => text.includes(secret)), false, "Private source or credential data leaked") }
|
|
let requests = 0, protocolErrors = 0, serverId: number | undefined
|
|
let secondValidated!: () => void, secondClosed!: () => void, releaseSecondResponse!: () => void
|
|
const secondRequest = new Promise<void>(yes => { secondValidated = yes })
|
|
const secondSocketClosed = new Promise<void>(yes => { secondClosed = yes })
|
|
const secondResponseGate = new Promise<void>(yes => { releaseSecondResponse = yes })
|
|
const boundedBarrier = async (pending: Promise<void>, message: string) => {
|
|
let timer: ReturnType<typeof setTimeout> | undefined
|
|
try { await Promise.race([pending, new Promise<never>((_yes, no) => { timer = setTimeout(() => no(Error(message)), 15000) })]) }
|
|
finally { if (timer) clearTimeout(timer) }
|
|
}
|
|
const connectionIds: string[] = []
|
|
const router = createServer(async (request, response) => {
|
|
response.setHeader("content-type", "application/json")
|
|
try {
|
|
requests++
|
|
assert.equal(request.socket.remoteAddress, "127.0.0.1")
|
|
assert.equal(request.method, "POST"); assert.equal(request.url, "/rest/tool/ping")
|
|
assert.equal(request.headers.authorization === "Basic " + Buffer.from(`${username}:${password}`).toString("base64"), true)
|
|
let raw = ""; for await (const chunk of request) raw += chunk
|
|
const body = JSON.parse(raw) as Record<string, unknown>
|
|
assert.equal(body.address, "127.0.0.1"); assert.equal(body.count, "1")
|
|
if (requests === 2) {
|
|
response.once("close", secondClosed)
|
|
secondValidated()
|
|
await secondResponseGate
|
|
if (response.destroyed) return
|
|
}
|
|
response.end(JSON.stringify([{ seq: "0", ttl: "64", time: "1ms", status: `${username}:${password}` }, { sent: "1", received: "1", "packet-loss": "0%" }]))
|
|
} catch { protocolErrors++; response.statusCode = 502; response.end('{"error":"Owned loopback protocol mismatch"}') }
|
|
})
|
|
await new Promise<void>((yes, no) => { router.once("error", no); router.listen(0, "127.0.0.1", yes) })
|
|
const priorSettings = (await pool.query("SELECT enabled FROM mcp_settings WHERE id=1")).rows[0]
|
|
assert(priorSettings)
|
|
const wait = async (operationId: string) => {
|
|
for (const deadline = Date.now() + 90000; Date.now() < deadline;) {
|
|
const row = (await pool.query("SELECT id,state,verification,error_code,result FROM mcp_operations WHERE id=$1", [operationId])).rows[0]
|
|
assert(row)
|
|
if (!["pending", "running"].includes(row.state)) return row
|
|
await delay(25)
|
|
}
|
|
throw Error("Owned host operation remains active; no retry permitted")
|
|
}
|
|
try {
|
|
await pool.query("UPDATE mcp_settings SET enabled=true WHERE id=1")
|
|
serverId = Number((await pool.query("INSERT INTO servers(name,host,port,use_ssl,verify_ssl,enabled,username,password) VALUES('owned-storage-host','127.0.0.1',$1,false,false,true,$2,$3) RETURNING id", [(router.address() as { port: number }).port, username, password])).rows[0].id)
|
|
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: "Owned storage host", mode: "approval", expiresAt: new Date(actor.expiresAt!).toISOString(),
|
|
permissions: ["mm:servers:write", "mm:servers:read"], scope: { serverIds: [serverId], operationIds: [SERVER_PROBE_OPERATION, SERVER_PROBE_RECEIPT_READ_OPERATION, SERVER_PROBE_CANCEL_OPERATION], allowGlobal: false }, limits: { callsPerMinute: 120, mutationsPerHour: 1000 } }), actor)
|
|
connectionIds.push(access.connection.id); privateValues.push(access.token)
|
|
const principal = await authenticateMcpToken(access.token), raw = { serverId, request: { tool: "ping", target: "127.0.0.1", pingCount: 1 } }
|
|
const cancel = (operationId: string) => invokeMcpTool(principal, "mm_server_probe_cancel", { operationId }) as ReturnType<typeof host.cancel>
|
|
|
|
// Actual planner transactionClient authority/source reads share one backend;
|
|
// a detached ALS child must refuse instead of falling back to the pool.
|
|
const borrowed = await pool.connect(), originalPoolQuery = pool.query, originalQuery = borrowed.query.bind(borrowed)
|
|
const plannerPids = new Set<number>()
|
|
let releaseChild!: () => void, child: Promise<unknown> | undefined
|
|
const gate = new Promise<void>(yes => { releaseChild = yes })
|
|
const context = { principal, connection: await getConnection(principal.connectionId) }
|
|
const plannerClient: Pick<PoolClient, "query"> = { query: (async (text: string, values?: unknown[]) => {
|
|
if (!child) child = gate.then(() => host.target(raw, context))
|
|
plannerPids.add(Number((await originalQuery("SELECT pg_backend_pid() AS pid")).rows[0].pid))
|
|
return originalQuery(text, values)
|
|
}) as PoolClient["query"] }
|
|
try {
|
|
await originalQuery("BEGIN")
|
|
pool.query = (() => { throw Error("Pool query forbidden during actual planner lease") }) as typeof pool.query
|
|
const target = await host.target(raw, { ...context, transactionClient: plannerClient })
|
|
assert.deepEqual(target, { scope: "servers", serverIds: [serverId] }); assert.equal(plannerPids.size, 1)
|
|
assert(child)
|
|
const stale = assert.rejects(child, error => error instanceof McpError && error.code === "STATE_CONFLICT")
|
|
releaseChild(); await stale
|
|
await originalQuery("COMMIT")
|
|
} finally { pool.query = originalPoolQuery; releaseChild(); borrowed.release(true) }
|
|
|
|
const plan = await invokeMcpTool(principal, "mm_server_probe_plan", raw) as { planId: string; state: string }
|
|
hidden(plan); assert.equal(plan.state, "awaiting-approval"); assert.equal(requests, 0)
|
|
await assert.rejects(applyPlan(principal, plan.planId, randomUUID())); assert.equal(requests, 0)
|
|
await approvePlan(plan.planId, actor, true)
|
|
const key = randomUUID(), operation = await applyPlan(principal, plan.planId, key), terminal = await wait(operation.id)
|
|
assert.equal(terminal.state, "applied", "Owned host did not apply: " + String(terminal.error_code ?? "none")); hidden(terminal)
|
|
assert.equal(terminal.result.confirmedWrites, 0); assert.equal(terminal.result.verificationScope, "acknowledgement")
|
|
assert.equal(requests, 1); assert.equal(protocolErrors, 0)
|
|
const step = (await pool.query("SELECT state,result FROM mcp_operation_steps WHERE operation_id=$1 AND step_index=0", [operation.id])).rows[0]
|
|
assert.equal(step.state, "acknowledged"); hidden(step.result)
|
|
assert.equal(step.result.status, "completed"); assert.equal(step.result.attemptedStages, 1); assert.equal(step.result.completedStages, 1)
|
|
for (const field of ["output", "data", "contentB64", "bindingCiphertext"]) assert.equal(Object.hasOwn(step.result, field), false)
|
|
const receiptId = String(step.result.receiptId)
|
|
const stored = (await pool.query(`SELECT j.*,p.receipt_ciphertext,p.receipt_proof AS payload_proof,
|
|
to_jsonb(j) ? 'receipt_ciphertext' AS anchor_has_ciphertext,a.owner_id,a.claim_token,a.claim_attempted_at
|
|
FROM server_probe_executions j JOIN server_probe_execution_payloads p USING(receipt_id)
|
|
JOIN server_probe_storage_admissions a USING(receipt_id) WHERE j.receipt_id=$1`, [receiptId])).rows
|
|
assert.equal(stored.length, 1); const row = stored[0]
|
|
assert.equal(row.anchor_has_ciphertext, false); assert.equal(row.owner_id, actor.id); assert.equal(row.execution_id, operation.id)
|
|
assert.equal(row.state, "completed"); assert.equal(row.claim_token, null); assert(row.claim_attempted_at)
|
|
assert.equal(row.receipt_proof === row.payload_proof, true)
|
|
const envelope = JSON.parse(await decryptSecret(row.receipt_ciphertext)); hidden(envelope)
|
|
assert.equal(envelope.receipt.state, "completed"); assert.equal(envelope.receipt.forwardingVerified, false)
|
|
const usage = (await pool.query("SELECT admissions,reserved_files,reserved_bytes FROM server_probe_storage_usage WHERE scope=$1", ["owner:" + actor.id])).rows[0]
|
|
assert.equal(Number(usage.admissions), 1); assert.equal(Number(usage.reserved_files), 1); assert.equal(Number(usage.reserved_bytes), 294912)
|
|
assert.equal((await applyPlan(principal, plan.planId, key)).id, operation.id); assert.equal(requests, 1)
|
|
const readRequest = { operationId: operation.id, receiptId, page: 0 }
|
|
const firstPage = await invokeMcpTool(principal, "mm_server_probe_receipt_get", readRequest) as { contentB64: string; sha256: string }
|
|
hidden(firstPage); hidden(JSON.parse(Buffer.from(firstPage.contentB64, "base64").toString("utf8")))
|
|
|
|
// A SECOND real runner operation is cancelled only after native dispatch.
|
|
// The owned HTTP fixture withholds its response; the production watcher
|
|
// must observe durable cancellation and abort the actual request socket.
|
|
const cancelPlan = await invokeMcpTool(principal, "mm_server_probe_plan", raw) as { planId: string; state: string }
|
|
hidden(cancelPlan); assert.equal(cancelPlan.state, "awaiting-approval")
|
|
await approvePlan(cancelPlan.planId, actor, true)
|
|
const cancelKey = randomUUID(), cancelOperation = await applyPlan(principal, cancelPlan.planId, cancelKey)
|
|
await boundedBarrier(secondRequest, "Second native request did not reach the owned HTTP barrier")
|
|
assert.equal(requests, 2); assert.equal(protocolErrors, 0)
|
|
// Queue both actual cancellation requests before either can commit. This
|
|
// prevents the watcher/terminal transition racing the duplicate assertion,
|
|
// without replacing or delaying any production authority/watcher callback.
|
|
const blocker = await connectProbeStorage({ deadlineAt: Date.now() + 20000 })
|
|
let cancellationResults: PromiseSettledResult<Awaited<ReturnType<typeof host.cancel>>>[] | undefined
|
|
try {
|
|
await blocker.query("BEGIN")
|
|
await blocker.query("SELECT pg_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0))")
|
|
const cancellations = Promise.allSettled([cancel(cancelOperation.id), cancel(cancelOperation.id)])
|
|
let queued = false
|
|
for (const deadline = Date.now() + 12000; Date.now() < deadline;) {
|
|
const count = Number((await pool.query("SELECT count(*)::int AS n FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND wait_event='advisory' AND query LIKE '%probe-storage:lock%'")).rows[0].n)
|
|
if (count >= 2) { queued = true; break }
|
|
await delay(10)
|
|
}
|
|
assert.equal(queued, true, "Both real cancellation calls must queue before releasing the fixture barrier")
|
|
await blocker.query("COMMIT")
|
|
cancellationResults = await cancellations
|
|
} finally { blocker.release(true) }
|
|
assert(cancellationResults)
|
|
for (const result of cancellationResults) {
|
|
assert.equal(result.status, "fulfilled")
|
|
if (result.status !== "fulfilled") assert.fail("Owned cancellation was not acknowledged")
|
|
hidden(result.value); assert.equal(result.value.cancelRequested, true); assert.equal(result.value.terminal, false); assert.equal(result.value.automaticReplay, false)
|
|
}
|
|
const cancelled = await wait(cancelOperation.id); hidden(cancelled)
|
|
assert.equal(cancelled.state, "unknown"); assert.equal(cancelled.verification, "inconclusive")
|
|
assert.equal(cancelled.result.confirmedWrites, 0); assert.equal(cancelled.result.needsReview, true)
|
|
const progress = cancelled.result.diagnosticProgress
|
|
assert(progress); assert.equal(progress.operationId, cancelOperation.id); assert.equal(progress.state, "unknown")
|
|
assert.equal(progress.attemptedStages, 1); assert.equal(progress.completedStages, 0); assert.equal(progress.automaticReplay, false)
|
|
await boundedBarrier(secondSocketClosed, "Durable cancellation did not close the pending owned native socket")
|
|
const cancelledReceiptId = String(progress.receiptId)
|
|
const stopped = (await pool.query(`SELECT j.state,a.owner_id,a.claim_token,a.claim_attempted_at,a.cancelled_at,
|
|
(SELECT count(*)::int FROM server_probe_execution_quarantines q WHERE q.receipt_id=j.receipt_id AND q.execution_id=j.execution_id AND q.attempted_stages=1 AND q.completed_stages=0) AS quarantines,
|
|
(SELECT count(*)::int FROM server_probe_execution_payloads p WHERE p.receipt_id=j.receipt_id) AS payloads,
|
|
(SELECT count(*)::int FROM server_probe_execution_stages s WHERE s.receipt_id=j.receipt_id AND s.state='intent') AS pending_stages
|
|
FROM server_probe_executions j JOIN server_probe_storage_admissions a USING(receipt_id)
|
|
WHERE j.receipt_id=$1 AND j.execution_id=$2`, [cancelledReceiptId, cancelOperation.id])).rows
|
|
assert.equal(stopped.length, 1); assert.equal(stopped[0].state, "claimed"); assert.equal(stopped[0].owner_id, actor.id)
|
|
assert.equal(stopped[0].claim_token, null); assert(stopped[0].claim_attempted_at); assert(stopped[0].cancelled_at)
|
|
assert.equal(stopped[0].quarantines, 1); assert.equal(stopped[0].payloads, 0); assert.equal(stopped[0].pending_stages, 1)
|
|
const cancellationCount = async () => (await pool.query(`SELECT
|
|
(SELECT count(*)::int FROM mcp_diagnostic_cancellations WHERE operation_id=$1) AS records,
|
|
(SELECT count(*)::int FROM mcp_audit WHERE action='diagnostic.cancel-requested' AND actor_id=$2 AND payload->>'operationId'=$1::text) AS audits`, [cancelOperation.id, actor.id])).rows[0]
|
|
assert.deepEqual(await cancellationCount(), { records: 1, audits: 1 })
|
|
const afterCancellation = (await pool.query("SELECT admissions,reserved_files,reserved_bytes FROM server_probe_storage_usage WHERE scope=$1", ["owner:" + actor.id])).rows[0]
|
|
assert.equal(Number(afterCancellation.admissions), 2); assert.equal(Number(afterCancellation.reserved_files), 2); assert.equal(Number(afterCancellation.reserved_bytes), 2 * 294912)
|
|
await assert.rejects(invokeMcpTool(principal, "mm_server_probe_receipt_get", { operationId: cancelOperation.id, receiptId: cancelledReceiptId, page: 0 }))
|
|
assert.equal((await applyPlan(principal, cancelPlan.planId, cancelKey)).id, cancelOperation.id)
|
|
const terminalCancellation = await cancel(cancelOperation.id)
|
|
assert.equal(terminalCancellation.terminal, true); assert.equal(terminalCancellation.cancelRequested, false)
|
|
assert.deepEqual(await cancellationCount(), { records: 1, audits: 1 }); assert.equal(requests, 2)
|
|
|
|
const rotated = await rotateConnectionToken(principal.connectionId, actor); privateValues.push(rotated.token)
|
|
await pool.query("UPDATE mcp_connections SET owner_permissions=$2::jsonb,policy=jsonb_set(jsonb_set(jsonb_set(policy,'{permissions}',$2::jsonb),'{scope,operationIds}',$3::jsonb),'{mode}','\"observer\"'::jsonb) WHERE id=$1", [principal.connectionId, JSON.stringify(["mm:servers:read"]), JSON.stringify([SERVER_PROBE_RECEIPT_READ_OPERATION])])
|
|
const currentReader = await authenticateMcpToken(rotated.token)
|
|
assert.notEqual(currentReader.credentialId, principal.credentialId)
|
|
await assert.rejects(invokeMcpTool(principal, "mm_server_probe_receipt_get", readRequest))
|
|
const currentPage = await invokeMcpTool(currentReader, "mm_server_probe_receipt_get", readRequest) as { contentB64: string; sha256: string }
|
|
hidden(currentPage); hidden(JSON.parse(Buffer.from(currentPage.contentB64, "base64").toString("utf8")))
|
|
assert.equal(currentPage.sha256, firstPage.sha256)
|
|
await assert.rejects(invokeMcpTool(currentReader, "mm_server_probe_plan", raw)); assert.equal(requests, 2)
|
|
return { actualRuntimeInvokeApproveApply: true, realInheritedVerifiedGuard: true, ownedLoopbackNativeRequests: requests,
|
|
modelMetadataOnly: true, sanitizedPublicHistory: true, separateEncryptedPayload: true, quotaChargedOnce: true,
|
|
idempotentOperationReplayWithoutNative: true, currentReadCredentialOriginalEnvelope: true, readOnlyCannotExecute: true,
|
|
actualRunningCancellation: true, realWatcherAbortedNativeSocket: true, duplicateCancellationAuditOnce: true,
|
|
cancelledAttemptSpentWithoutPayload: true, quarantineBlocksHistory: true, chargedAdmissions: 2,
|
|
plannerTransactionBackends: plannerPids.size, stalePlannerChildRefused: true, productionAuthorityOverrides: false,
|
|
fullRunnerPoolMaxOneClaimed: false, productionDdlApplied: false, realRouterOrDns: false,
|
|
actualPublicRegistry: true, actualPublicRunReceiptAndCancellationTools: true, fixtureRegistrationOnly: false }
|
|
} finally {
|
|
releaseSecondResponse()
|
|
if (connectionIds.length) await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{status}','\"revoked\"'::jsonb) WHERE id=ANY($1::uuid[])", [connectionIds])
|
|
if (serverId !== undefined) await pool.query("DELETE FROM servers WHERE id=$1", [serverId])
|
|
await pool.query("UPDATE mcp_settings SET enabled=$1 WHERE id=1", [priorSettings.enabled])
|
|
router.closeAllConnections(); await new Promise<void>(yes => router.close(() => yes()))
|
|
}
|
|
}
|
|
|