211 lines
21 KiB
TypeScript
211 lines
21 KiB
TypeScript
import assert from "node:assert/strict"
|
|
import { createHash, randomUUID } from "node:crypto"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
import Fastify from "fastify"
|
|
import { mcpConnectionCreateSchema } from "@mmapp/contracts/mcp"
|
|
import { logFiltersSchema } from "@mmapp/contracts"
|
|
import { pool } from "../db/index.js"
|
|
import type { AuthUser } from "../lib/permissions.js"
|
|
import { authenticateMcpToken, createMcpConnection, effectiveMcpConnection } from "../modules/mcp/store.js"
|
|
import { invokeMcpTool } from "../modules/mcp/runtime.js"
|
|
import { findAction } from "../modules/mcp/actions.js"
|
|
import { encryptSecret, redact } from "../modules/mcp/security.js"
|
|
import { MAX_LOG_ARTIFACT_CHUNK_BYTES, createLogExportArtifact, MAX_LOG_ARTIFACT_CIPHER_BYTES_PER_CONNECTION } from "../modules/mcp/log-artifacts.js"
|
|
import { MAX_LOG_EXPORT_BYTES, readBoundedLogExportRows, LogExportLimitError } from "../services/router-logs-export.js"
|
|
import { PARTITION_WRITER_ADVISORY_KEY } from "../services/partition-write-transaction.js"
|
|
import { logWhere } from "../services/router-logs-model.js"
|
|
|
|
export async function verifyMcpLogArtifacts(actor: AuthUser) {
|
|
const ids = [93, 94], marker = "artifact-" + randomUUID()
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM servers WHERE id=ANY($1::int[])", [ids])).rows[0].n, 0)
|
|
const operationIds = ["logs.export", "logs.export.read"]
|
|
const issue = async (serverIds = ids, permissions = ["mm:servers:read"]) => {
|
|
const issued = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: marker, mode: "observer", permissions, expiresAt: new Date(Date.now() + 1800000).toISOString(), scope: { serverIds, operationIds, allowGlobal: false }, limits: { callsPerMinute: 120 } }), actor)
|
|
return { issued, principal: await authenticateMcpToken(issued.token) }
|
|
}
|
|
type Issued = Awaited<ReturnType<typeof issue>>
|
|
const exportFile = (source: Issued, serverId?: number, format = "json") => invokeMcpTool(source.principal, "mm_logs_export", { ...(serverId ? { serverId } : {}), q: marker, format }) as Promise<{ artifactId: string; bytes: number; sha256: string; rows: number }>
|
|
const read = (source: Issued, artifactId: string, options = {}, signal?: AbortSignal) => invokeMcpTool(source.principal, "mm_logs_export_read", { artifactId, ...options }, signal) as Promise<{ dataBase64: string; nextOffset: number | null; offset: number; complete: boolean; sha256: string; bytes: number }>
|
|
const blockedPayloadRace = async (source: Issued, file: { artifactId: string }, mutate: () => Promise<void>, expectedCode: string, signal?: AbortSignal) => {
|
|
const stored = (await pool.query("SELECT ciphertext FROM mcp_artifacts WHERE id=$1", [file.artifactId])).rows[0].ciphertext
|
|
const blocker = await pool.connect()
|
|
let pending: Promise<{ error?: unknown }> | undefined
|
|
try {
|
|
await blocker.query("BEGIN"); await blocker.query("SELECT id FROM mcp_artifacts WHERE id=$1 FOR UPDATE", [file.artifactId])
|
|
pending = read(source, file.artifactId, {}, signal).then(() => ({}), error => ({ error }))
|
|
let blocked = false
|
|
for (let n = 0; n < 300; n++) {
|
|
blocked = (await pool.query("SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query LIKE '%MCP log artifact payload%' AND query LIKE '%FOR SHARE%' LIMIT 1")).rowCount === 1
|
|
if (blocked) break
|
|
await delay(10)
|
|
}
|
|
assert.equal(blocked, true, "Must prove the actual scoped payload SELECT is blocked, not merely observe a request intent")
|
|
await mutate()
|
|
await blocker.query("UPDATE mcp_artifacts SET ciphertext='opaque corrupted late payload' WHERE id=$1", [file.artifactId])
|
|
await blocker.query("COMMIT")
|
|
const outcome = await pending
|
|
assert.ok(outcome.error && typeof outcome.error === "object" && "code" in outcome.error)
|
|
assert.equal(outcome.error.code, expectedCode, "Lost authority/cancellation must win before payload hashing or decryption")
|
|
} finally {
|
|
await blocker.query("ROLLBACK").catch(() => {}); blocker.release()
|
|
if (pending) await pending
|
|
await pool.query("UPDATE mcp_artifacts SET ciphertext=$2 WHERE id=$1", [file.artifactId, stored])
|
|
}
|
|
}
|
|
try {
|
|
await pool.query("INSERT INTO servers(id,name,host,enabled,username,password) SELECT n,$1,'127.0.0.1',false,'fixture','NEVER_ARTIFACT_LOGIN' FROM unnest($2::int[]) AS n", [marker, ids])
|
|
const message = marker + " password=NEVER_ARTIFACT_PASSWORD " + "😀".repeat(3500)
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES(93,$1,'192.0.2.93','visible','00:00:00',now(),now(),ARRAY['system'],'info',$2,0),
|
|
(94,$1,'192.0.2.94','hidden','00:00:00',now(),now(),ARRAY['system'],'info',$3,0)`, [marker, message, marker + " NEVER_HIDDEN_ARTIFACT"])
|
|
const narrow = await issue([93]), wide = await issue()
|
|
const context = { principal: narrow.principal, connection: await effectiveMcpConnection(narrow.principal) }
|
|
const expired = randomUUID()
|
|
await pool.query("INSERT INTO mcp_artifacts(id,connection_id,filename,media_type,ciphertext,expires_at) VALUES($1,$2,'expired.json','application/json',$3,now()-interval '1 second')",
|
|
[expired, context.connection.id, await encryptSecret("[]")])
|
|
const writerBlocker = await pool.connect()
|
|
try {
|
|
await writerBlocker.query("BEGIN")
|
|
await writerBlocker.query("SELECT pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
|
|
await assert.rejects(createLogExportArtifact("[]", "json", [93], context),
|
|
(error: { code?: string }) => error.code === "55P03" || error.code === "57014")
|
|
const { rows } = await writerBlocker.query("SELECT id FROM mcp_artifacts WHERE connection_id=$1", [context.connection.id])
|
|
assert.deepEqual(rows, [{ id: expired }])
|
|
} finally {
|
|
try { await writerBlocker.query("ROLLBACK") } finally { writerBlocker.release() }
|
|
}
|
|
const file = await exportFile(narrow)
|
|
assert.equal((await pool.query("SELECT id FROM mcp_artifacts WHERE id=$1", [expired])).rowCount, 0)
|
|
assert.equal(file.rows, 1); assert.ok(file.bytes > MAX_LOG_ARTIFACT_CHUNK_BYTES)
|
|
const chunks: Buffer[] = []; let offset = 0
|
|
do {
|
|
const part = await read(narrow, file.artifactId, offset === 0 ? { offset, limit: 1 } : { offset })
|
|
assert.equal(part.offset, offset); assert.equal(part.sha256, file.sha256); assert.equal(part.bytes, file.bytes)
|
|
assert.ok(part.dataBase64.length <= 16000); chunks.push(Buffer.from(part.dataBase64, "base64"))
|
|
if (part.nextOffset === null) { assert.equal(part.complete, true); break }
|
|
assert.equal(part.complete, false); assert.ok(part.nextOffset > offset); offset = part.nextOffset
|
|
} while (true)
|
|
const body = Buffer.concat(chunks)
|
|
assert.equal(body.length, file.bytes); assert.equal(createHash("sha256").update(body).digest("hex"), file.sha256)
|
|
assert.ok(body.toString("utf8").includes("😀")); assert.ok(!body.toString("utf8").includes("NEVER_")); assert.equal(JSON.parse(body.toString("utf8")).length, 1)
|
|
const rest = Fastify()
|
|
await rest.register((await import("../routes/logs.js")).default, { prefix: "/api" })
|
|
try {
|
|
const response = await rest.inject({ method: "GET", url: `/api/logs/export?serverId=93&q=${marker}&format=json` })
|
|
assert.equal(response.statusCode, 200); assert.deepEqual(JSON.parse(body.toString("utf8")), redact(response.json().map(({ host: _host, ...row }: Record<string, unknown>) => row))) // MCP artifacts never carry router hosts
|
|
const csv = await rest.inject({ method: "GET", url: `/api/logs/export?serverId=93&q=${marker}&format=csv` })
|
|
assert.equal(csv.statusCode, 200); assert.ok(csv.body.split("\r\n")[0].includes('"host"'))
|
|
} finally { await rest.close() }
|
|
assert.equal((await read(narrow, file.artifactId, { offset: file.bytes })).dataBase64, "")
|
|
await assert.rejects(read(narrow, file.artifactId, { offset: file.bytes + 1 }), (error: { code?: string }) => error.code === "INVALID_INPUT")
|
|
await assert.rejects(read(narrow, file.artifactId, { limit: MAX_LOG_ARTIFACT_CHUNK_BYTES + 1 }))
|
|
const denied = await issue([93], [])
|
|
await assert.rejects(read(denied, file.artifactId))
|
|
const mixed = await exportFile(wide), stored = (await pool.query("SELECT * FROM mcp_artifacts WHERE id=$1", [mixed.artifactId])).rows[0]
|
|
assert.equal(mixed.rows, 2)
|
|
const singleWide = await exportFile(wide, 93), originalQuery = pool.query.bind(pool)
|
|
let metadataPoolReads = 0
|
|
pool.query = ((...args: unknown[]) => {
|
|
const text = typeof args[0] === "string" ? args[0] : (args[0] as { text?: string } | undefined)?.text
|
|
if (text?.includes("MCP log artifact metadata")) metadataPoolReads++
|
|
return (originalQuery as (...values: unknown[]) => unknown)(...args)
|
|
}) as typeof pool.query
|
|
try { await read(wide, singleWide.artifactId) } finally { pool.query = originalQuery }
|
|
assert.ok(metadataPoolReads <= 1, "Target lookup uses the reservation client, not a second pool connection while holding the first")
|
|
assert.deepEqual((await pool.query("SELECT server_ids FROM mcp_requests WHERE connection_id=$1 AND tool='mm_logs_export_read' AND input->>'artifactId'=$2 ORDER BY created_at DESC LIMIT 1", [wide.issued.connection.id, singleWide.artifactId])).rows[0].server_ids.map(Number), [93])
|
|
await assert.rejects(read(narrow, mixed.artifactId), (error: { code?: string }) => error.code === "NOT_FOUND")
|
|
const grant = randomUUID()
|
|
await pool.query("INSERT INTO mcp_grants(id,connection_id,payload) VALUES($1,$2,$3::jsonb)", [grant, wide.issued.connection.id, JSON.stringify({ id: grant, credentialId: wide.principal.credentialId, startsAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60000).toISOString(), scope: { serverIds: [93], operationIds, allowGlobal: false } })])
|
|
await pool.query("UPDATE mcp_artifacts SET ciphertext='corrupt opaque payload' WHERE id=$1", [mixed.artifactId])
|
|
await assert.rejects(read(wide, mixed.artifactId), (error: { code?: string }) => error.code === "GRANT_SCOPE_DENIED")
|
|
await pool.query("DELETE FROM mcp_grants WHERE id=$1", [grant])
|
|
await pool.query("UPDATE mcp_artifacts SET ciphertext=$2,access_scope=jsonb_set(access_scope,'{serverIds}','[93]') WHERE id=$1", [mixed.artifactId, stored.ciphertext])
|
|
await assert.rejects(read(wide, mixed.artifactId), (error: { code?: string }) => error.code === "ARTIFACT_INVALID")
|
|
await pool.query("UPDATE mcp_artifacts SET access_scope=$2::jsonb,ciphertext='corrupt opaque payload' WHERE id=$1", [mixed.artifactId, JSON.stringify(stored.access_scope)])
|
|
await assert.rejects(read(wide, mixed.artifactId), (error: { code?: string }) => error.code === "ARTIFACT_INVALID")
|
|
await pool.query("UPDATE mcp_artifacts SET ciphertext=$2 WHERE id=$1", [mixed.artifactId, stored.ciphertext])
|
|
const legacy = randomUUID()
|
|
await pool.query("INSERT INTO mcp_artifacts(id,connection_id,filename,media_type,ciphertext,expires_at) VALUES($1,$2,'legacy.json','application/json',$3,now()+interval '15 minutes')", [legacy, narrow.issued.connection.id, await encryptSecret("NEVER_LEGACY_PAYLOAD")])
|
|
await assert.rejects(read(narrow, legacy), (error: { code?: string }) => error.code === "ARTIFACT_SCOPE_UNKNOWN")
|
|
await pool.query("UPDATE mcp_artifacts SET expires_at=now()-interval '1 second' WHERE id=$1", [legacy])
|
|
await assert.rejects(read(narrow, legacy), (error: { code?: string }) => error.code === "NOT_FOUND")
|
|
await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{version}',to_jsonb((policy->>'version')::int+1)) WHERE id=$1", [wide.issued.connection.id])
|
|
await assert.rejects(read(wide, mixed.artifactId), (error: { code?: string }) => error.code === "RESET_REQUIRED")
|
|
const late = await issue(), lateFile = await exportFile(late, 93), lateGrant = randomUUID()
|
|
await pool.query("INSERT INTO mcp_grants(id,connection_id,payload) VALUES($1,$2,$3::jsonb)", [lateGrant, late.issued.connection.id, JSON.stringify({ id: lateGrant, credentialId: late.principal.credentialId, startsAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60000).toISOString(), scope: { serverIds: ids, operationIds, allowGlobal: false } })])
|
|
await blockedPayloadRace(late, lateFile, async () => { await pool.query("UPDATE mcp_grants SET payload=jsonb_set(payload,'{scope,serverIds}','[94]') WHERE id=$1", [lateGrant]) }, "RESET_REQUIRED")
|
|
await pool.query("DELETE FROM mcp_grants WHERE id=$1", [lateGrant])
|
|
const revoked = await issue([93]), revokedFile = await exportFile(revoked)
|
|
await blockedPayloadRace(revoked, revokedFile, async () => { await pool.query("UPDATE mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [revoked.issued.connection.id]) }, "PERMISSION_DENIED")
|
|
const paused = await issue([93]), pausedFile = await exportFile(paused)
|
|
await blockedPayloadRace(paused, pausedFile, async () => { await pool.query("UPDATE mcp_connections SET policy=jsonb_set(policy,'{status}','\"paused\"') WHERE id=$1", [paused.issued.connection.id]) }, "UNAUTHORIZED")
|
|
const cancelled = await issue([93]), cancelledFile = await exportFile(cancelled), abort = new AbortController()
|
|
await blockedPayloadRace(cancelled, cancelledFile, async () => { abort.abort() }, "READ_CANCELLED", abort.signal)
|
|
const budgetRevoked = await issue([93]), blocker = await pool.connect()
|
|
let blockedExport: Promise<{ error?: unknown }> | undefined
|
|
try {
|
|
await blocker.query("BEGIN"); await blocker.query("LOCK TABLE router_log_entries IN ACCESS EXCLUSIVE MODE")
|
|
blockedExport = exportFile(budgetRevoked).then(() => ({}), error => ({ error }))
|
|
let waiting = false
|
|
for (let n = 0; n < 300; n++) {
|
|
waiting = (await pool.query("SELECT 1 FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock' AND query LIKE 'WITH chosen AS MATERIALIZED%' LIMIT 1")).rowCount === 1
|
|
if (waiting) break
|
|
await delay(10)
|
|
}
|
|
assert.equal(waiting, true)
|
|
await pool.query("UPDATE mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [budgetRevoked.issued.connection.id])
|
|
await blocker.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES(93,$1,'192.0.2.93','late-budget','00:00:00',now(),now(),ARRAY['system'],'info',$1||repeat('x',$2::int),0)`, [marker, MAX_LOG_EXPORT_BYTES + 1])
|
|
await blocker.query("COMMIT")
|
|
const outcome = await blockedExport
|
|
assert.ok(outcome.error && typeof outcome.error === "object" && "code" in outcome.error)
|
|
assert.equal(outcome.error.code, "PERMISSION_DENIED", "Loss of rights wins even when the source reports a byte limit")
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_artifacts WHERE connection_id=$1", [budgetRevoked.issued.connection.id])).rows[0].n, 0)
|
|
} finally { await blocker.query("ROLLBACK").catch(() => {}); blocker.release(); if (blockedExport) await blockedExport; await pool.query("DELETE FROM router_log_entries WHERE server_id=93 AND server_name=$1 AND router_id='late-budget'", [marker]) }
|
|
const quota = await issue([93]), quotaContext = { principal: quota.principal, connection: await effectiveMcpConnection(quota.principal) }
|
|
for (let n = 0; n < 7; n++) await createLogExportArtifact("[]", "json", [93], quotaContext)
|
|
const concurrentQuota = await Promise.allSettled([createLogExportArtifact("[]", "json", [93], quotaContext), createLogExportArtifact("[]", "json", [93], quotaContext)])
|
|
assert.equal(concurrentQuota.filter(result => result.status === "fulfilled").length, 1)
|
|
assert.ok(concurrentQuota.some(result => result.status === "rejected" && result.reason.code === "EXPORT_QUOTA_EXCEEDED"))
|
|
assert.equal((await pool.query("SELECT count(*)::int AS n FROM mcp_artifacts WHERE connection_id=$1", [quota.issued.connection.id])).rows[0].n, 8)
|
|
const byteQuota = await issue([93]), byteContext = { principal: byteQuota.principal, connection: await effectiveMcpConnection(byteQuota.principal) }, large = " ".repeat(MAX_LOG_EXPORT_BYTES - 2) + "[]"
|
|
await createLogExportArtifact(large, "json", [93], byteContext); await createLogExportArtifact(large, "json", [93], byteContext)
|
|
await assert.rejects(createLogExportArtifact(large, "json", [93], byteContext), (error: { code?: string }) => error.code === "EXPORT_QUOTA_EXCEEDED")
|
|
const byteUsage = (await pool.query("SELECT count(*)::int AS n,sum(octet_length(ciphertext))::text AS bytes FROM mcp_artifacts WHERE connection_id=$1", [byteQuota.issued.connection.id])).rows[0]
|
|
assert.equal(byteUsage.n, 2); assert.ok(BigInt(byteUsage.bytes) <= BigInt(MAX_LOG_ARTIFACT_CIPHER_BYTES_PER_CONNECTION))
|
|
const capabilityAccess = await createMcpConnection(mcpConnectionCreateSchema.parse({ name: marker, mode: "observer", permissions: ["mm:servers:read"], expiresAt: new Date(Date.now() + 1800000).toISOString(), scope: { serverIds: ids, operationIds: ["capabilities.read"], allowGlobal: false } }), actor)
|
|
const capabilityPrincipal = await authenticateMcpToken(capabilityAccess.token), capabilityGrant = randomUUID()
|
|
await pool.query("INSERT INTO mcp_grants(id,connection_id,payload) VALUES($1,$2,$3::jsonb)", [capabilityGrant, capabilityAccess.connection.id, JSON.stringify({ id: capabilityGrant, credentialId: capabilityPrincipal.credentialId, startsAt: new Date(Date.now() - 1000).toISOString(), expiresAt: new Date(Date.now() + 60000).toISOString(), scope: { serverIds: ids, operationIds: ["capabilities.read"], allowGlobal: false } })])
|
|
const heldRead = async (mutate: () => Promise<void>) => {
|
|
const action = findAction("mm_capabilities_get"), original = action.read!
|
|
let release!: () => void, entered!: () => void
|
|
const gate = new Promise<void>(resolve => { release = resolve }), started = new Promise<void>(resolve => { entered = resolve })
|
|
action.read = async (input, context) => { assert.equal(context.transactionClient, undefined, "A released reservation client must never reach the read handler"); entered(); await gate; return original(input, context) }
|
|
const pending = invokeMcpTool(capabilityPrincipal, "mm_capabilities_get", {}).then(() => ({} as { error?: unknown }), error => ({ error }))
|
|
try {
|
|
await Promise.race([started, pending.then(() => { throw Error("Read did not reach its source") })])
|
|
await mutate(); release()
|
|
const outcome = await pending
|
|
assert.ok(outcome.error && typeof outcome.error === "object" && "code" in outcome.error); assert.equal(outcome.error.code, "STATE_CONFLICT")
|
|
} finally { release(); action.read = original; await pending }
|
|
}
|
|
await heldRead(async () => { await pool.query("UPDATE mcp_grants SET payload=jsonb_set(payload,'{scope,serverIds}','[93]') WHERE id=$1", [capabilityGrant]) })
|
|
await pool.query("DELETE FROM mcp_grants WHERE id=$1", [capabilityGrant])
|
|
await heldRead(async () => { await pool.query("UPDATE mcp_connections SET owner_permissions='[]'::jsonb WHERE id=$1", [capabilityAccess.connection.id]) })
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES(94,$1,'192.0.2.94','oversize-bytes','00:00:00',now(),now(),ARRAY['system'],'info',$1||repeat('x',$2::int),0)`, [marker, MAX_LOG_EXPORT_BYTES + 1])
|
|
await assert.rejects(readBoundedLogExportRows(logWhere(logFiltersSchema.parse({ serverId: 94, q: marker }))), (error: unknown) => error instanceof LogExportLimitError && error.kind === "bytes")
|
|
assert.equal((await exportFile(narrow)).rows, 1, "Hidden oversized rows are excluded before the source budget")
|
|
await pool.query("DELETE FROM router_log_entries WHERE server_id=94 AND server_name=$1 AND router_id='oversize-bytes'", [marker])
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
SELECT 93,$1,'192.0.2.93','row-cap','00:00:00',now(),now(),ARRAY['system'],'info',$1,0 FROM generate_series(1,50001)`, [marker])
|
|
await assert.rejects(readBoundedLogExportRows(logWhere(logFiltersSchema.parse({ serverId: 93, q: marker }))), (error: unknown) => error instanceof LogExportLimitError && error.kind === "rows")
|
|
assert.ok(!JSON.stringify((await pool.query("SELECT result FROM mcp_requests WHERE connection_id=ANY($1::uuid[])", [[narrow.issued.connection.id, wide.issued.connection.id]])).rows).includes("NEVER_"))
|
|
console.log("✓ Log artifacts: scoped REST/MCP source and byte/row budgets, Unicode/chunk/hash, signed scope and tamper/expiry/legacy guards, real pre-decrypt grant/owner/pause/abort and source-budget races, atomic count/byte quotas, reservation client reuse and generic late-effective-access fence")
|
|
} finally {
|
|
await pool.query("DELETE FROM mcp_artifacts WHERE connection_id IN (SELECT id FROM mcp_connections WHERE name=$1)", [marker])
|
|
await pool.query("DELETE FROM router_log_entries WHERE server_id=ANY($1::int[]) AND server_name=$2", [ids, marker])
|
|
await pool.query("DELETE FROM servers WHERE id=ANY($1::int[]) AND name=$2", [ids, marker])
|
|
}
|
|
}
|