import { isDeepStrictEqual } from "node:util" import { withConfigMutationLock } from "../../services/config-mutation-lock.js" import { acknowledgedCreateId } from "./native-create-identity.js" import { readRosSnapshot } from "./resources.js" import { proveRosDriftExpectation } from "./ros-step-verification.js" import { randomUUID } from "node:crypto" import { retainServerProbeStorage } from "../../services/server-probe-retention.js" import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js" import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js" import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js" import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js" import { pool } from "../../db/index.js" import { sharedDatabaseGate } from "../../services/shared-database-gate.js" import { recoverMcpOperations } from "./runtime.js" import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js" import { audit, getMcpSettings } from "./store.js" import { accessGeneration, decryptSecretExisting, McpError, redact } from "./security.js" import { withMcpDatabaseRequest } from "./database-request-lease.js" import { tickRoutingProfiles } from "./routing-profiles.js" import type { ChangeEvidence } from "./action-types.js" import { PartitionWriterBusy, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js" import { retainManualInverseChargesWithinOwner } from "./routing-manual-inverse-store.js" import { recoverScopedRecoveryIntents } from "./routing-community-scoped-recovery.js" type DriftCheckpoint = { before_sealed: string; after_sealed: string; result: unknown; endpoint_epoch: string } const driftCheckpointSql = `SELECT k.before_sealed,k.after_sealed,s.result,device.routing_endpoint_epoch AS endpoint_epoch FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id JOIN mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=$2 JOIN mcp_operation_checkpoints k ON k.operation_id=s.operation_id AND k.step_index=s.step_index JOIN servers device ON device.id=s.server_id WHERE o.id=$1 AND c.id=$7 AND p.connection_id=c.id AND o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days' AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$3 AND p.evidence->>'sealed'=$4 AND s.server_id=$5 AND s.intended_state=$6::jsonb AND device.enabled AND s.state='verified' AND s.result IS NOT NULL AND k.after_sealed IS NOT NULL AND NOT EXISTS(SELECT 1 FROM mcp_operations newer JOIN mcp_operation_steps ns ON ns.operation_id=newer.id WHERE newer.created_at>o.created_at AND ns.server_id=s.server_id)` export async function detectMcpDrift() { const generation = await accessGeneration() const { rows } = await pool.query<{ id: string; connection_id: string; evidence: { sealed: string } }>(`SELECT o.id,o.connection_id,p.evidence FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id WHERE o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days' AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$1 ORDER BY o.finished_at DESC LIMIT 50`, [generation]) const seen = new Set() for (const row of rows) { let steps: ChangeEvidence['steps'] try { const evidence = JSON.parse(await decryptSecretExisting(row.evidence.sealed)) as ChangeEvidence if (!Array.isArray(evidence.steps)) continue steps = evidence.steps } catch { continue } const uncertainResources = new Set() for (let index = steps.length - 1; index >= 0; index--) { const step = steps[index] if (step?.kind !== "ros" || !step.step) continue const { serverId, resource, action, id } = step.step const resourceKey = `${serverId}:${resource}` if (uncertainResources.has(resourceKey)) continue try { const result = action === "create" ? (await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1 AND step_index=$2 AND server_id=$3", [row.id, index, serverId])).rows[0]?.result : undefined const nativeId = action === "create" ? acknowledgedCreateId(result) : id if (action === "create" && !nativeId) { // A later create without a durable native identity can shadow an older target. uncertainResources.add(resourceKey) continue } const targetKey = JSON.stringify([serverId, resource, nativeId ?? null]) if (seen.has(targetKey)) continue seen.add(targetKey) await withConfigMutationLock([serverId], async () => { const expected = JSON.stringify(redact(step)) const args = [row.id, index, generation, row.evidence.sealed, serverId, expected, row.connection_id] const { rows: [checkpoint] } = await pool.query(driftCheckpointSql, args) if (!checkpoint || await accessGeneration() !== generation) return const before = JSON.parse(await decryptSecretExisting(checkpoint.before_sealed)) const after = JSON.parse(await decryptSecretExisting(checkpoint.after_sealed)) const expectation = proveRosDriftExpectation(step.step, checkpoint.result, before, after) if (!expectation || expectation.nativeId !== nativeId) return const current = await readRosSnapshot(serverId, resource, undefined, expectation.sourceProof) if (expectation.matches(current) !== false) return const observedAt = new Date().toISOString() await withBackgroundPartitionWrite(async ({ client }) => { const { rows: [locked] } = await client.query( `${driftCheckpointSql} FOR SHARE OF o,p,c,s,k,device`, args) if (!isDeepStrictEqual(locked, checkpoint) || await accessGeneration() !== generation) return const inserted = await client.query(`INSERT INTO mcp_drift(id,operation_id,server_id,expected,observed) VALUES($1,$2,$3,$4::jsonb,$5::jsonb) ON CONFLICT DO NOTHING RETURNING id`, [randomUUID(), row.id, serverId, JSON.stringify(redact({ stepIndex: index, step })), JSON.stringify({ matchesExpected: false, observedAt })]) if (inserted.rows.length) { await audit(client, "mcp-drift", "drift.detected", row.connection_id, { operationId: row.id, stepIndex: index, serverId }) await client.query("UPDATE mcp_routing_profiles SET enabled=false,state=state || $2::jsonb WHERE connection_id=$1 AND enabled", [row.connection_id, JSON.stringify({ pausedReason: "Обнаружен дрейф конфигурации" })]) } }) }) } catch { /* Unavailable proof or persistence leaves the next candidate eligible. */ } } } } const mcpRetentionBatchSize = 1000 async function retainMcpData() { await withBackgroundPartitionWrite(async ({ client }) => { const { requestsRetentionDays } = await getMcpSettings(client) await client.query(`WITH expired AS ( SELECT id FROM mcp_requests WHERE created_at$1) AND status NOT IN ('running') ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED ) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [requestsRetentionDays, mcpRetentionBatchSize]) }) await withBackgroundPartitionWrite(async ({ client }) => { const { auditRetentionDays } = await getMcpSettings(client) await client.query(`WITH expired AS ( SELECT id FROM mcp_audit WHERE created_at$1) ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED ) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [auditRetentionDays, mcpRetentionBatchSize]) }) await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS ( SELECT id FROM mcp_artifacts WHERE expires_at client.query("DELETE FROM mcp_oauth_requests WHERE expires_at client.query("DELETE FROM mcp_budget_buckets WHERE bucket retainManualInverseChargesWithinOwner(client)) await withBackgroundPartitionWrite(async ({ client }) => { const { requestsRetentionDays } = await getMcpSettings(client) await client.query(`WITH candidates AS MATERIALIZED (SELECT source_operation_id FROM mcp_routing_forward_charges WHERE created_at$1) ORDER BY created_at,source_operation_id LIMIT 1000), expired AS (SELECT c.source_operation_id FROM candidates c WHERE NOT EXISTS(SELECT 1 FROM mcp_operations o WHERE o.id=c.source_operation_id AND o.state IN('pending','running'))) DELETE FROM mcp_routing_forward_charges c USING expired e WHERE c.source_operation_id=e.source_operation_id`, [requestsRetentionDays]) }) await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_routing_observations WHERE created_at client.query("DELETE FROM mcp_path_checks WHERE created_at client.query("DELETE FROM mcp_unauthorized_buckets WHERE bucket client.query("DELETE FROM mcp_portal_events WHERE created_at withMcpDatabaseRequest(async () => { entered = true await tickAdmittedMcpWorkers() })) } catch (error) { if (entered || !(error instanceof McpError && ["PROTOCOL_STORAGE_BUSY", "RATE_LIMITED"].includes(error.code))) throw error } } async function tickAdmittedMcpWorkers() { const lease = await pool.connect() let locked = false, unsafe = false try { locked = (await lease.query("SELECT pg_try_advisory_lock(hashtextextended('mcp-workers',0)) AS locked")).rows[0].locked if (!locked) return await recoverMcpOperations() await recoverScopedRecoveryIntents() await retainMcpData() await retainServerProbeStorage() await compactCompletedServerPollStorage() await retireCompletedServerPollOperations() await collectExpiredServerPollStorage() await retireOrdinaryMcpOperations() if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() } await maintainServerProbeOperationRetirement() } catch (error) { if (!(error instanceof PartitionWriterBusy)) throw error } finally { if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true } lease.release(unsafe) } } export function startMcpWorkers(onError: (error: unknown) => void) { let stopped = false, running: Promise | undefined const tick = () => { if (stopped || running) return running = tickMcpWorkers().catch(onError).finally(() => { running = undefined }) } const timer = setInterval(tick, 30_000); timer.unref(); tick() return async () => { stopped = true; clearInterval(timer); await running } }