191 lines
12 KiB
TypeScript
191 lines
12 KiB
TypeScript
import { isDeepStrictEqual } from "node:util"
|
|
import { withConfigMutationLock } from "../../services/config-mutation-lock.js"
|
|
import { acknowledgedCreateId } from "./native-create-identity.js"
|
|
import { readRosSnapshot } from "./resources.js"
|
|
import { proveRosDriftExpectation } from "./ros-step-verification.js"
|
|
import { randomUUID } from "node:crypto"
|
|
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
|
|
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
|
|
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
|
|
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
|
|
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
|
|
import { pool } from "../../db/index.js"
|
|
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
|
|
import { recoverMcpOperations } from "./runtime.js"
|
|
import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js"
|
|
import { audit, getMcpSettings } from "./store.js"
|
|
import { accessGeneration, decryptSecretExisting, McpError, redact } from "./security.js"
|
|
import { withMcpDatabaseRequest } from "./database-request-lease.js"
|
|
import { tickRoutingProfiles } from "./routing-profiles.js"
|
|
import type { ChangeEvidence } from "./action-types.js"
|
|
import { PartitionWriterBusy, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js"
|
|
import { retainManualInverseChargesWithinOwner } from "./routing-manual-inverse-store.js"
|
|
import { recoverScopedRecoveryIntents } from "./routing-community-scoped-recovery.js"
|
|
|
|
type DriftCheckpoint = { before_sealed: string; after_sealed: string; result: unknown; endpoint_epoch: string }
|
|
|
|
const driftCheckpointSql = `SELECT k.before_sealed,k.after_sealed,s.result,device.routing_endpoint_epoch AS endpoint_epoch
|
|
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id
|
|
JOIN mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=$2
|
|
JOIN mcp_operation_checkpoints k ON k.operation_id=s.operation_id AND k.step_index=s.step_index
|
|
JOIN servers device ON device.id=s.server_id
|
|
WHERE o.id=$1 AND c.id=$7 AND p.connection_id=c.id
|
|
AND o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days'
|
|
AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$3
|
|
AND p.evidence->>'sealed'=$4 AND s.server_id=$5 AND s.intended_state=$6::jsonb AND device.enabled
|
|
AND s.state='verified' AND s.result IS NOT NULL AND k.after_sealed IS NOT NULL
|
|
AND NOT EXISTS(SELECT 1 FROM mcp_operations newer JOIN mcp_operation_steps ns ON ns.operation_id=newer.id
|
|
WHERE newer.created_at>o.created_at AND ns.server_id=s.server_id)`
|
|
|
|
export async function detectMcpDrift() {
|
|
const generation = await accessGeneration()
|
|
const { rows } = await pool.query<{ id: string; connection_id: string; evidence: { sealed: string } }>(`SELECT o.id,o.connection_id,p.evidence
|
|
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id
|
|
WHERE o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days'
|
|
AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$1
|
|
ORDER BY o.finished_at DESC LIMIT 50`, [generation])
|
|
const seen = new Set<string>()
|
|
for (const row of rows) {
|
|
let steps: ChangeEvidence['steps']
|
|
try {
|
|
const evidence = JSON.parse(await decryptSecretExisting(row.evidence.sealed)) as ChangeEvidence
|
|
if (!Array.isArray(evidence.steps)) continue
|
|
steps = evidence.steps
|
|
} catch { continue }
|
|
|
|
const uncertainResources = new Set<string>()
|
|
for (let index = steps.length - 1; index >= 0; index--) {
|
|
const step = steps[index]
|
|
if (step?.kind !== "ros" || !step.step) continue
|
|
const { serverId, resource, action, id } = step.step
|
|
const resourceKey = `${serverId}:${resource}`
|
|
if (uncertainResources.has(resourceKey)) continue
|
|
try {
|
|
const result = action === "create"
|
|
? (await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1 AND step_index=$2 AND server_id=$3", [row.id, index, serverId])).rows[0]?.result
|
|
: undefined
|
|
const nativeId = action === "create" ? acknowledgedCreateId(result) : id
|
|
if (action === "create" && !nativeId) {
|
|
// A later create without a durable native identity can shadow an older target.
|
|
uncertainResources.add(resourceKey)
|
|
continue
|
|
}
|
|
const targetKey = JSON.stringify([serverId, resource, nativeId ?? null])
|
|
if (seen.has(targetKey)) continue
|
|
seen.add(targetKey)
|
|
|
|
await withConfigMutationLock([serverId], async () => {
|
|
const expected = JSON.stringify(redact(step))
|
|
const args = [row.id, index, generation, row.evidence.sealed, serverId, expected, row.connection_id]
|
|
const { rows: [checkpoint] } = await pool.query<DriftCheckpoint>(driftCheckpointSql, args)
|
|
if (!checkpoint || await accessGeneration() !== generation) return
|
|
const before = JSON.parse(await decryptSecretExisting(checkpoint.before_sealed))
|
|
const after = JSON.parse(await decryptSecretExisting(checkpoint.after_sealed))
|
|
const expectation = proveRosDriftExpectation(step.step, checkpoint.result, before, after)
|
|
if (!expectation || expectation.nativeId !== nativeId) return
|
|
const current = await readRosSnapshot(serverId, resource, undefined, expectation.sourceProof)
|
|
if (expectation.matches(current) !== false) return
|
|
const observedAt = new Date().toISOString()
|
|
|
|
await withBackgroundPartitionWrite(async ({ client }) => {
|
|
const { rows: [locked] } = await client.query<DriftCheckpoint>(
|
|
`${driftCheckpointSql} FOR SHARE OF o,p,c,s,k,device`, args)
|
|
if (!isDeepStrictEqual(locked, checkpoint) || await accessGeneration() !== generation) return
|
|
const inserted = await client.query(`INSERT INTO mcp_drift(id,operation_id,server_id,expected,observed)
|
|
VALUES($1,$2,$3,$4::jsonb,$5::jsonb) ON CONFLICT DO NOTHING RETURNING id`, [randomUUID(), row.id, serverId,
|
|
JSON.stringify(redact({ stepIndex: index, step })), JSON.stringify({ matchesExpected: false, observedAt })])
|
|
if (inserted.rows.length) {
|
|
await audit(client, "mcp-drift", "drift.detected", row.connection_id, { operationId: row.id, stepIndex: index, serverId })
|
|
await client.query("UPDATE mcp_routing_profiles SET enabled=false,state=state || $2::jsonb WHERE connection_id=$1 AND enabled", [row.connection_id, JSON.stringify({ pausedReason: "Обнаружен дрейф конфигурации" })])
|
|
}
|
|
})
|
|
})
|
|
} catch { /* Unavailable proof or persistence leaves the next candidate eligible. */ }
|
|
}
|
|
}
|
|
}
|
|
|
|
const mcpRetentionBatchSize = 1000
|
|
|
|
async function retainMcpData() {
|
|
await withBackgroundPartitionWrite(async ({ client }) => {
|
|
const { requestsRetentionDays } = await getMcpSettings(client)
|
|
await client.query(`WITH expired AS (
|
|
SELECT id FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')
|
|
ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED
|
|
) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [requestsRetentionDays, mcpRetentionBatchSize])
|
|
})
|
|
await withBackgroundPartitionWrite(async ({ client }) => {
|
|
const { auditRetentionDays } = await getMcpSettings(client)
|
|
await client.query(`WITH expired AS (
|
|
SELECT id FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)
|
|
ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED
|
|
) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [auditRetentionDays, mcpRetentionBatchSize])
|
|
})
|
|
await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS (
|
|
SELECT id FROM mcp_artifacts WHERE expires_at<now()
|
|
ORDER BY expires_at,id LIMIT $1 FOR UPDATE
|
|
) DELETE FROM mcp_artifacts USING expired WHERE mcp_artifacts.id=expired.id`, [mcpRetentionBatchSize]))
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_oauth_requests WHERE expires_at<now()-interval '1 day'"))
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_budget_buckets WHERE bucket<now()-interval '2 days'"))
|
|
await withBackgroundPartitionWrite(({ client }) => retainManualInverseChargesWithinOwner(client))
|
|
await withBackgroundPartitionWrite(async ({ client }) => {
|
|
const { requestsRetentionDays } = await getMcpSettings(client)
|
|
await client.query(`WITH candidates AS MATERIALIZED (SELECT source_operation_id FROM mcp_routing_forward_charges
|
|
WHERE created_at<now()-make_interval(days=>$1) ORDER BY created_at,source_operation_id LIMIT 1000),
|
|
expired AS (SELECT c.source_operation_id FROM candidates c WHERE NOT EXISTS(SELECT 1 FROM mcp_operations o
|
|
WHERE o.id=c.source_operation_id AND o.state IN('pending','running')))
|
|
DELETE FROM mcp_routing_forward_charges c USING expired e WHERE c.source_operation_id=e.source_operation_id`, [requestsRetentionDays])
|
|
})
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_routing_observations WHERE created_at<now()-interval '7 days'"))
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_path_checks WHERE created_at<now()-interval '7 days'"))
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_unauthorized_buckets WHERE bucket<now()-interval '7 days'"))
|
|
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_portal_events WHERE created_at<now()-interval '7 days'"))
|
|
}
|
|
|
|
export async function tickMcpWorkers() {
|
|
let entered = false
|
|
try {
|
|
await sharedDatabaseGate.request(() => withMcpDatabaseRequest(async () => {
|
|
entered = true
|
|
await tickAdmittedMcpWorkers()
|
|
}))
|
|
} catch (error) {
|
|
if (entered || !(error instanceof McpError && ["PROTOCOL_STORAGE_BUSY", "RATE_LIMITED"].includes(error.code))) throw error
|
|
}
|
|
}
|
|
|
|
async function tickAdmittedMcpWorkers() {
|
|
const lease = await pool.connect()
|
|
let locked = false, unsafe = false
|
|
try {
|
|
locked = (await lease.query("SELECT pg_try_advisory_lock(hashtextextended('mcp-workers',0)) AS locked")).rows[0].locked
|
|
if (!locked) return
|
|
await recoverMcpOperations()
|
|
await recoverScopedRecoveryIntents()
|
|
await retainMcpData()
|
|
await retainServerProbeStorage()
|
|
await compactCompletedServerPollStorage()
|
|
await retireCompletedServerPollOperations()
|
|
await collectExpiredServerPollStorage()
|
|
await retireOrdinaryMcpOperations()
|
|
if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() }
|
|
await maintainServerProbeOperationRetirement()
|
|
} catch (error) {
|
|
if (!(error instanceof PartitionWriterBusy)) throw error
|
|
} finally {
|
|
if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true }
|
|
lease.release(unsafe)
|
|
}
|
|
}
|
|
|
|
export function startMcpWorkers(onError: (error: unknown) => void) {
|
|
let stopped = false, running: Promise<void> | undefined
|
|
const tick = () => {
|
|
if (stopped || running) return
|
|
running = tickMcpWorkers().catch(onError).finally(() => { running = undefined })
|
|
}
|
|
const timer = setInterval(tick, 30_000); timer.unref(); tick()
|
|
return async () => { stopped = true; clearInterval(timer); await running }
|
|
}
|