Files
MikrotikManager/backend/src/modules/mcp/workers.ts
T

191 lines
12 KiB
TypeScript

import { isDeepStrictEqual } from "node:util"
import { withConfigMutationLock } from "../../services/config-mutation-lock.js"
import { acknowledgedCreateId } from "./native-create-identity.js"
import { readRosSnapshot } from "./resources.js"
import { proveRosDriftExpectation } from "./ros-step-verification.js"
import { randomUUID } from "node:crypto"
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
import { pool } from "../../db/index.js"
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
import { recoverMcpOperations } from "./runtime.js"
import { retireOrdinaryMcpOperations } from "./ordinary-operation-retirement.js"
import { audit, getMcpSettings } from "./store.js"
import { accessGeneration, decryptSecretExisting, McpError, redact } from "./security.js"
import { withMcpDatabaseRequest } from "./database-request-lease.js"
import { tickRoutingProfiles } from "./routing-profiles.js"
import type { ChangeEvidence } from "./action-types.js"
import { PartitionWriterBusy, withBackgroundPartitionWrite } from "../../services/partition-write-transaction.js"
import { retainManualInverseChargesWithinOwner } from "./routing-manual-inverse-store.js"
import { recoverScopedRecoveryIntents } from "./routing-community-scoped-recovery.js"
type DriftCheckpoint = { before_sealed: string; after_sealed: string; result: unknown; endpoint_epoch: string }
const driftCheckpointSql = `SELECT k.before_sealed,k.after_sealed,s.result,device.routing_endpoint_epoch AS endpoint_epoch
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id
JOIN mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=$2
JOIN mcp_operation_checkpoints k ON k.operation_id=s.operation_id AND k.step_index=s.step_index
JOIN servers device ON device.id=s.server_id
WHERE o.id=$1 AND c.id=$7 AND p.connection_id=c.id
AND o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days'
AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$3
AND p.evidence->>'sealed'=$4 AND s.server_id=$5 AND s.intended_state=$6::jsonb AND device.enabled
AND s.state='verified' AND s.result IS NOT NULL AND k.after_sealed IS NOT NULL
AND NOT EXISTS(SELECT 1 FROM mcp_operations newer JOIN mcp_operation_steps ns ON ns.operation_id=newer.id
WHERE newer.created_at>o.created_at AND ns.server_id=s.server_id)`
export async function detectMcpDrift() {
const generation = await accessGeneration()
const { rows } = await pool.query<{ id: string; connection_id: string; evidence: { sealed: string } }>(`SELECT o.id,o.connection_id,p.evidence
FROM mcp_operations o JOIN mcp_plans p ON p.id=o.plan_id JOIN mcp_connections c ON c.id=o.connection_id
WHERE o.state='applied' AND o.verification='passed' AND o.finished_at>now()-interval '7 days'
AND c.policy->>'status'='active' AND (c.policy->>'expiresAt')::timestamptz>now() AND c.generation=$1
ORDER BY o.finished_at DESC LIMIT 50`, [generation])
const seen = new Set<string>()
for (const row of rows) {
let steps: ChangeEvidence['steps']
try {
const evidence = JSON.parse(await decryptSecretExisting(row.evidence.sealed)) as ChangeEvidence
if (!Array.isArray(evidence.steps)) continue
steps = evidence.steps
} catch { continue }
const uncertainResources = new Set<string>()
for (let index = steps.length - 1; index >= 0; index--) {
const step = steps[index]
if (step?.kind !== "ros" || !step.step) continue
const { serverId, resource, action, id } = step.step
const resourceKey = `${serverId}:${resource}`
if (uncertainResources.has(resourceKey)) continue
try {
const result = action === "create"
? (await pool.query("SELECT result FROM mcp_operation_steps WHERE operation_id=$1 AND step_index=$2 AND server_id=$3", [row.id, index, serverId])).rows[0]?.result
: undefined
const nativeId = action === "create" ? acknowledgedCreateId(result) : id
if (action === "create" && !nativeId) {
// A later create without a durable native identity can shadow an older target.
uncertainResources.add(resourceKey)
continue
}
const targetKey = JSON.stringify([serverId, resource, nativeId ?? null])
if (seen.has(targetKey)) continue
seen.add(targetKey)
await withConfigMutationLock([serverId], async () => {
const expected = JSON.stringify(redact(step))
const args = [row.id, index, generation, row.evidence.sealed, serverId, expected, row.connection_id]
const { rows: [checkpoint] } = await pool.query<DriftCheckpoint>(driftCheckpointSql, args)
if (!checkpoint || await accessGeneration() !== generation) return
const before = JSON.parse(await decryptSecretExisting(checkpoint.before_sealed))
const after = JSON.parse(await decryptSecretExisting(checkpoint.after_sealed))
const expectation = proveRosDriftExpectation(step.step, checkpoint.result, before, after)
if (!expectation || expectation.nativeId !== nativeId) return
const current = await readRosSnapshot(serverId, resource, undefined, expectation.sourceProof)
if (expectation.matches(current) !== false) return
const observedAt = new Date().toISOString()
await withBackgroundPartitionWrite(async ({ client }) => {
const { rows: [locked] } = await client.query<DriftCheckpoint>(
`${driftCheckpointSql} FOR SHARE OF o,p,c,s,k,device`, args)
if (!isDeepStrictEqual(locked, checkpoint) || await accessGeneration() !== generation) return
const inserted = await client.query(`INSERT INTO mcp_drift(id,operation_id,server_id,expected,observed)
VALUES($1,$2,$3,$4::jsonb,$5::jsonb) ON CONFLICT DO NOTHING RETURNING id`, [randomUUID(), row.id, serverId,
JSON.stringify(redact({ stepIndex: index, step })), JSON.stringify({ matchesExpected: false, observedAt })])
if (inserted.rows.length) {
await audit(client, "mcp-drift", "drift.detected", row.connection_id, { operationId: row.id, stepIndex: index, serverId })
await client.query("UPDATE mcp_routing_profiles SET enabled=false,state=state || $2::jsonb WHERE connection_id=$1 AND enabled", [row.connection_id, JSON.stringify({ pausedReason: "Обнаружен дрейф конфигурации" })])
}
})
})
} catch { /* Unavailable proof or persistence leaves the next candidate eligible. */ }
}
}
}
const mcpRetentionBatchSize = 1000
async function retainMcpData() {
await withBackgroundPartitionWrite(async ({ client }) => {
const { requestsRetentionDays } = await getMcpSettings(client)
await client.query(`WITH expired AS (
SELECT id FROM mcp_requests WHERE created_at<now()-make_interval(days=>$1) AND status NOT IN ('running')
ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED
) DELETE FROM mcp_requests USING expired WHERE mcp_requests.id=expired.id`, [requestsRetentionDays, mcpRetentionBatchSize])
})
await withBackgroundPartitionWrite(async ({ client }) => {
const { auditRetentionDays } = await getMcpSettings(client)
await client.query(`WITH expired AS (
SELECT id FROM mcp_audit WHERE created_at<now()-make_interval(days=>$1)
ORDER BY created_at,id LIMIT $2 FOR UPDATE SKIP LOCKED
) DELETE FROM mcp_audit USING expired WHERE mcp_audit.id=expired.id`, [auditRetentionDays, mcpRetentionBatchSize])
})
await withBackgroundPartitionWrite(({ client }) => client.query(`WITH expired AS (
SELECT id FROM mcp_artifacts WHERE expires_at<now()
ORDER BY expires_at,id LIMIT $1 FOR UPDATE
) DELETE FROM mcp_artifacts USING expired WHERE mcp_artifacts.id=expired.id`, [mcpRetentionBatchSize]))
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_oauth_requests WHERE expires_at<now()-interval '1 day'"))
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_budget_buckets WHERE bucket<now()-interval '2 days'"))
await withBackgroundPartitionWrite(({ client }) => retainManualInverseChargesWithinOwner(client))
await withBackgroundPartitionWrite(async ({ client }) => {
const { requestsRetentionDays } = await getMcpSettings(client)
await client.query(`WITH candidates AS MATERIALIZED (SELECT source_operation_id FROM mcp_routing_forward_charges
WHERE created_at<now()-make_interval(days=>$1) ORDER BY created_at,source_operation_id LIMIT 1000),
expired AS (SELECT c.source_operation_id FROM candidates c WHERE NOT EXISTS(SELECT 1 FROM mcp_operations o
WHERE o.id=c.source_operation_id AND o.state IN('pending','running')))
DELETE FROM mcp_routing_forward_charges c USING expired e WHERE c.source_operation_id=e.source_operation_id`, [requestsRetentionDays])
})
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_routing_observations WHERE created_at<now()-interval '7 days'"))
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_path_checks WHERE created_at<now()-interval '7 days'"))
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_unauthorized_buckets WHERE bucket<now()-interval '7 days'"))
await withBackgroundPartitionWrite(({ client }) => client.query("DELETE FROM mcp_portal_events WHERE created_at<now()-interval '7 days'"))
}
export async function tickMcpWorkers() {
let entered = false
try {
await sharedDatabaseGate.request(() => withMcpDatabaseRequest(async () => {
entered = true
await tickAdmittedMcpWorkers()
}))
} catch (error) {
if (entered || !(error instanceof McpError && ["PROTOCOL_STORAGE_BUSY", "RATE_LIMITED"].includes(error.code))) throw error
}
}
async function tickAdmittedMcpWorkers() {
const lease = await pool.connect()
let locked = false, unsafe = false
try {
locked = (await lease.query("SELECT pg_try_advisory_lock(hashtextextended('mcp-workers',0)) AS locked")).rows[0].locked
if (!locked) return
await recoverMcpOperations()
await recoverScopedRecoveryIntents()
await retainMcpData()
await retainServerProbeStorage()
await compactCompletedServerPollStorage()
await retireCompletedServerPollOperations()
await collectExpiredServerPollStorage()
await retireOrdinaryMcpOperations()
if ((await getMcpSettings()).enabled) { await detectMcpDrift(); await tickRoutingProfiles() }
await maintainServerProbeOperationRetirement()
} catch (error) {
if (!(error instanceof PartitionWriterBusy)) throw error
} finally {
if (locked) try { await lease.query("SELECT pg_advisory_unlock(hashtextextended('mcp-workers',0))") } catch { unsafe = true }
lease.release(unsafe)
}
}
export function startMcpWorkers(onError: (error: unknown) => void) {
let stopped = false, running: Promise<void> | undefined
const tick = () => {
if (stopped || running) return
running = tickMcpWorkers().catch(onError).finally(() => { running = undefined })
}
const timer = setInterval(tick, 30_000); timer.unref(); tick()
return async () => { stopped = true; clearInterval(timer); await running }
}