Files
MikrotikManager/backend/src/services/server-probe-operation-retirement.ts
T

377 lines
27 KiB
TypeScript

/** Exact internal maintenance entry. No scheduler, public registration, or native work. */
import { randomUUID } from "node:crypto"
import { z } from "zod"
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js"
import { PROBE_PAYLOAD_MAX_BYTES, PROBE_PERMANENT_RESERVATION_BYTES, ProbeStorageRefused, ProbeStorageUnknown, type ProbeStorageSql } from "./server-probe-journal-maintenance.js"
import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js"
import { matchesServerProbeRetiredTerminal, openServerProbeRetiredSource, projectServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js"
import { connectProbeStorage } from "./server-probe-storage-connection.js"
import { sharedDatabaseGate } from "./shared-database-gate.js"
import { admitPartitionWriter } from "./partition-write-transaction.js"
const targetSchema = z.object({ receiptId: z.string().uuid(), operationId: z.string().uuid(), connectionId: z.string().uuid() }).strict()
type Target = z.infer<typeof targetSchema>
function requireThat(value: unknown): asserts value { if (!value) throw new ProbeStorageRefused() }
const ORIGINAL = `jsonb_build_object(
'operation_id',o.id,'connection_id',o.connection_id,'owner_id',c.owner_id,'plan_id',p.id,
'operation_name',p.operation_id,'operation_version',p.operation_version,'operation_state',o.state,
'operation_verification',o.verification,'operation_error_code',o.error_code,'operation_result',o.result,
'plan_state',p.state,'approved_by',p.approved_by,'evidence',p.evidence,'step_index',s.step_index,
'step_state',s.state,'intended_state',s.intended_state,'step_result',s.result,
'operation_created_at',o.created_at,'operation_finished_at',o.finished_at,'plan_created_at',p.created_at,
'plan_approved_at',p.approved_at,'plan_expires_at',p.expires_at,'step_created_at',s.created_at,'step_finished_at',s.finished_at
) AS original,o.idempotency_key,k.before_sealed,k.after_sealed`
const ORIGINAL_JOINS = `JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
JOIN public.mcp_connections c ON c.id=o.connection_id
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=s.step_index`
const ORIGINAL_ELIGIBLE = ` AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
AND isfinite(p.expires_at)
AND s.state='acknowledged' AND s.server_id IS NULL AND s.before_state IS NULL
AND octet_length(k.before_sealed)<=1024 AND octet_length(k.after_sealed)<=1024
AND octet_length(p.evidence::text)+octet_length(s.intended_state::text)+octet_length(s.result::text)+octet_length(o.result::text)<=2097152
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=o.id AND step_index<>0)
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id AND step_index<>0)
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
`
const SOURCE = `SELECT ${ORIGINAL},to_jsonb(a) AS admission,to_jsonb(j) AS anchor,to_jsonb(b) AS payload
FROM public.server_probe_storage_admissions a
JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id
JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id
JOIN public.mcp_operations o ON o.id=a.execution_id
${ORIGINAL_JOINS}
WHERE a.receipt_id=$1 AND o.id=$2 AND c.id=$3 AND c.owner_id=a.owner_id AND a.identity_kind='mcp'
AND a.claim_token IS NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
AND (($5::timestamptz IS NULL AND a.authority_expires_at<transaction_timestamp() AND p.expires_at<transaction_timestamp())
OR o.finished_at=$5::timestamptz)
AND (NOT $4::boolean OR o.finished_at<transaction_timestamp()-interval '180 days')
${ORIGINAL_ELIGIBLE} FOR UPDATE OF a,j,b,o,p,s,k NOWAIT`
const CLOSED_SOURCE = `SELECT ${ORIGINAL},f.source,f.source_proof,f.idempotency_digest,f.capture_proof
FROM public.mcp_operations o ${ORIGINAL_JOINS}
JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id AND f.receipt_id=$1
WHERE o.id=$2 AND c.id=$3 AND o.finished_at<transaction_timestamp()-interval '180 days' AND p.expires_at<transaction_timestamp()
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
AND NOT EXISTS(SELECT 1 FROM public.server_probe_executions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_payloads WHERE receipt_id=$1)
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_stages WHERE receipt_id=$1)
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1 OR operation_id=o.id)
${ORIGINAL_ELIGIBLE} FOR UPDATE OF f,o,p,s,k NOWAIT`
async function transaction<T>(deadlineAt: number, work: (sql: ProbeStorageSql) => Promise<T>, mode: "default" | "fenced-write" = "default"): Promise<T> {
const sql = await connectProbeStorage({ deadlineAt })
let committing = false, committed = false
let timer: ReturnType<typeof setTimeout> | undefined
try {
await sql.query("BEGIN")
if (mode === "fenced-write") {
// There is no mutation to reconcile if the common writer fence refuses.
try { await admitPartitionWriter(sql, { deadlineAt: Math.min(deadlineAt, Date.now()+1000) }) }
catch { throw new ProbeStorageRefused() }
}
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
// The same order as admission, publication, cancellation and payload GC.
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)) AS locked")).rows[0]?.locked === true)
// Bound non-SQL proof work as well. Any late continuation retains only
// this permanently released checkout and cannot issue further SQL.
const expired = new Promise<never>((_resolve,reject) => {
timer = setTimeout(() => {
sql.release(true)
reject(new ProbeStorageRefused())
}, Math.max(1,deadlineAt-Date.now()))
})
const result = await Promise.race([work(sql),expired])
clearTimeout(timer)
requireThat(Date.now()<deadlineAt)
committing = true
await sql.query("COMMIT")
committed = true
return result
} catch (error) {
if (committing) throw new ProbeStorageUnknown()
try { await sql.query("ROLLBACK") } catch { /* Checkout is discarded. */ }
throw error
} finally {
clearTimeout(timer)
sql.release(!committed)
}
}
async function lockOriginal(sql: ProbeStorageSql, { operationId, connectionId }: Target) {
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT", [connectionId])).rows.length === 1)
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + operationId])).rows[0]?.locked === true)
}
async function assertEmptyCheckpoints(row: Record<string, unknown>) {
for (const sealed of [row.before_sealed, row.after_sealed]) {
requireThat(typeof sealed === "string")
let decoded: unknown
try { decoded = JSON.parse(await decryptSecretExisting(sealed)) }
catch { throw new ProbeStorageRefused() }
requireThat(Array.isArray(decoded) && decoded.length === 0)
}
}
async function prepare(sql: ProbeStorageSql, target: Target, requireOld = true, expectedFinishedAt?: string) {
const { receiptId, operationId, connectionId } = target
await lockOriginal(sql, target)
const rows = (await sql.query(SOURCE, [receiptId, operationId, connectionId, requireOld, expectedFinishedAt ?? null])).rows
requireThat(rows.length === 1)
const { original, admission, anchor, payload, idempotency_key: key } = rows[0]
requireThat(typeof key === "string")
await assertEmptyCheckpoints(rows[0])
const stages = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows.map(row => row.stage)
const signed = await projectServerProbeRetiredSource({ original, admission, anchor, payload, stages, quarantineCount: 0, cancellationCount: 0 })
requireThat(signed.source.receiptId === receiptId && signed.source.executionId === operationId && signed.source.connectionId === connectionId)
return { signed, key }
}
const captureHash = z.string().regex(/^[a-f0-9]{64}$/)
async function terminalCaptureProof(sourceProof: string, idempotencyDigest: string) {
const sign = await privateFingerprinterExisting("server-probe-terminal-capture-v1")
return sign(fingerprint({ sourceProof, idempotencyDigest }))
}
function captureTerminalProof(target: Target, deadlineAt: number, expectedFinishedAt?: string) {
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
const { signed, key } = await prepare(sql, target, false, expectedFinishedAt)
const digest = operationIdempotencyDigest(target.connectionId, key)
const proof = await terminalCaptureProof(signed.proof, digest)
const saved = (await sql.query(`INSERT INTO public.server_probe_terminal_proofs
(operation_id,receipt_id,source,source_proof,idempotency_digest,capture_proof)
VALUES($1,$2,$3::jsonb,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
[target.operationId,target.receiptId,JSON.stringify(signed.source),signed.proof,digest,proof])).rows
requireThat(saved.length === 1 && saved[0].receipt_id === target.receiptId)
return { captured:true as const, automaticReplay:false as const }
}, "fenced-write"))
}
/** Captures evidence only while the normal retention owner can still read the original. */
export function captureServerProbeTerminalProof(raw: Target, deadlineAt = Date.now()+20000) {
return captureTerminalProof(targetSchema.parse(raw), deadlineAt)
}
/** Only the operation runner supplies the positively acknowledged terminal SQL timestamp. */
export function capturePublishedServerProbeTerminalProof(raw: Target & { finishedAt: string }) {
const { finishedAt, ...identity } = raw
const stamp = z.string().regex(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/).parse(finishedAt)
return captureTerminalProof(targetSchema.parse(identity), Date.now()+1200, stamp)
}
async function prepareClosed(sql: ProbeStorageSql, target: Target) {
const { receiptId, operationId, connectionId } = target
await lockOriginal(sql, target)
const rows = (await sql.query(CLOSED_SOURCE, [receiptId, operationId, connectionId])).rows
requireThat(rows.length === 1)
const row = rows[0], key = z.string().parse(row.idempotency_key)
await assertEmptyCheckpoints(row)
const proof = captureHash.parse(row.source_proof), digest = operationIdempotencyDigest(connectionId, key)
requireThat(digest === row.idempotency_digest
&& constantEqual(captureHash.parse(row.capture_proof), await terminalCaptureProof(proof, digest)))
const source = await openServerProbeRetiredSource({ source:row.source, proof })
requireThat(source.receiptId === receiptId && source.executionId === operationId && source.connectionId === connectionId
&& matchesServerProbeRetiredTerminal(source, row.original))
return { signed:{source,proof},key }
}
function retireServerProbeOperation(raw: Target, kind: "retained" | "expired", deadlineAt = Date.now()+20000) {
const target = targetSchema.parse(raw)
const { receiptId, operationId, connectionId } = target
return sharedDatabaseGate.request(async () => {
const token = randomUUID()
const read = kind === "retained" ? prepare : prepareClosed
const claim = await transaction(deadlineAt, async sql => {
const { signed, key } = await read(sql, target)
const digest = operationIdempotencyDigest(connectionId, key)
const result = await sql.query(`INSERT INTO public.server_probe_retirement_claims
(receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token)
VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
[receiptId, operationId, signed.source.planId, connectionId, signed.proof, token])
requireThat(result.rows.length === 1 && result.rows[0].receipt_id === receiptId)
return { proof: signed.proof, planId: signed.source.planId, digest }
}, "fenced-write")
return transaction(deadlineAt, async sql => {
const { signed, key } = await read(sql, target), { planId, proof, digest } = claim
requireThat(signed.proof === proof && signed.source.planId === planId && operationIdempotencyDigest(connectionId, key) === digest)
const consumed = await sql.query(`SELECT receipt_id FROM public.server_probe_retirement_claims
WHERE receipt_id=$1 AND operation_id=$2 AND plan_id=$3 AND connection_id=$4 AND source_proof=$5
AND claim_token=$6 AND claim_xid<>pg_current_xact_id()`, [receiptId, operationId, planId, connectionId, proof, token])
requireThat(consumed.rows.length === 1 && consumed.rows[0].receipt_id === receiptId)
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
VALUES($1,$2,$3,$4,'servers.probes.run',$5::bigint[],false) RETURNING operation_id`,
[operationId, planId, connectionId, digest, signed.source.serverIds])
requireThat(tombstone.rows.length === 1 && tombstone.rows[0].operation_id === operationId)
await sql.query("SELECT set_config('mm.probe_retirement_claim',$1,true)", [token])
if (kind === "retained") {
const archive = await sql.query(`INSERT INTO public.server_probe_retired_sources(receipt_id,operation_id,source,proof)
VALUES($1,$2,$3::jsonb,$4) RETURNING receipt_id`, [receiptId, operationId, JSON.stringify(signed.source), proof])
requireThat(archive.rows.length === 1 && archive.rows[0].receipt_id === receiptId)
}
const released = await sql.query("DELETE FROM public.server_probe_terminal_proofs WHERE operation_id=$1 RETURNING receipt_id", [operationId])
requireThat(released.rows.length <= 1 && released.rows.every(row => row.receipt_id === receiptId)
&& (kind !== "expired" || released.rows.length === 1))
const checkpoint = await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
requireThat(checkpoint.rows.length === 1 && checkpoint.rows[0].operation_id === operationId)
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
requireThat(step.rows.length === 1 && step.rows[0].operation_id === operationId)
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id", [operationId, connectionId, planId])
requireThat(operation.rows.length === 1 && operation.rows[0].id === operationId)
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id", [planId, connectionId])
requireThat(plan.rows.length === 1 && plan.rows[0].id === planId)
if (kind === "retained") {
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
{ ...target, ownerId: signed.source.ownerId, scopeServerIds: signed.source.serverIds })
requireThat(observed && !observed.payloadPurged && fingerprint(observed.source) === fingerprint(signed.source))
}
return { retired: true as const, automaticReplay: false as const }
}, "fenced-write")
})
}
export function retireCompletedServerProbeOperation(raw: Target) {
return retireServerProbeOperation(raw, "retained")
}
export function retireExpiredServerProbeOperation(raw: Target) {
return retireServerProbeOperation(raw, "expired")
}
/** Exact internal GC. This entry has no worker or public registration. */
export function collectExpiredRetiredServerProbe(raw: Target, deadlineAt = Date.now()+20000) {
const target = targetSchema.parse(raw)
const { receiptId, operationId, connectionId } = target
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
const settings = (await sql.query("SELECT requests_retention_days FROM public.mcp_settings WHERE id=1 FOR SHARE NOWAIT")).rows
const retentionDays = settings[0]?.requests_retention_days
requireThat(settings.length === 1 && typeof retentionDays === "number" && Number.isInteger(retentionDays)
&& retentionDays >= 1 && retentionDays <= 365)
const pinned = (await sql.query(`SELECT c.owner_id,r.proof,t.target_server_ids::text[] AS server_ids FROM public.server_probe_retired_sources r
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
JOIN public.mcp_connections c ON c.id=t.connection_id
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.owner_id=c.owner_id
JOIN public.server_probe_executions j ON j.receipt_id=r.receipt_id
WHERE r.receipt_id=$1 AND r.operation_id=$2 AND c.id=$3 FOR UPDATE OF c,r,a,j NOWAIT`,
[receiptId, operationId, connectionId])).rows
const identity = z.object({ owner_id: z.string().min(1).max(200), proof: z.string().regex(/^[a-f0-9]{64}$/),
server_ids: z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(z.number().int().positive().safe())).min(1).max(2) }).strict().parse(pinned[0])
requireThat(pinned.length === 1)
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
{ ...target, ownerId: identity.owner_id, scopeServerIds: identity.server_ids })
requireThat(observed)
const stageRows = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows
requireThat(stageRows.length >= 1 && stageRows.length <= 11
&& fingerprint(stageRows.map(row => row.stage)) === observed.source.stageSnapshotDigest)
await sql.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true),
set_config('mm.probe_retired_gc_proof',$2,true)`, [receiptId, identity.proof])
requireThat((await sql.query("SELECT public.server_probe_storage_gc_allowed($1) AS allowed", [receiptId])).rows[0]?.allowed === true)
const deletedPayload = (await sql.query("DELETE FROM public.server_probe_execution_payloads WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
const files = observed.payloadPurged ? 0 : 1
requireThat(deletedPayload.length === files && deletedPayload.every(row => row.receipt_id === receiptId))
const deletedStages = (await sql.query("DELETE FROM public.server_probe_execution_stages WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
requireThat(deletedStages.length === stageRows.length && deletedStages.every(row => row.receipt_id === receiptId))
const deletedAnchor = (await sql.query("DELETE FROM public.server_probe_executions WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
requireThat(deletedAnchor.length === 1 && deletedAnchor[0].receipt_id === receiptId)
// Admission deletion cascades to the exact authenticated archive only.
const deletedAdmission = (await sql.query("DELETE FROM public.server_probe_storage_admissions WHERE receipt_id=$1 AND owner_id=$2 RETURNING receipt_id",
[receiptId, identity.owner_id])).rows
requireThat(deletedAdmission.length === 1 && deletedAdmission[0].receipt_id === receiptId)
const scopes = ["global", "owner:" + identity.owner_id], bytes = PROBE_PERMANENT_RESERVATION_BYTES + files * PROBE_PAYLOAD_MAX_BYTES
const released = (await sql.query(`UPDATE public.server_probe_storage_usage SET admissions=admissions-1,
reserved_files=reserved_files-$2,reserved_bytes=reserved_bytes-$3 WHERE scope=ANY($1::text[])
AND admissions>=1 AND reserved_files>=$2 AND reserved_bytes>=$3 RETURNING scope`, [scopes, files, bytes])).rows
requireThat(released.length === 2 && scopes.every(scope => released.some(row => row.scope === scope)))
await sql.query("DELETE FROM public.server_probe_storage_usage WHERE scope=$1 AND admissions=0 AND reserved_files=0 AND reserved_bytes=0", [scopes[1]])
const state = (await sql.query(`SELECT
NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1) AS archive_gone,
EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones WHERE operation_id=$2 AND connection_id=$3) AS replay_retained,
EXISTS(SELECT 1 FROM public.server_probe_retirement_claims WHERE receipt_id=$1 AND operation_id=$2 AND connection_id=$3) AS claim_retained`,
[receiptId, operationId, connectionId])).rows[0]
requireThat(state?.archive_gone === true && state.replay_retained === true && state.claim_retained === true)
return { collected: true as const, automaticReplay: false as const }
}, "fenced-write"))
}
const RETIRE_CANDIDATES = `SELECT COALESCE(a.receipt_id,f.receipt_id) AS receipt_id,o.id AS operation_id,o.connection_id,
o.finished_at::text AS ordered_at,CASE WHEN a.receipt_id IS NULL THEN 'expired' ELSE 'retained' END AS kind
FROM public.mcp_operations o
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
JOIN public.mcp_connections c ON c.id=o.connection_id
LEFT JOIN public.server_probe_storage_admissions a ON a.identity_kind='mcp' AND a.execution_id=o.id AND a.owner_id=c.owner_id
LEFT JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id
WHERE o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
AND ((a.receipt_id IS NULL AND f.receipt_id IS NOT NULL)
OR (a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
AND EXISTS(SELECT 1 FROM public.server_probe_execution_payloads b WHERE b.receipt_id=a.receipt_id)))
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retirement_claims x WHERE x.operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
ORDER BY o.finished_at,o.id LIMIT 8`
const GC_CANDIDATES = `SELECT r.receipt_id,r.operation_id,t.connection_id,a.gc_not_before::text AS ordered_at,'gc' AS kind
FROM public.server_probe_retired_sources r
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id
WHERE isfinite(a.gc_not_before) AND a.gc_not_before<transaction_timestamp()
AND t.action_id='servers.probes.run' AND NOT t.target_global
AND ($1::timestamptz IS NULL OR (a.gc_not_before,r.operation_id)>($1::timestamptz,$2::uuid))
ORDER BY a.gc_not_before,r.operation_id LIMIT 8`
type Cursor = { orderedAt: string; operationId: string }
let retireCursor: Cursor | undefined, gcCursor: Cursor | undefined
const workerRow = z.object({ receipt_id:z.string().uuid(),operation_id:z.string().uuid(),connection_id:z.string().uuid(),
ordered_at:z.string().refine(value=>Number.isFinite(Date.parse(value))),kind:z.enum(["retained","expired","gc"]) }).strict()
/** Invoked only under the existing scheduler owner. Keep expiry and one-shot
* retirement independent; a claimed operation is never selected again. */
export function maintainServerProbeOperationRetirement() {
return sharedDatabaseGate.request(async () => {
const deadlineAt = Date.now()+20000
const scan = async (lane: "retire" | "gc") => {
const cursor = lane === "retire" ? retireCursor : gcCursor
const setCursor = (value: Cursor | undefined) => { if (lane === "retire") retireCursor=value; else gcCursor=value }
const rows = await transaction(deadlineAt,async sql => (await sql.query(lane === "retire" ? RETIRE_CANDIDATES : GC_CANDIDATES,
[cursor?.orderedAt ?? null,cursor?.operationId ?? null])).rows)
requireThat(rows.length<=8)
let scanned=0, completed=0
for (const raw of rows) {
if (Date.now()>=deadlineAt-2000 || completed>=2) break
const row=workerRow.parse(raw)
requireThat((row.kind === "gc") === (lane === "gc"))
const target={receiptId:row.receipt_id,operationId:row.operation_id,connectionId:row.connection_id}
setCursor({orderedAt:row.ordered_at,operationId:row.operation_id}); scanned++
try {
if (row.kind === "gc") await collectExpiredRetiredServerProbe(target,deadlineAt)
else await retireServerProbeOperation(target,row.kind,deadlineAt)
completed++
} catch (error) {
if (!(error instanceof ProbeStorageRefused || error instanceof ServerProbeRetirementSourceRefused || error instanceof z.ZodError)) throw error
}
}
if (scanned === rows.length && rows.length < 8) setCursor(undefined)
return completed
}
const collected=await scan("gc")
const retired=Date.now()<deadlineAt-2000 ? await scan("retire") : 0
return {collected,retired,automaticReplay:false as const}
})
}