377 lines
27 KiB
TypeScript
377 lines
27 KiB
TypeScript
/** Exact internal maintenance entry. No scheduler, public registration, or native work. */
|
|
import { randomUUID } from "node:crypto"
|
|
import { z } from "zod"
|
|
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
|
|
import { constantEqual, decryptSecretExisting, fingerprint, privateFingerprinterExisting } from "../modules/mcp/security.js"
|
|
import { PROBE_PAYLOAD_MAX_BYTES, PROBE_PERMANENT_RESERVATION_BYTES, ProbeStorageRefused, ProbeStorageUnknown, type ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
|
import { findRetiredServerProbeHistory } from "./server-probe-retired-history.js"
|
|
import { matchesServerProbeRetiredTerminal, openServerProbeRetiredSource, projectServerProbeRetiredSource, ServerProbeRetirementSourceRefused } from "./server-probe-retired-source.js"
|
|
import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
|
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
|
import { admitPartitionWriter } from "./partition-write-transaction.js"
|
|
|
|
const targetSchema = z.object({ receiptId: z.string().uuid(), operationId: z.string().uuid(), connectionId: z.string().uuid() }).strict()
|
|
type Target = z.infer<typeof targetSchema>
|
|
function requireThat(value: unknown): asserts value { if (!value) throw new ProbeStorageRefused() }
|
|
|
|
const ORIGINAL = `jsonb_build_object(
|
|
'operation_id',o.id,'connection_id',o.connection_id,'owner_id',c.owner_id,'plan_id',p.id,
|
|
'operation_name',p.operation_id,'operation_version',p.operation_version,'operation_state',o.state,
|
|
'operation_verification',o.verification,'operation_error_code',o.error_code,'operation_result',o.result,
|
|
'plan_state',p.state,'approved_by',p.approved_by,'evidence',p.evidence,'step_index',s.step_index,
|
|
'step_state',s.state,'intended_state',s.intended_state,'step_result',s.result,
|
|
'operation_created_at',o.created_at,'operation_finished_at',o.finished_at,'plan_created_at',p.created_at,
|
|
'plan_approved_at',p.approved_at,'plan_expires_at',p.expires_at,'step_created_at',s.created_at,'step_finished_at',s.finished_at
|
|
) AS original,o.idempotency_key,k.before_sealed,k.after_sealed`
|
|
const ORIGINAL_JOINS = `JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
|
JOIN public.mcp_connections c ON c.id=o.connection_id
|
|
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
|
JOIN public.mcp_operation_checkpoints k ON k.operation_id=o.id AND k.step_index=s.step_index`
|
|
const ORIGINAL_ELIGIBLE = ` AND o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
|
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()
|
|
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
|
AND isfinite(p.expires_at)
|
|
AND s.state='acknowledged' AND s.server_id IS NULL AND s.before_state IS NULL
|
|
AND octet_length(k.before_sealed)<=1024 AND octet_length(k.after_sealed)<=1024
|
|
AND octet_length(p.evidence::text)+octet_length(s.intended_state::text)+octet_length(s.result::text)+octet_length(o.result::text)<=2097152
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_steps WHERE operation_id=o.id AND step_index<>0)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id AND step_index<>0)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_quarantines WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_poll_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
|
|
`
|
|
const SOURCE = `SELECT ${ORIGINAL},to_jsonb(a) AS admission,to_jsonb(j) AS anchor,to_jsonb(b) AS payload
|
|
FROM public.server_probe_storage_admissions a
|
|
JOIN public.server_probe_executions j ON j.receipt_id=a.receipt_id
|
|
JOIN public.server_probe_execution_payloads b ON b.receipt_id=a.receipt_id
|
|
JOIN public.mcp_operations o ON o.id=a.execution_id
|
|
${ORIGINAL_JOINS}
|
|
WHERE a.receipt_id=$1 AND o.id=$2 AND c.id=$3 AND c.owner_id=a.owner_id AND a.identity_kind='mcp'
|
|
AND a.claim_token IS NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
|
|
AND (($5::timestamptz IS NULL AND a.authority_expires_at<transaction_timestamp() AND p.expires_at<transaction_timestamp())
|
|
OR o.finished_at=$5::timestamptz)
|
|
AND (NOT $4::boolean OR o.finished_at<transaction_timestamp()-interval '180 days')
|
|
${ORIGINAL_ELIGIBLE} FOR UPDATE OF a,j,b,o,p,s,k NOWAIT`
|
|
const CLOSED_SOURCE = `SELECT ${ORIGINAL},f.source,f.source_proof,f.idempotency_digest,f.capture_proof
|
|
FROM public.mcp_operations o ${ORIGINAL_JOINS}
|
|
JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id AND f.receipt_id=$1
|
|
WHERE o.id=$2 AND c.id=$3 AND o.finished_at<transaction_timestamp()-interval '180 days' AND p.expires_at<transaction_timestamp()
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_executions WHERE receipt_id=$1 OR(identity_kind='mcp' AND execution_id=o.id))
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_payloads WHERE receipt_id=$1)
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_execution_stages WHERE receipt_id=$1)
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1 OR operation_id=o.id)
|
|
${ORIGINAL_ELIGIBLE} FOR UPDATE OF f,o,p,s,k NOWAIT`
|
|
|
|
async function transaction<T>(deadlineAt: number, work: (sql: ProbeStorageSql) => Promise<T>, mode: "default" | "fenced-write" = "default"): Promise<T> {
|
|
const sql = await connectProbeStorage({ deadlineAt })
|
|
let committing = false, committed = false
|
|
let timer: ReturnType<typeof setTimeout> | undefined
|
|
try {
|
|
await sql.query("BEGIN")
|
|
if (mode === "fenced-write") {
|
|
// There is no mutation to reconcile if the common writer fence refuses.
|
|
try { await admitPartitionWriter(sql, { deadlineAt: Math.min(deadlineAt, Date.now()+1000) }) }
|
|
catch { throw new ProbeStorageRefused() }
|
|
}
|
|
await sql.query("SELECT set_config('TimeZone','UTC',true),set_config('statement_timeout','5000ms',true),set_config('lock_timeout','1000ms',true)")
|
|
// The same order as admission, publication, cancellation and payload GC.
|
|
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended('server-probe-storage-v2',0)) AS locked")).rows[0]?.locked === true)
|
|
// Bound non-SQL proof work as well. Any late continuation retains only
|
|
// this permanently released checkout and cannot issue further SQL.
|
|
const expired = new Promise<never>((_resolve,reject) => {
|
|
timer = setTimeout(() => {
|
|
sql.release(true)
|
|
reject(new ProbeStorageRefused())
|
|
}, Math.max(1,deadlineAt-Date.now()))
|
|
})
|
|
const result = await Promise.race([work(sql),expired])
|
|
clearTimeout(timer)
|
|
requireThat(Date.now()<deadlineAt)
|
|
committing = true
|
|
await sql.query("COMMIT")
|
|
committed = true
|
|
return result
|
|
} catch (error) {
|
|
if (committing) throw new ProbeStorageUnknown()
|
|
try { await sql.query("ROLLBACK") } catch { /* Checkout is discarded. */ }
|
|
throw error
|
|
} finally {
|
|
clearTimeout(timer)
|
|
sql.release(!committed)
|
|
}
|
|
}
|
|
|
|
async function lockOriginal(sql: ProbeStorageSql, { operationId, connectionId }: Target) {
|
|
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 FOR UPDATE NOWAIT", [connectionId])).rows.length === 1)
|
|
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked", ["mcp-operation:" + operationId])).rows[0]?.locked === true)
|
|
}
|
|
async function assertEmptyCheckpoints(row: Record<string, unknown>) {
|
|
for (const sealed of [row.before_sealed, row.after_sealed]) {
|
|
requireThat(typeof sealed === "string")
|
|
let decoded: unknown
|
|
try { decoded = JSON.parse(await decryptSecretExisting(sealed)) }
|
|
catch { throw new ProbeStorageRefused() }
|
|
requireThat(Array.isArray(decoded) && decoded.length === 0)
|
|
}
|
|
}
|
|
async function prepare(sql: ProbeStorageSql, target: Target, requireOld = true, expectedFinishedAt?: string) {
|
|
const { receiptId, operationId, connectionId } = target
|
|
await lockOriginal(sql, target)
|
|
const rows = (await sql.query(SOURCE, [receiptId, operationId, connectionId, requireOld, expectedFinishedAt ?? null])).rows
|
|
requireThat(rows.length === 1)
|
|
const { original, admission, anchor, payload, idempotency_key: key } = rows[0]
|
|
requireThat(typeof key === "string")
|
|
await assertEmptyCheckpoints(rows[0])
|
|
const stages = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
|
|
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows.map(row => row.stage)
|
|
const signed = await projectServerProbeRetiredSource({ original, admission, anchor, payload, stages, quarantineCount: 0, cancellationCount: 0 })
|
|
requireThat(signed.source.receiptId === receiptId && signed.source.executionId === operationId && signed.source.connectionId === connectionId)
|
|
return { signed, key }
|
|
}
|
|
|
|
const captureHash = z.string().regex(/^[a-f0-9]{64}$/)
|
|
async function terminalCaptureProof(sourceProof: string, idempotencyDigest: string) {
|
|
const sign = await privateFingerprinterExisting("server-probe-terminal-capture-v1")
|
|
return sign(fingerprint({ sourceProof, idempotencyDigest }))
|
|
}
|
|
|
|
function captureTerminalProof(target: Target, deadlineAt: number, expectedFinishedAt?: string) {
|
|
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
|
|
const { signed, key } = await prepare(sql, target, false, expectedFinishedAt)
|
|
const digest = operationIdempotencyDigest(target.connectionId, key)
|
|
const proof = await terminalCaptureProof(signed.proof, digest)
|
|
const saved = (await sql.query(`INSERT INTO public.server_probe_terminal_proofs
|
|
(operation_id,receipt_id,source,source_proof,idempotency_digest,capture_proof)
|
|
VALUES($1,$2,$3::jsonb,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
|
|
[target.operationId,target.receiptId,JSON.stringify(signed.source),signed.proof,digest,proof])).rows
|
|
requireThat(saved.length === 1 && saved[0].receipt_id === target.receiptId)
|
|
return { captured:true as const, automaticReplay:false as const }
|
|
}, "fenced-write"))
|
|
}
|
|
/** Captures evidence only while the normal retention owner can still read the original. */
|
|
export function captureServerProbeTerminalProof(raw: Target, deadlineAt = Date.now()+20000) {
|
|
return captureTerminalProof(targetSchema.parse(raw), deadlineAt)
|
|
}
|
|
|
|
/** Only the operation runner supplies the positively acknowledged terminal SQL timestamp. */
|
|
export function capturePublishedServerProbeTerminalProof(raw: Target & { finishedAt: string }) {
|
|
const { finishedAt, ...identity } = raw
|
|
const stamp = z.string().regex(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{6}Z$/).parse(finishedAt)
|
|
return captureTerminalProof(targetSchema.parse(identity), Date.now()+1200, stamp)
|
|
}
|
|
|
|
async function prepareClosed(sql: ProbeStorageSql, target: Target) {
|
|
const { receiptId, operationId, connectionId } = target
|
|
await lockOriginal(sql, target)
|
|
const rows = (await sql.query(CLOSED_SOURCE, [receiptId, operationId, connectionId])).rows
|
|
requireThat(rows.length === 1)
|
|
const row = rows[0], key = z.string().parse(row.idempotency_key)
|
|
await assertEmptyCheckpoints(row)
|
|
const proof = captureHash.parse(row.source_proof), digest = operationIdempotencyDigest(connectionId, key)
|
|
requireThat(digest === row.idempotency_digest
|
|
&& constantEqual(captureHash.parse(row.capture_proof), await terminalCaptureProof(proof, digest)))
|
|
const source = await openServerProbeRetiredSource({ source:row.source, proof })
|
|
requireThat(source.receiptId === receiptId && source.executionId === operationId && source.connectionId === connectionId
|
|
&& matchesServerProbeRetiredTerminal(source, row.original))
|
|
return { signed:{source,proof},key }
|
|
}
|
|
|
|
function retireServerProbeOperation(raw: Target, kind: "retained" | "expired", deadlineAt = Date.now()+20000) {
|
|
const target = targetSchema.parse(raw)
|
|
const { receiptId, operationId, connectionId } = target
|
|
return sharedDatabaseGate.request(async () => {
|
|
const token = randomUUID()
|
|
const read = kind === "retained" ? prepare : prepareClosed
|
|
const claim = await transaction(deadlineAt, async sql => {
|
|
const { signed, key } = await read(sql, target)
|
|
const digest = operationIdempotencyDigest(connectionId, key)
|
|
const result = await sql.query(`INSERT INTO public.server_probe_retirement_claims
|
|
(receipt_id,operation_id,plan_id,connection_id,source_proof,claim_token)
|
|
VALUES($1,$2,$3,$4,$5,$6) ON CONFLICT DO NOTHING RETURNING receipt_id`,
|
|
[receiptId, operationId, signed.source.planId, connectionId, signed.proof, token])
|
|
requireThat(result.rows.length === 1 && result.rows[0].receipt_id === receiptId)
|
|
return { proof: signed.proof, planId: signed.source.planId, digest }
|
|
}, "fenced-write")
|
|
return transaction(deadlineAt, async sql => {
|
|
const { signed, key } = await read(sql, target), { planId, proof, digest } = claim
|
|
requireThat(signed.proof === proof && signed.source.planId === planId && operationIdempotencyDigest(connectionId, key) === digest)
|
|
const consumed = await sql.query(`SELECT receipt_id FROM public.server_probe_retirement_claims
|
|
WHERE receipt_id=$1 AND operation_id=$2 AND plan_id=$3 AND connection_id=$4 AND source_proof=$5
|
|
AND claim_token=$6 AND claim_xid<>pg_current_xact_id()`, [receiptId, operationId, planId, connectionId, proof, token])
|
|
requireThat(consumed.rows.length === 1 && consumed.rows[0].receipt_id === receiptId)
|
|
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
|
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
|
VALUES($1,$2,$3,$4,'servers.probes.run',$5::bigint[],false) RETURNING operation_id`,
|
|
[operationId, planId, connectionId, digest, signed.source.serverIds])
|
|
requireThat(tombstone.rows.length === 1 && tombstone.rows[0].operation_id === operationId)
|
|
await sql.query("SELECT set_config('mm.probe_retirement_claim',$1,true)", [token])
|
|
if (kind === "retained") {
|
|
const archive = await sql.query(`INSERT INTO public.server_probe_retired_sources(receipt_id,operation_id,source,proof)
|
|
VALUES($1,$2,$3::jsonb,$4) RETURNING receipt_id`, [receiptId, operationId, JSON.stringify(signed.source), proof])
|
|
requireThat(archive.rows.length === 1 && archive.rows[0].receipt_id === receiptId)
|
|
}
|
|
const released = await sql.query("DELETE FROM public.server_probe_terminal_proofs WHERE operation_id=$1 RETURNING receipt_id", [operationId])
|
|
requireThat(released.rows.length <= 1 && released.rows.every(row => row.receipt_id === receiptId)
|
|
&& (kind !== "expired" || released.rows.length === 1))
|
|
const checkpoint = await sql.query("DELETE FROM public.mcp_operation_checkpoints WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
|
|
requireThat(checkpoint.rows.length === 1 && checkpoint.rows[0].operation_id === operationId)
|
|
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id", [operationId])
|
|
requireThat(step.rows.length === 1 && step.rows[0].operation_id === operationId)
|
|
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id", [operationId, connectionId, planId])
|
|
requireThat(operation.rows.length === 1 && operation.rows[0].id === operationId)
|
|
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id", [planId, connectionId])
|
|
requireThat(plan.rows.length === 1 && plan.rows[0].id === planId)
|
|
if (kind === "retained") {
|
|
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
|
|
{ ...target, ownerId: signed.source.ownerId, scopeServerIds: signed.source.serverIds })
|
|
requireThat(observed && !observed.payloadPurged && fingerprint(observed.source) === fingerprint(signed.source))
|
|
}
|
|
return { retired: true as const, automaticReplay: false as const }
|
|
}, "fenced-write")
|
|
})
|
|
}
|
|
export function retireCompletedServerProbeOperation(raw: Target) {
|
|
return retireServerProbeOperation(raw, "retained")
|
|
}
|
|
export function retireExpiredServerProbeOperation(raw: Target) {
|
|
return retireServerProbeOperation(raw, "expired")
|
|
}
|
|
|
|
/** Exact internal GC. This entry has no worker or public registration. */
|
|
export function collectExpiredRetiredServerProbe(raw: Target, deadlineAt = Date.now()+20000) {
|
|
const target = targetSchema.parse(raw)
|
|
const { receiptId, operationId, connectionId } = target
|
|
return sharedDatabaseGate.request(() => transaction(deadlineAt, async sql => {
|
|
const settings = (await sql.query("SELECT requests_retention_days FROM public.mcp_settings WHERE id=1 FOR SHARE NOWAIT")).rows
|
|
const retentionDays = settings[0]?.requests_retention_days
|
|
requireThat(settings.length === 1 && typeof retentionDays === "number" && Number.isInteger(retentionDays)
|
|
&& retentionDays >= 1 && retentionDays <= 365)
|
|
const pinned = (await sql.query(`SELECT c.owner_id,r.proof,t.target_server_ids::text[] AS server_ids FROM public.server_probe_retired_sources r
|
|
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
|
|
JOIN public.mcp_connections c ON c.id=t.connection_id
|
|
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.owner_id=c.owner_id
|
|
JOIN public.server_probe_executions j ON j.receipt_id=r.receipt_id
|
|
WHERE r.receipt_id=$1 AND r.operation_id=$2 AND c.id=$3 FOR UPDATE OF c,r,a,j NOWAIT`,
|
|
[receiptId, operationId, connectionId])).rows
|
|
const identity = z.object({ owner_id: z.string().min(1).max(200), proof: z.string().regex(/^[a-f0-9]{64}$/),
|
|
server_ids: z.array(z.string().regex(/^[1-9]\d{0,15}$/).transform(Number).pipe(z.number().int().positive().safe())).min(1).max(2) }).strict().parse(pinned[0])
|
|
requireThat(pinned.length === 1)
|
|
const observed = await findRetiredServerProbeHistory(async (text, values) => (await sql.query(text, values)).rows,
|
|
{ ...target, ownerId: identity.owner_id, scopeServerIds: identity.server_ids })
|
|
requireThat(observed)
|
|
const stageRows = (await sql.query(`SELECT to_jsonb(s) AS stage FROM public.server_probe_execution_stages s
|
|
WHERE s.receipt_id=$1 ORDER BY s.stage_index LIMIT 12 FOR UPDATE NOWAIT`, [receiptId])).rows
|
|
requireThat(stageRows.length >= 1 && stageRows.length <= 11
|
|
&& fingerprint(stageRows.map(row => row.stage)) === observed.source.stageSnapshotDigest)
|
|
await sql.query(`SELECT set_config('mm.probe_storage_gc','on',true),set_config('mm.probe_retired_gc_receipt',$1,true),
|
|
set_config('mm.probe_retired_gc_proof',$2,true)`, [receiptId, identity.proof])
|
|
requireThat((await sql.query("SELECT public.server_probe_storage_gc_allowed($1) AS allowed", [receiptId])).rows[0]?.allowed === true)
|
|
const deletedPayload = (await sql.query("DELETE FROM public.server_probe_execution_payloads WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
|
const files = observed.payloadPurged ? 0 : 1
|
|
requireThat(deletedPayload.length === files && deletedPayload.every(row => row.receipt_id === receiptId))
|
|
const deletedStages = (await sql.query("DELETE FROM public.server_probe_execution_stages WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
|
requireThat(deletedStages.length === stageRows.length && deletedStages.every(row => row.receipt_id === receiptId))
|
|
const deletedAnchor = (await sql.query("DELETE FROM public.server_probe_executions WHERE receipt_id=$1 RETURNING receipt_id", [receiptId])).rows
|
|
requireThat(deletedAnchor.length === 1 && deletedAnchor[0].receipt_id === receiptId)
|
|
// Admission deletion cascades to the exact authenticated archive only.
|
|
const deletedAdmission = (await sql.query("DELETE FROM public.server_probe_storage_admissions WHERE receipt_id=$1 AND owner_id=$2 RETURNING receipt_id",
|
|
[receiptId, identity.owner_id])).rows
|
|
requireThat(deletedAdmission.length === 1 && deletedAdmission[0].receipt_id === receiptId)
|
|
const scopes = ["global", "owner:" + identity.owner_id], bytes = PROBE_PERMANENT_RESERVATION_BYTES + files * PROBE_PAYLOAD_MAX_BYTES
|
|
const released = (await sql.query(`UPDATE public.server_probe_storage_usage SET admissions=admissions-1,
|
|
reserved_files=reserved_files-$2,reserved_bytes=reserved_bytes-$3 WHERE scope=ANY($1::text[])
|
|
AND admissions>=1 AND reserved_files>=$2 AND reserved_bytes>=$3 RETURNING scope`, [scopes, files, bytes])).rows
|
|
requireThat(released.length === 2 && scopes.every(scope => released.some(row => row.scope === scope)))
|
|
await sql.query("DELETE FROM public.server_probe_storage_usage WHERE scope=$1 AND admissions=0 AND reserved_files=0 AND reserved_bytes=0", [scopes[1]])
|
|
const state = (await sql.query(`SELECT
|
|
NOT EXISTS(SELECT 1 FROM public.server_probe_retired_sources WHERE receipt_id=$1) AS archive_gone,
|
|
EXISTS(SELECT 1 FROM public.mcp_operation_replay_tombstones WHERE operation_id=$2 AND connection_id=$3) AS replay_retained,
|
|
EXISTS(SELECT 1 FROM public.server_probe_retirement_claims WHERE receipt_id=$1 AND operation_id=$2 AND connection_id=$3) AS claim_retained`,
|
|
[receiptId, operationId, connectionId])).rows[0]
|
|
requireThat(state?.archive_gone === true && state.replay_retained === true && state.claim_retained === true)
|
|
return { collected: true as const, automaticReplay: false as const }
|
|
}, "fenced-write"))
|
|
}
|
|
|
|
const RETIRE_CANDIDATES = `SELECT COALESCE(a.receipt_id,f.receipt_id) AS receipt_id,o.id AS operation_id,o.connection_id,
|
|
o.finished_at::text AS ordered_at,CASE WHEN a.receipt_id IS NULL THEN 'expired' ELSE 'retained' END AS kind
|
|
FROM public.mcp_operations o
|
|
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
|
JOIN public.mcp_connections c ON c.id=o.connection_id
|
|
LEFT JOIN public.server_probe_storage_admissions a ON a.identity_kind='mcp' AND a.execution_id=o.id AND a.owner_id=c.owner_id
|
|
LEFT JOIN public.server_probe_terminal_proofs f ON f.operation_id=o.id
|
|
WHERE o.state='applied' AND o.verification='inconclusive' AND o.error_code IS NULL
|
|
AND isfinite(o.finished_at) AND o.finished_at<transaction_timestamp()-interval '180 days'
|
|
AND p.operation_id='servers.probes.run' AND p.operation_version=1 AND p.state='completed'
|
|
AND ((a.receipt_id IS NULL AND f.receipt_id IS NOT NULL)
|
|
OR (a.claim_token IS NULL AND a.claim_attempted_at IS NOT NULL AND a.purged_at IS NULL AND a.cancelled_at IS NULL
|
|
AND EXISTS(SELECT 1 FROM public.server_probe_execution_payloads b WHERE b.receipt_id=a.receipt_id)))
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_retirement_claims x WHERE x.operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)
|
|
AND ($1::timestamptz IS NULL OR (o.finished_at,o.id)>($1::timestamptz,$2::uuid))
|
|
ORDER BY o.finished_at,o.id LIMIT 8`
|
|
const GC_CANDIDATES = `SELECT r.receipt_id,r.operation_id,t.connection_id,a.gc_not_before::text AS ordered_at,'gc' AS kind
|
|
FROM public.server_probe_retired_sources r
|
|
JOIN public.server_probe_storage_admissions a ON a.receipt_id=r.receipt_id AND a.execution_id=r.operation_id
|
|
JOIN public.mcp_operation_replay_tombstones t ON t.operation_id=r.operation_id
|
|
JOIN public.mcp_connections c ON c.id=t.connection_id AND c.owner_id=a.owner_id
|
|
WHERE isfinite(a.gc_not_before) AND a.gc_not_before<transaction_timestamp()
|
|
AND t.action_id='servers.probes.run' AND NOT t.target_global
|
|
AND ($1::timestamptz IS NULL OR (a.gc_not_before,r.operation_id)>($1::timestamptz,$2::uuid))
|
|
ORDER BY a.gc_not_before,r.operation_id LIMIT 8`
|
|
|
|
type Cursor = { orderedAt: string; operationId: string }
|
|
let retireCursor: Cursor | undefined, gcCursor: Cursor | undefined
|
|
const workerRow = z.object({ receipt_id:z.string().uuid(),operation_id:z.string().uuid(),connection_id:z.string().uuid(),
|
|
ordered_at:z.string().refine(value=>Number.isFinite(Date.parse(value))),kind:z.enum(["retained","expired","gc"]) }).strict()
|
|
|
|
/** Invoked only under the existing scheduler owner. Keep expiry and one-shot
|
|
* retirement independent; a claimed operation is never selected again. */
|
|
export function maintainServerProbeOperationRetirement() {
|
|
return sharedDatabaseGate.request(async () => {
|
|
const deadlineAt = Date.now()+20000
|
|
const scan = async (lane: "retire" | "gc") => {
|
|
const cursor = lane === "retire" ? retireCursor : gcCursor
|
|
const setCursor = (value: Cursor | undefined) => { if (lane === "retire") retireCursor=value; else gcCursor=value }
|
|
const rows = await transaction(deadlineAt,async sql => (await sql.query(lane === "retire" ? RETIRE_CANDIDATES : GC_CANDIDATES,
|
|
[cursor?.orderedAt ?? null,cursor?.operationId ?? null])).rows)
|
|
requireThat(rows.length<=8)
|
|
let scanned=0, completed=0
|
|
for (const raw of rows) {
|
|
if (Date.now()>=deadlineAt-2000 || completed>=2) break
|
|
const row=workerRow.parse(raw)
|
|
requireThat((row.kind === "gc") === (lane === "gc"))
|
|
const target={receiptId:row.receipt_id,operationId:row.operation_id,connectionId:row.connection_id}
|
|
setCursor({orderedAt:row.ordered_at,operationId:row.operation_id}); scanned++
|
|
try {
|
|
if (row.kind === "gc") await collectExpiredRetiredServerProbe(target,deadlineAt)
|
|
else await retireServerProbeOperation(target,row.kind,deadlineAt)
|
|
completed++
|
|
} catch (error) {
|
|
if (!(error instanceof ProbeStorageRefused || error instanceof ServerProbeRetirementSourceRefused || error instanceof z.ZodError)) throw error
|
|
}
|
|
}
|
|
if (scanned === rows.length && rows.length < 8) setCursor(undefined)
|
|
return completed
|
|
}
|
|
const collected=await scan("gc")
|
|
const retired=Date.now()<deadlineAt-2000 ? await scan("retire") : 0
|
|
return {collected,retired,automaticReplay:false as const}
|
|
})
|
|
}
|