178 lines
13 KiB
TypeScript
178 lines
13 KiB
TypeScript
/** Scheduled/internal maintenance only. No public HTTP or MCP registration. */
|
|
import { randomUUID } from "node:crypto"
|
|
import { z } from "zod"
|
|
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
|
|
import { fingerprint } from "../modules/mcp/security.js"
|
|
import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js"
|
|
import { SERVER_POLL_ARCHIVE_READ_SQL } from "./server-poll-terminal-compaction.js"
|
|
import { openServerPollRetiredArchive, projectServerPollRetiredArchive, type LockedServerPollRetirementSource } from "./server-poll-retired-archive.js"
|
|
import { admitPollStorageWriter, createPollMaintenanceWriter, PollStorageRefused, runPollStorageTransaction, tryAdmitPollStorageMaintenanceWriter } from "./server-poll-storage.js"
|
|
import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js"
|
|
import { connectProbeStorage } from "./server-probe-storage-connection.js"
|
|
import { sharedDatabaseGate } from "./shared-database-gate.js"
|
|
|
|
const request = z.object({ receiptId: z.string().uuid(), oldArchiveProof: z.string().regex(/^[a-f0-9]{64}$/) }).strict()
|
|
const requireThat: (value: unknown) => asserts value = value => {
|
|
if (!value) throw new PollStorageRefused()
|
|
}
|
|
|
|
/** The storage owner already holds its global xact advisory lock. Do not wait
|
|
* for a connection writer or runner that might need the storage lock. */
|
|
const ELIGIBLE = `FROM public.server_poll_storage_admissions a JOIN public.server_poll_archives r USING(receipt_id)
|
|
JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id
|
|
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
|
|
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
|
|
WHERE a.identity_kind='mcp' AND a.storage_state='compacted'
|
|
AND r.version=1 AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id
|
|
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at)
|
|
AND o.finished_at<transaction_timestamp()-interval '180 days'
|
|
AND p.operation_id='servers.poll' AND p.operation_version=1 AND p.state='completed' AND p.approved_by IS NOT NULL
|
|
AND s.state='verified' AND s.server_id IS NULL AND s.finished_at IS NOT NULL
|
|
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
|
|
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
|
|
const SOURCE = `SELECT CASE WHEN octet_length(to_jsonb(a)::text)<=1048576 THEN to_jsonb(a) END AS admission,
|
|
CASE WHEN octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)<=16777216
|
|
THEN jsonb_build_object('operation',to_jsonb(o),'plan',to_jsonb(p),'step',to_jsonb(s)) END AS terminal,
|
|
o.id AS operation_id,p.id AS plan_id,o.idempotency_key,clock_timestamp() AS observed_at
|
|
${ELIGIBLE} AND a.receipt_id=$1 AND r.archive_proof=$2 FOR UPDATE OF a,r,o,p,s NOWAIT`
|
|
const CANDIDATES = `SELECT a.receipt_id,r.archive_proof,o.finished_at::text AS finished_at ${ELIGIBLE}
|
|
AND NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c WHERE c.receipt_id=a.receipt_id)
|
|
AND ($1::timestamptz IS NULL OR (o.finished_at,a.receipt_id)>($1::timestamptz,$2::uuid))
|
|
ORDER BY o.finished_at,a.receipt_id LIMIT 8`
|
|
|
|
const deps = { connect:connectProbeStorage,now:Date.now }
|
|
type Target = z.infer<typeof request>
|
|
|
|
async function prepare(sql: ProbeStorageSql,{receiptId,oldArchiveProof}:Target) {
|
|
const identity = (await sql.query(`SELECT a.connection_id,a.execution_id,a.owner_id FROM public.server_poll_storage_admissions a
|
|
JOIN public.server_poll_archives r USING(receipt_id) WHERE a.receipt_id=$1 AND a.storage_state='compacted'
|
|
AND a.identity_kind='mcp' AND r.version=1 AND r.archive_proof=$2`, [receiptId,oldArchiveProof])).rows
|
|
requireThat(identity.length===1)
|
|
const { connection_id: connectionId,execution_id: operationId,owner_id: ownerId } = identity[0]
|
|
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 AND owner_id=$2 FOR UPDATE NOWAIT",
|
|
[connectionId,ownerId])).rows.length===1)
|
|
for (const key of ["mcp-operation:"+operationId,"server-poll-execution-v1:mcp:"+operationId,"server-poll-receipt-v1:"+receiptId]) {
|
|
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",[key])).rows[0]?.locked===true)
|
|
}
|
|
const rows = (await sql.query(SOURCE,[receiptId,oldArchiveProof])).rows
|
|
requireThat(rows.length===1 && rows[0].operation_id===operationId && rows[0].admission && rows[0].terminal)
|
|
const { admission,terminal,plan_id: planId,idempotency_key: key,observed_at: observedAt } = rows[0]
|
|
requireThat(typeof key==="string" && typeof connectionId==="string" && typeof planId==="string")
|
|
await assertServerPollArchiveAccounting(sql)
|
|
const oldArchive = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
|
|
requireThat(oldArchive.length===1 && oldArchive[0].archive_proof===oldArchiveProof)
|
|
const archived = await projectServerPollRetiredArchive({ archive:oldArchive[0],admission,terminal } as LockedServerPollRetirementSource,
|
|
new Date(observedAt as string | Date).getTime())
|
|
requireThat(archived)
|
|
const projection = await openServerPollRetiredArchive(archived)
|
|
requireThat(projection.metadata.receipt_id===receiptId && projection.metadata.execution_id===operationId
|
|
&& projection.metadata.connection_id===connectionId && projection.metadata.owner_id===ownerId && projection.history.planId===planId)
|
|
return {connectionId,operationId,planId,key,archived,projection,previousBytes:oldArchive[0].ciphertext_bytes}
|
|
}
|
|
|
|
async function retire(target:Target,deadlineAt:number) {
|
|
const writer=createPollMaintenanceWriter(deps,deadlineAt)
|
|
try{
|
|
const result=await retireOnWriter(target,deadlineAt,writer)
|
|
await writer.finish(deadlineAt)
|
|
return result
|
|
}finally{writer.destroy()}
|
|
}
|
|
|
|
async function retireOnWriter(target:Target,deadlineAt:number,writer:ReturnType<typeof createPollMaintenanceWriter>) {
|
|
const {receiptId,oldArchiveProof}=target
|
|
const token=randomUUID()
|
|
await runPollStorageTransaction(writer.deps,{deadlineAt},async()=>{},async sql=>{
|
|
// Check the live hold predicates, all locks and both archive proofs before
|
|
// spending the irreversible one-shot claim.
|
|
await prepare(sql,target)
|
|
const claim=await sql.query(`INSERT INTO public.server_poll_retirement_claims(receipt_id,old_archive_proof,claim_token)
|
|
VALUES($1,$2,$3) ON CONFLICT DO NOTHING RETURNING receipt_id`,[receiptId,oldArchiveProof,token])
|
|
requireThat(claim.rows.length===1&&claim.rows[0].receipt_id===receiptId)
|
|
},async sql=>{
|
|
await admitPollStorageWriter(sql,deadlineAt)
|
|
// Extend the shared writer fence from this transaction through publication.
|
|
requireThat(await writer.tryAcquire(sql))
|
|
})
|
|
// No destructive transaction after an unacknowledged claim. A claim is
|
|
// never cleared, even when a later checkout/hold/COMMIT is inconclusive.
|
|
return runPollStorageTransaction(writer.deps,{deadlineAt},async()=>{},async sql=>{
|
|
const claimed=await sql.query(`SELECT receipt_id FROM public.server_poll_retirement_claims
|
|
WHERE receipt_id=$1 AND old_archive_proof=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,
|
|
[receiptId,oldArchiveProof,token])
|
|
requireThat(claimed.rows.length===1&&claimed.rows[0].receipt_id===receiptId)
|
|
const {connectionId,operationId,planId,key,archived,projection,previousBytes}=await prepare(sql,target)
|
|
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
|
|
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
|
|
VALUES($1,$2,$3,$4,'servers.poll',$5::bigint[],true) RETURNING operation_id`,
|
|
[operationId,planId,connectionId,operationIdempotencyDigest(connectionId,key),projection.history.affectedServerIds])
|
|
requireThat(tombstone.rows.length===1 && tombstone.rows[0].operation_id===operationId)
|
|
await sql.query("SELECT set_config('mm.poll_retire_receipt',$1,true),set_config('mm.poll_retire_claim',$2,true)",[receiptId,token])
|
|
const updated = await sql.query(`UPDATE public.server_poll_archives SET version=2,source_digest=$3,source_sha256=$4,
|
|
decoded_bytes=$5,compressed_bytes=$6,ciphertext=$7,ciphertext_bytes=$8,archive_proof=$9
|
|
WHERE receipt_id=$1 AND version=1 AND archive_proof=$2 RETURNING receipt_id`,
|
|
[receiptId,oldArchiveProof,archived.source_digest,archived.source_sha256,archived.decoded_bytes,archived.compressed_bytes,
|
|
archived.ciphertext,archived.ciphertext_bytes,archived.archive_proof])
|
|
requireThat(updated.rows.length===1 && updated.rows[0].receipt_id===receiptId)
|
|
const charged = await sql.query(`UPDATE public.server_poll_storage_admissions SET archive_bytes=$2
|
|
WHERE receipt_id=$1 AND storage_state='compacted' AND archive_bytes=$3 RETURNING receipt_id`,
|
|
[receiptId,archived.ciphertext_bytes,previousBytes])
|
|
requireThat(charged.rows.length===1 && charged.rows[0].receipt_id===receiptId)
|
|
const readback = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
|
|
requireThat(readback.length===1 && readback[0].archive_proof===archived.archive_proof
|
|
&& fingerprint(await openServerPollRetiredArchive(readback[0]))===fingerprint(projection))
|
|
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
|
|
requireThat(step.rows.length===1 && step.rows[0].operation_id===operationId)
|
|
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id",
|
|
[operationId,connectionId,planId])
|
|
requireThat(operation.rows.length===1 && operation.rows[0].id===operationId)
|
|
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id",[planId,connectionId])
|
|
requireThat(plan.rows.length===1 && plan.rows[0].id===planId)
|
|
await assertServerPollArchiveAccounting(sql)
|
|
return { retired:true as const, automaticReplay:false as const }
|
|
},sql=>admitPollStorageWriter(sql,deadlineAt))
|
|
}
|
|
|
|
/** Exact internal entry used by owned fixtures. It cannot bypass the claim. */
|
|
export function retireCompletedServerPollOperation(raw:Target) {
|
|
const target=request.parse(raw)
|
|
return sharedDatabaseGate.request(()=>retire(target,Date.now()+20000))
|
|
}
|
|
|
|
let cursor:{finishedAt:string;receiptId:string}|undefined
|
|
export function retireCompletedServerPollOperations() {
|
|
return sharedDatabaseGate.request(async()=>{
|
|
const deadlineAt=Date.now()+20000
|
|
// A live POLL may hold both locks. Skip before requesting its storage key.
|
|
const rows=await runPollStorageTransaction<Record<string,unknown>[]|null>(deps,{deadlineAt},async()=>{},async sql=>(await sql.query(CANDIDATES,
|
|
[cursor?.finishedAt??null,cursor?.receiptId??null])).rows,async sql=>{
|
|
if(!await tryAdmitPollStorageMaintenanceWriter(sql,"transaction"))return {deferred:null}
|
|
})
|
|
if(rows===null)return {scanned:0,retired:0,automaticReplay:false as const}
|
|
requireThat(rows.length<=8)
|
|
let retired=0,scanned=0
|
|
for(const row of rows){
|
|
if(Date.now()>=deadlineAt-2000||retired>=2)break
|
|
const target=request.parse({receiptId:row.receipt_id,oldArchiveProof:row.archive_proof})
|
|
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
|
|
cursor={finishedAt:row.finished_at,receiptId:target.receiptId}
|
|
scanned++
|
|
try{await retire(target,deadlineAt);retired++}
|
|
catch(error){if(!(error instanceof PollStorageRefused))throw error}
|
|
}
|
|
// Keep the cursor for any unvisited rows. On exhaustion the next tick
|
|
// starts at the beginning, revisiting formerly held or busy candidates.
|
|
if(scanned===rows.length&&rows.length<8)cursor=undefined
|
|
return {scanned,retired,automaticReplay:false as const}
|
|
})
|
|
}
|