Files
MikrotikManager/backend/src/services/server-poll-operation-retirement.ts
T

178 lines
13 KiB
TypeScript

/** Scheduled/internal maintenance only. No public HTTP or MCP registration. */
import { randomUUID } from "node:crypto"
import { z } from "zod"
import { operationIdempotencyDigest } from "../modules/mcp/operation-replay-tombstones.js"
import { fingerprint } from "../modules/mcp/security.js"
import { assertServerPollArchiveAccounting } from "./server-poll-archive-accounting.js"
import { SERVER_POLL_ARCHIVE_READ_SQL } from "./server-poll-terminal-compaction.js"
import { openServerPollRetiredArchive, projectServerPollRetiredArchive, type LockedServerPollRetirementSource } from "./server-poll-retired-archive.js"
import { admitPollStorageWriter, createPollMaintenanceWriter, PollStorageRefused, runPollStorageTransaction, tryAdmitPollStorageMaintenanceWriter } from "./server-poll-storage.js"
import type { ProbeStorageSql } from "./server-probe-journal-maintenance.js"
import { connectProbeStorage } from "./server-probe-storage-connection.js"
import { sharedDatabaseGate } from "./shared-database-gate.js"
const request = z.object({ receiptId: z.string().uuid(), oldArchiveProof: z.string().regex(/^[a-f0-9]{64}$/) }).strict()
const requireThat: (value: unknown) => asserts value = value => {
if (!value) throw new PollStorageRefused()
}
/** The storage owner already holds its global xact advisory lock. Do not wait
* for a connection writer or runner that might need the storage lock. */
const ELIGIBLE = `FROM public.server_poll_storage_admissions a JOIN public.server_poll_archives r USING(receipt_id)
JOIN public.mcp_operations o ON o.id=a.execution_id AND o.connection_id=a.connection_id
JOIN public.mcp_plans p ON p.id=o.plan_id AND p.connection_id=o.connection_id
JOIN public.mcp_operation_steps s ON s.operation_id=o.id AND s.step_index=0
WHERE a.identity_kind='mcp' AND a.storage_state='compacted'
AND r.version=1 AND r.execution_id=a.execution_id AND r.connection_id=a.connection_id AND r.owner_id=a.owner_id
AND o.state='applied' AND o.verification='passed' AND o.error_code IS NULL AND isfinite(o.finished_at)
AND o.finished_at<transaction_timestamp()-interval '180 days'
AND p.operation_id='servers.poll' AND p.operation_version=1 AND p.state='completed' AND p.approved_by IS NOT NULL
AND s.state='verified' AND s.server_id IS NULL AND s.finished_at IS NOT NULL
AND (SELECT count(*) FROM public.mcp_operation_steps WHERE operation_id=o.id)=1
AND NOT EXISTS(SELECT 1 FROM public.mcp_operation_checkpoints WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_drift WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_recovery_intents WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_scoped_recovery_intents WHERE source_operation_id=o.id OR recovery_operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_reservations WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_forward_charges WHERE source_operation_id=o.id OR source_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_routing_manual_inverse_charges WHERE source_operation_id=o.id OR recovery_operation_id=o.id OR source_plan_id=p.id OR recovery_plan_id=p.id)
AND NOT EXISTS(SELECT 1 FROM public.mcp_diagnostic_cancellations WHERE operation_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.server_probe_storage_admissions WHERE identity_kind='mcp' AND execution_id=o.id)
AND NOT EXISTS(SELECT 1 FROM public.vrf_production_tickets WHERE plan_id=p.id)`
const SOURCE = `SELECT CASE WHEN octet_length(to_jsonb(a)::text)<=1048576 THEN to_jsonb(a) END AS admission,
CASE WHEN octet_length(to_jsonb(o)::text)+octet_length(to_jsonb(p)::text)+octet_length(to_jsonb(s)::text)<=16777216
THEN jsonb_build_object('operation',to_jsonb(o),'plan',to_jsonb(p),'step',to_jsonb(s)) END AS terminal,
o.id AS operation_id,p.id AS plan_id,o.idempotency_key,clock_timestamp() AS observed_at
${ELIGIBLE} AND a.receipt_id=$1 AND r.archive_proof=$2 FOR UPDATE OF a,r,o,p,s NOWAIT`
const CANDIDATES = `SELECT a.receipt_id,r.archive_proof,o.finished_at::text AS finished_at ${ELIGIBLE}
AND NOT EXISTS(SELECT 1 FROM public.server_poll_retirement_claims c WHERE c.receipt_id=a.receipt_id)
AND ($1::timestamptz IS NULL OR (o.finished_at,a.receipt_id)>($1::timestamptz,$2::uuid))
ORDER BY o.finished_at,a.receipt_id LIMIT 8`
const deps = { connect:connectProbeStorage,now:Date.now }
type Target = z.infer<typeof request>
async function prepare(sql: ProbeStorageSql,{receiptId,oldArchiveProof}:Target) {
const identity = (await sql.query(`SELECT a.connection_id,a.execution_id,a.owner_id FROM public.server_poll_storage_admissions a
JOIN public.server_poll_archives r USING(receipt_id) WHERE a.receipt_id=$1 AND a.storage_state='compacted'
AND a.identity_kind='mcp' AND r.version=1 AND r.archive_proof=$2`, [receiptId,oldArchiveProof])).rows
requireThat(identity.length===1)
const { connection_id: connectionId,execution_id: operationId,owner_id: ownerId } = identity[0]
requireThat((await sql.query("SELECT id FROM public.mcp_connections WHERE id=$1 AND owner_id=$2 FOR UPDATE NOWAIT",
[connectionId,ownerId])).rows.length===1)
for (const key of ["mcp-operation:"+operationId,"server-poll-execution-v1:mcp:"+operationId,"server-poll-receipt-v1:"+receiptId]) {
requireThat((await sql.query("SELECT pg_try_advisory_xact_lock(hashtextextended($1,0)) AS locked",[key])).rows[0]?.locked===true)
}
const rows = (await sql.query(SOURCE,[receiptId,oldArchiveProof])).rows
requireThat(rows.length===1 && rows[0].operation_id===operationId && rows[0].admission && rows[0].terminal)
const { admission,terminal,plan_id: planId,idempotency_key: key,observed_at: observedAt } = rows[0]
requireThat(typeof key==="string" && typeof connectionId==="string" && typeof planId==="string")
await assertServerPollArchiveAccounting(sql)
const oldArchive = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
requireThat(oldArchive.length===1 && oldArchive[0].archive_proof===oldArchiveProof)
const archived = await projectServerPollRetiredArchive({ archive:oldArchive[0],admission,terminal } as LockedServerPollRetirementSource,
new Date(observedAt as string | Date).getTime())
requireThat(archived)
const projection = await openServerPollRetiredArchive(archived)
requireThat(projection.metadata.receipt_id===receiptId && projection.metadata.execution_id===operationId
&& projection.metadata.connection_id===connectionId && projection.metadata.owner_id===ownerId && projection.history.planId===planId)
return {connectionId,operationId,planId,key,archived,projection,previousBytes:oldArchive[0].ciphertext_bytes}
}
async function retire(target:Target,deadlineAt:number) {
const writer=createPollMaintenanceWriter(deps,deadlineAt)
try{
const result=await retireOnWriter(target,deadlineAt,writer)
await writer.finish(deadlineAt)
return result
}finally{writer.destroy()}
}
async function retireOnWriter(target:Target,deadlineAt:number,writer:ReturnType<typeof createPollMaintenanceWriter>) {
const {receiptId,oldArchiveProof}=target
const token=randomUUID()
await runPollStorageTransaction(writer.deps,{deadlineAt},async()=>{},async sql=>{
// Check the live hold predicates, all locks and both archive proofs before
// spending the irreversible one-shot claim.
await prepare(sql,target)
const claim=await sql.query(`INSERT INTO public.server_poll_retirement_claims(receipt_id,old_archive_proof,claim_token)
VALUES($1,$2,$3) ON CONFLICT DO NOTHING RETURNING receipt_id`,[receiptId,oldArchiveProof,token])
requireThat(claim.rows.length===1&&claim.rows[0].receipt_id===receiptId)
},async sql=>{
await admitPollStorageWriter(sql,deadlineAt)
// Extend the shared writer fence from this transaction through publication.
requireThat(await writer.tryAcquire(sql))
})
// No destructive transaction after an unacknowledged claim. A claim is
// never cleared, even when a later checkout/hold/COMMIT is inconclusive.
return runPollStorageTransaction(writer.deps,{deadlineAt},async()=>{},async sql=>{
const claimed=await sql.query(`SELECT receipt_id FROM public.server_poll_retirement_claims
WHERE receipt_id=$1 AND old_archive_proof=$2 AND claim_token=$3 AND claim_xid<>pg_current_xact_id()`,
[receiptId,oldArchiveProof,token])
requireThat(claimed.rows.length===1&&claimed.rows[0].receipt_id===receiptId)
const {connectionId,operationId,planId,key,archived,projection,previousBytes}=await prepare(sql,target)
const tombstone = await sql.query(`INSERT INTO public.mcp_operation_replay_tombstones
(operation_id,plan_id,connection_id,idempotency_digest,action_id,target_server_ids,target_global)
VALUES($1,$2,$3,$4,'servers.poll',$5::bigint[],true) RETURNING operation_id`,
[operationId,planId,connectionId,operationIdempotencyDigest(connectionId,key),projection.history.affectedServerIds])
requireThat(tombstone.rows.length===1 && tombstone.rows[0].operation_id===operationId)
await sql.query("SELECT set_config('mm.poll_retire_receipt',$1,true),set_config('mm.poll_retire_claim',$2,true)",[receiptId,token])
const updated = await sql.query(`UPDATE public.server_poll_archives SET version=2,source_digest=$3,source_sha256=$4,
decoded_bytes=$5,compressed_bytes=$6,ciphertext=$7,ciphertext_bytes=$8,archive_proof=$9
WHERE receipt_id=$1 AND version=1 AND archive_proof=$2 RETURNING receipt_id`,
[receiptId,oldArchiveProof,archived.source_digest,archived.source_sha256,archived.decoded_bytes,archived.compressed_bytes,
archived.ciphertext,archived.ciphertext_bytes,archived.archive_proof])
requireThat(updated.rows.length===1 && updated.rows[0].receipt_id===receiptId)
const charged = await sql.query(`UPDATE public.server_poll_storage_admissions SET archive_bytes=$2
WHERE receipt_id=$1 AND storage_state='compacted' AND archive_bytes=$3 RETURNING receipt_id`,
[receiptId,archived.ciphertext_bytes,previousBytes])
requireThat(charged.rows.length===1 && charged.rows[0].receipt_id===receiptId)
const readback = (await sql.query(SERVER_POLL_ARCHIVE_READ_SQL,[receiptId])).rows
requireThat(readback.length===1 && readback[0].archive_proof===archived.archive_proof
&& fingerprint(await openServerPollRetiredArchive(readback[0]))===fingerprint(projection))
const step = await sql.query("DELETE FROM public.mcp_operation_steps WHERE operation_id=$1 AND step_index=0 RETURNING operation_id",[operationId])
requireThat(step.rows.length===1 && step.rows[0].operation_id===operationId)
const operation = await sql.query("DELETE FROM public.mcp_operations WHERE id=$1 AND connection_id=$2 AND plan_id=$3 RETURNING id",
[operationId,connectionId,planId])
requireThat(operation.rows.length===1 && operation.rows[0].id===operationId)
const plan = await sql.query("DELETE FROM public.mcp_plans WHERE id=$1 AND connection_id=$2 RETURNING id",[planId,connectionId])
requireThat(plan.rows.length===1 && plan.rows[0].id===planId)
await assertServerPollArchiveAccounting(sql)
return { retired:true as const, automaticReplay:false as const }
},sql=>admitPollStorageWriter(sql,deadlineAt))
}
/** Exact internal entry used by owned fixtures. It cannot bypass the claim. */
export function retireCompletedServerPollOperation(raw:Target) {
const target=request.parse(raw)
return sharedDatabaseGate.request(()=>retire(target,Date.now()+20000))
}
let cursor:{finishedAt:string;receiptId:string}|undefined
export function retireCompletedServerPollOperations() {
return sharedDatabaseGate.request(async()=>{
const deadlineAt=Date.now()+20000
// A live POLL may hold both locks. Skip before requesting its storage key.
const rows=await runPollStorageTransaction<Record<string,unknown>[]|null>(deps,{deadlineAt},async()=>{},async sql=>(await sql.query(CANDIDATES,
[cursor?.finishedAt??null,cursor?.receiptId??null])).rows,async sql=>{
if(!await tryAdmitPollStorageMaintenanceWriter(sql,"transaction"))return {deferred:null}
})
if(rows===null)return {scanned:0,retired:0,automaticReplay:false as const}
requireThat(rows.length<=8)
let retired=0,scanned=0
for(const row of rows){
if(Date.now()>=deadlineAt-2000||retired>=2)break
const target=request.parse({receiptId:row.receipt_id,oldArchiveProof:row.archive_proof})
requireThat(typeof row.finished_at==="string"&&Number.isFinite(Date.parse(row.finished_at)))
cursor={finishedAt:row.finished_at,receiptId:target.receiptId}
scanned++
try{await retire(target,deadlineAt);retired++}
catch(error){if(!(error instanceof PollStorageRefused))throw error}
}
// Keep the cursor for any unvisited rows. On exhaustion the next tick
// starts at the beginning, revisiting formerly held or busy candidates.
if(scanned===rows.length&&rows.length<8)cursor=undefined
return {scanned,retired,automaticReplay:false as const}
})
}