Files
MikrotikManager/backend/src/services/server-poll-execution.ts
T

194 lines
21 KiB
TypeScript

import { capturePollError, carryCapturedPollError, recordPollExecutionError, type PollExecutionProgress } from "./server-poll-error-provenance.js"
/** Atomic maintenance is available only through an explicit private executor capability. */
import {groupAtomicMaintenance,publicationLayout,projectAtomicMaintenance,type AtomicMaintenanceGroup,type AtomicMaintenanceAck} from "./server-poll-maintenance-groups.js"
import { SERVER_POLL_WALL_BUDGET_MS } from "./server-poll-limits.js"
import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js"
import { SERVER_POLL_NATIVE_READS, ServerPollProblem, pollProblem, validateServerPollContext, planServerPollMaintenance, planServerPollMaintenanceV2, serverPollSnapshot, projectServerPollEffect, type ServerPollContext,type ServerPollNativeRead,type ServerPollEffect,type ServerPollAck,type PollRow } from "./server-poll-execution-model.js"
export interface ServerPollGrant {global:boolean;serverIds:number[];authorityProof:string}
export interface ServerPollRequest {serverId:number;receiptId:string}
export interface ServerPollPlan {version:1|2|3|4;approvalProof?:string;maintenanceGroupSize?:number;requiresHuman:true;scope:"global";request:ServerPollRequest;around:string;beforeProof:string;authorityProof:string;bindingProof:string;maintenance:ServerPollEffect[];affectedServerIds:number[];mutationCost:number}
export interface ServerPollExecutionStart {fullProof:string;approvalProof:string}
export interface ServerPollReceipt {receiptId:string;configurationVerified:boolean;complete:boolean;maintenanceComplete:boolean;nativeReads:number;confirmedPublications:number;completedPublications:number;afterProof:string;observation:"online"|"offline";snapshot:PollRow}
export class ServerPollExecutionError extends ServerPollProblem {
constructor(code:"MUTATION_OUTCOME_UNKNOWN"|"VERIFICATION_FAILED",readonly phase:string,readonly nativeReads:number,readonly confirmedPublications:number,readonly completedPublications:number){super(code,"Server poll завершён не полностью; повтор без новой сверки запрещён")}
}
export interface ServerPollDependencies {
/** Private opt-in candidate; never derived from an external request. */
maintenanceGroup?:(group:AtomicMaintenanceGroup,options:{manifest:ServerPollEffect[];receiptId:string;stage:number;deadlineAt:number;beforeDispatch:()=>Promise<void>;verifyBeforeCommit:(observe:(context:ServerPollContext)=>AtomicMaintenanceAck)=>Promise<AtomicMaintenanceAck>})=>Promise<AtomicMaintenanceAck>
/** Fixed private session coordinator; never obtained from request data. */
boundary?:ServerPollBoundaryCoordinator
now:()=>number
/** Checks owner/policy/generation/global+all-server admission/protections and
* cancellation. Must not read privileged sources before validating access. */
authorize:(request:ServerPollRequest,affectedServerIds:number[],signal?:AbortSignal)=>Promise<ServerPollGrant>
/** Bounded complete authoritative reader; every internal SQL read must invoke
* beforeRead. First read only bounded catalogue IDs and refuse any ID outside
* permittedServerIds BEFORE loading private server/settings/partition rows.
* No implicit empty catalog/fallback after a provider failure. */
readContext:(request:ServerPollRequest,beforeRead:()=>Promise<void>,permittedServerIds:readonly number[],options:{signal?:AbortSignal;deadlineAt:number})=>Promise<ServerPollContext>
/** A private, purpose-separated HMAC over canonical complete input. */
proof:(value:unknown)=>Promise<string>
/** Fixed authoritative reader capability; never supplied by plan JSON. */
approvalProof?:(context:ServerPollContext)=>Promise<string>
/** Future production implementation must own a cross-replica, DB-wide
* maintenance exclusion covering affected collectors/background writers and
* drain owned work. An in-memory mutex alone does not satisfy this contract. */
withGlobalLock:<T>(work:()=>Promise<T>,options:{signal?:AbortSignal;deadlineAt:number})=>Promise<T>
native:(read:ServerPollNativeRead,server:PollRow,options:{signal?:AbortSignal;deadlineAt:number;beforeDispatch:()=>Promise<void>})=>Promise<unknown>
/** Persist intent before invoking beforeDispatch; every actual publication,
* including individual DDL and cache invalidation, must invoke it immediately
* at dispatch. Return only after ACK; unknown dispatch outcomes must throw. */
effect:(effect:ServerPollEffect,options:{receiptId:string;stage:number;deadlineAt:number;beforeDispatch:()=>Promise<void>})=>Promise<ServerPollAck>
intent:(value:{receiptId:string;stage:string;beforeProof:string;effectProof:string})=>Promise<void>
acknowledge:(value:{receiptId:string;stage:string;resultProof:string;outcome:"confirmed"|"read-failed"})=>Promise<void>
claim:(request:ServerPollRequest,bindingProof:string,start?:ServerPollExecutionStart)=>Promise<{state:"new"}|{state:"known";bindingProof:string;receipt:ServerPollReceipt}|{state:"unresolved";bindingProof:string}>
complete:(receipt:ServerPollReceipt,bindingProof:string)=>Promise<void>
}
const input=(request:ServerPollRequest)=>{if(!Number.isSafeInteger(request.serverId)||request.serverId<=0||!/^[a-f\d]{8}(?:-[a-f\d]{4}){3}-[a-f\d]{12}$/i.test(request.receiptId))pollProblem("INVALID_INPUT","Некорректный server poll request")}
function grantScope(grant:ServerPollGrant,ids:number[]){if(!grant.global||ids.some(id=>!grant.serverIds.includes(id)))pollProblem("PERMISSION_DENIED","Полный server poll требует global scope и все затронутые серверы")}
function safeSnapshotReceipt(row:PollRow,privateSources:unknown):PollRow{
const secrets=new Set<string>()
const visit=(value:unknown,key="")=>{if(typeof value==="string"&&value.length&&/password|secret|private.?key|token|authorization/i.test(key))secrets.add(value);else if(Array.isArray(value))for(const item of value)visit(item,key);else if(value&&typeof value==="object")for(const [name,item]of Object.entries(value))visit(item,name)}
visit(privateSources)
return Object.fromEntries(Object.entries(row).filter(([key])=>["id","serverId","polledAt","status","latencyMs","rosVersion","boardName","uptime","cpuLoad","freeMemory","totalMemory","identityName"].includes(key)).map(([key,value])=>[key,typeof value==="string"?[...secrets].reduce((text,secret)=>text.replaceAll(secret,"[redacted]"),value):value]))
}
/** Inert injectable foundation ONLY. No production reader, SQL, native client,
* partition executor, route or MCP registration is wired by this module.
* Maintenance is awaited under the same authority instead of escaping into a
* detached background task; the snapshot alone is never a full-run receipt. */
export function createServerPollExecution(deps:ServerPollDependencies){
const capture=async(request:ServerPollRequest,ids:number[],signal?:AbortSignal,deadline=Infinity,forApproval=false)=>{
const required=[...new Set([request.serverId,...ids])],initial=await deps.authorize(request,required,signal)
grantScope(initial,required)
if(typeof initial.authorityProof!=="string"||!/^[a-f0-9]{64}$/.test(initial.authorityProof))pollProblem("PERMISSION_DENIED","Не подтверждён private authority proof server poll")
const local=()=>{signal?.throwIfAborted();if(deps.now()>=deadline)pollProblem("ABORTED","Истёк deadline server poll")}
const access=deps.boundary?.fence(required,initial.authorityProof,{signal,deadlineAt:deadline})??(async()=>{local();const grant=await deps.authorize(request,required,signal);grantScope(grant,required);if(grant.authorityProof!==initial.authorityProof)pollProblem("STATE_CONFLICT","Допуск poll изменился");local()})
local();const context=await deps.readContext(request,access,initial.serverIds,{signal,deadlineAt:deadline});local();validateServerPollContext(context,request.serverId)
const all=context.servers.map(row=>Number(row.id)).sort((a,b)=>a-b),allGrant=await deps.authorize(request,all,signal);grantScope(allGrant,all)
if(allGrant.authorityProof!==initial.authorityProof)pollProblem("STATE_CONFLICT","Допуск poll изменился после полного каталога")
local()
const approvalProof=forApproval?await deps.approvalProof?.(context):undefined
if(forApproval&&(typeof approvalProof!=="string"||!/^[a-f0-9]{64}$/.test(approvalProof)))pollProblem("STATE_UNAVAILABLE","Не подтверждена версия конфигурации server poll")
local()
return {context,ids:all,proof:await deps.proof(context),authorityProof:initial.authorityProof,approvalProof}
}
const prepare=async(request:ServerPollRequest,signal?:AbortSignal):Promise<ServerPollPlan>=>{
input(request);const around=new Date(deps.now()).toISOString(),before=await capture(request,[],signal,deps.now()+45000,!!deps.approvalProof),maintenance=planServerPollMaintenanceV2(before.context,around)
if(deps.maintenanceGroup&&!before.approvalProof)pollProblem("STATE_UNAVAILABLE","Atomic maintenance requires a current configuration approval proof")
const core={version:(deps.maintenanceGroup?4:before.approvalProof?3:2) as 2|3|4,...(deps.maintenanceGroup?{maintenanceGroupSize:8}:{}),...(before.approvalProof?{approvalProof:before.approvalProof}:{}),requiresHuman:true as const,scope:"global" as const,request:{...request},around,beforeProof:before.proof,authorityProof:before.authorityProof,maintenance,affectedServerIds:before.ids,mutationCost:maintenance.length+4+4}
return {...core,bindingProof:await deps.proof(core)}
}
const execute=async(plan:ServerPollPlan,signal?:AbortSignal):Promise<ServerPollReceipt>=>{
const executionDeadline=deps.now()+SERVER_POLL_WALL_BUDGET_MS
signal=AbortSignal.any([...(signal?[signal]:[]),AbortSignal.timeout(SERVER_POLL_WALL_BUDGET_MS)])
return deps.withGlobalLock(async()=>{
input(plan.request)
if(!Array.isArray(plan.maintenance)||plan.maintenance.length>2000||!Array.isArray(plan.affectedServerIds)||plan.affectedServerIds.length>1000)pollProblem("INVALID_INPUT","Превышен полный бюджет плана server poll")
const {bindingProof,...core}=plan
const epochPlan=plan.version===3||plan.version===4
if((plan.version!==1&&plan.version!==2&&plan.version!==3&&plan.version!==4)||(epochPlan?typeof plan.approvalProof!=="string"||!/^[a-f0-9]{64}$/.test(plan.approvalProof)||!deps.approvalProof:Object.hasOwn(plan,"approvalProof"))||plan.requiresHuman!==true||plan.scope!=="global"||bindingProof!==await deps.proof(core)||plan.mutationCost!==plan.maintenance.length+8)pollProblem("STATE_CONFLICT","Изменён полный план server poll")
publicationLayout(plan)
if(plan.version===4&&!deps.maintenanceGroup)pollProblem("STATE_UNAVAILABLE","Atomic maintenance executor is unavailable")
const executionAround=new Date(deps.now()).toISOString(),deadline=executionDeadline,read=(forApproval=false)=>capture(plan.request,plan.affectedServerIds,signal,deadline,forApproval)
let current=await read(epochPlan)
if(current.authorityProof!==plan.authorityProof)pollProblem("STATE_CONFLICT","Допуск server poll изменился после подготовки")
if(epochPlan&&(current.approvalProof!==plan.approvalProof||await deps.proof(current.ids)!==await deps.proof(plan.affectedServerIds)||await deps.proof(planServerPollMaintenanceV2(current.context,executionAround))!==await deps.proof(plan.maintenance)))pollProblem("STATE_CONFLICT","Изменилась конфигурация или окно обслуживания после подтверждения poll")
const claim=await deps.claim(plan.request,bindingProof,epochPlan?{fullProof:current.proof,approvalProof:current.approvalProof!}:undefined)
if(claim.state!=="new"){
if(claim.bindingProof!==bindingProof)pollProblem("STATE_CONFLICT","Receipt poll принадлежит другому плану")
if(claim.state==="known"){if(claim.receipt.receiptId!==plan.request.receiptId||!claim.receipt.complete||!claim.receipt.maintenanceComplete)pollProblem("STATE_CONFLICT","Повреждён сохранённый receipt poll");return {...claim.receipt,configurationVerified:claim.receipt.afterProof===current.proof}}
pollProblem("MUTATION_OUTCOME_UNKNOWN","Незавершённый server poll не повторяется автоматически")
}
if((!epochPlan&&current.proof!==plan.beforeProof)||await deps.proof(current.ids)!==await deps.proof(plan.affectedServerIds)||await deps.proof((plan.version===1?planServerPollMaintenance:planServerPollMaintenanceV2)(current.context,executionAround))!==await deps.proof(plan.maintenance))pollProblem("STATE_CONFLICT","Изменились server/settings/partition источники или maintenance window poll")
let nativeReads=0,confirmedPublications=0,completedPublications=0,dispatchStarted=false
let phase: PollExecutionProgress["phase"] = "native"
let boundary: PollExecutionProgress["boundary"] = "gate"
const access=async()=>{signal?.throwIfAborted();if(deps.now()>=deadline)pollProblem("ABORTED","Истёк deadline server poll");const grant=await deps.authorize(plan.request,plan.affectedServerIds,signal);grantScope(grant,plan.affectedServerIds);if(grant.authorityProof!==plan.authorityProof)pollProblem("STATE_CONFLICT","Допуск server poll изменился");signal?.throwIfAborted()}
const gate=async()=>{const fresh=await read();if(fresh.proof!==current.proof)pollProblem("STATE_CONFLICT","Изменился полный контекст server poll перед эффектом");await access()}
const nativeGate=async()=>{await gate()}
try{
const start=deps.now(),server=current.context.servers.find(row=>row.id===plan.request.serverId)!
// Native reads can overlap. Their durable journal writes share one storage lock.
let journalTail=Promise.resolve()
const writeNativeJournal=(write:()=>Promise<void>)=>{
journalTail=journalTail.then(async()=>{
await access()
await write()
})
return journalTail
}
const observations=await Promise.allSettled(SERVER_POLL_NATIVE_READS.map(async key=>{
await writeNativeJournal(async()=>deps.intent({receiptId:plan.request.receiptId,stage:"native."+key,beforeProof:current.proof,effectProof:await deps.proof({read:key,source:server})}));await nativeGate()
let dispatched=false
let result:unknown,readFailed=false,nativeDiagnostic:ReturnType<typeof capturePollError>|undefined
try{result=await deps.native(key,structuredClone(server),{signal,deadlineAt:deadline,beforeDispatch:async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повторный dispatch native poll запрещён");await nativeGate();if(dispatched)pollProblem("STATE_CONFLICT","Повторный dispatch native poll запрещён");dispatched=true;nativeReads++}})}catch(error){if(error instanceof ServerPollProblem||signal?.aborted)throw error;nativeDiagnostic=capturePollError(error);readFailed=true}
if(!dispatched)throw carryCapturedPollError(new ServerPollProblem("STATE_UNAVAILABLE","Native transport пропустил dispatch fence"),nativeDiagnostic,"native.fence")
await writeNativeJournal(async()=>{
if(!readFailed){let bytes:number;try{bytes=Buffer.byteLength(JSON.stringify(result))}catch{pollProblem("STATE_UNAVAILABLE","Некорректный native ответ poll")};if(bytes>1048576)pollProblem("STATE_UNAVAILABLE","Превышен native budget poll")}
await deps.acknowledge({receiptId:plan.request.receiptId,stage:"native."+key,resultProof:await deps.proof(readFailed?{unavailable:true}:result),outcome:readFailed?"read-failed":"confirmed"})
})
return readFailed?undefined:result
}))
const failed=observations.find(result=>result.status==="rejected");if(failed?.status==="rejected")throw failed.reason
await gate()
const values=Object.fromEntries(SERVER_POLL_NATIVE_READS.map((key,index)=>[key,(observations[index]as PromiseFulfilledResult<unknown>).value]))as Record<ServerPollNativeRead,unknown>
let snapshot:PollRow={serverId:plan.request.serverId,polledAt:executionAround,status:"offline",latencyMs:null}
if(Object.values(values).every(value=>value!==undefined))snapshot=serverPollSnapshot(plan.request.serverId,executionAround,values,Math.max(0,deps.now()-start))
let stage=0,inserted:PollRow|undefined
const publish=async(effect:ServerPollEffect|AtomicMaintenanceGroup)=>{
phase=effect.kind;boundary="gate";await gate()
boundary="journal-intent";await deps.intent({receiptId:plan.request.receiptId,stage:"publish."+stage,beforeProof:current.proof,effectProof:await deps.proof(effect)})
let sent=false
const atomic=effect.kind==="partition.atomic"
const project=(ack:ServerPollAck|AtomicMaintenanceAck)=>atomic?projectAtomicMaintenance(current.context,plan.maintenance,effect as AtomicMaintenanceGroup,ack as AtomicMaintenanceAck):projectServerPollEffect(current.context,effect as ServerPollEffect,ack as ServerPollAck)
const options={receiptId:plan.request.receiptId,stage,deadlineAt:deadline,beforeDispatch:async()=>{
boundary="dispatch-fence"
if(sent)pollProblem("STATE_CONFLICT","Повторный dispatch publication poll запрещён");await gate();if(sent)pollProblem("STATE_CONFLICT","Повторный dispatch publication poll запрещён")
sent=true;dispatchStarted=true;boundary="effect"
}}
let precommitStarted=false,verifiedBeforeCommit=false
boundary="dispatch-fence"
const ack:ServerPollAck|AtomicMaintenanceAck=atomic?await deps.maintenanceGroup!(structuredClone(effect as AtomicMaintenanceGroup),{...options,manifest:structuredClone(plan.maintenance),verifyBeforeCommit:async observe=>{
boundary="precommit"
if(precommitStarted||!sent)pollProblem("STATE_CONFLICT","Invalid atomic precommit verification boundary")
precommitStarted=true
const fresh=await read()
const value=structuredClone(observe(structuredClone(fresh.context)))
const expected=project(value)
if(await deps.proof(expected)!==fresh.proof)pollProblem("VERIFICATION_FAILED","Atomic full source changed before COMMIT")
await access();verifiedBeforeCommit=true;boundary="after-precommit";return value
}}):await deps.effect(structuredClone(effect as ServerPollEffect),options)
if(!sent)pollProblem("STATE_UNAVAILABLE","Publication пропустила dispatch fence")
if(atomic&&!verifiedBeforeCommit)pollProblem("STATE_UNAVAILABLE","Atomic executor skipped mandatory precommit full source verification")
confirmedPublications++
boundary="independent-readback"
const projected=project(ack),after=await read()
if(await deps.proof(projected)!==after.proof)throw new ServerPollExecutionError("VERIFICATION_FAILED",phase,nativeReads,confirmedPublications,completedPublications)
boundary="journal-ack"
await access();await deps.acknowledge({receiptId:plan.request.receiptId,stage:"publish."+stage,resultProof:after.proof,outcome:"confirmed"})
current=after;completedPublications++;stage++
if(effect.kind==="snapshot.insert")inserted=(ack as ServerPollAck).row
}
if(snapshot.status==="online"&&snapshot.identityName!==server.name){await publish({kind:"server.name",serverId:plan.request.serverId,name:String(snapshot.identityName),updatedAt:executionAround});await publish({kind:"flow-cache.invalidate",serverId:plan.request.serverId})}
await publish({kind:"snapshot.insert",snapshot})
if(!inserted)pollProblem("VERIFICATION_FAILED","Не подтверждён snapshot receipt")
const old=current.context.snapshots.filter(row=>row.serverId===plan.request.serverId&&String(row.polledAt)<String(inserted!.polledAt)&&(row.rawInterfaces!=null||row.rawIpAddresses!=null))
await publish({kind:"snapshot.raw.clear",serverId:plan.request.serverId,beforeAt:String(inserted.polledAt),ids:old.map(row=>row.id)})
if(plan.version===4){for(const group of groupAtomicMaintenance(plan.maintenance))await publish(group)}else for(const effect of plan.maintenance)await publish(effect)
phase="receipt";boundary="gate";await gate()
boundary="receipt-write"
const safeSnapshot=safeSnapshotReceipt(inserted,{context:current.context,observations:values})
const receipt:ServerPollReceipt={receiptId:plan.request.receiptId,configurationVerified:true,complete:true,maintenanceComplete:true,nativeReads,confirmedPublications,completedPublications,afterProof:current.proof,observation:snapshot.status==="online"?"online":"offline",snapshot:safeSnapshot}
await deps.complete(receipt,bindingProof);boundary="receipt-authority";await access();return receipt
}catch(error){
const progress = { phase, boundary, confirmedPublications, completedPublications }
if(error instanceof ServerPollExecutionError)throw recordPollExecutionError(error,error,progress)
if(dispatchStarted)throw recordPollExecutionError(new ServerPollExecutionError("MUTATION_OUTCOME_UNKNOWN",phase,nativeReads,confirmedPublications,completedPublications),error,progress)
throw error
}
},{signal,deadlineAt:executionDeadline})
}
return {prepare,execute}
}