import { capturePollError, carryCapturedPollError, recordPollExecutionError, type PollExecutionProgress } from "./server-poll-error-provenance.js" /** Atomic maintenance is available only through an explicit private executor capability. */ import {groupAtomicMaintenance,publicationLayout,projectAtomicMaintenance,type AtomicMaintenanceGroup,type AtomicMaintenanceAck} from "./server-poll-maintenance-groups.js" import { SERVER_POLL_WALL_BUDGET_MS } from "./server-poll-limits.js" import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js" import { SERVER_POLL_NATIVE_READS, ServerPollProblem, pollProblem, validateServerPollContext, planServerPollMaintenance, planServerPollMaintenanceV2, serverPollSnapshot, projectServerPollEffect, type ServerPollContext,type ServerPollNativeRead,type ServerPollEffect,type ServerPollAck,type PollRow } from "./server-poll-execution-model.js" export interface ServerPollGrant {global:boolean;serverIds:number[];authorityProof:string} export interface ServerPollRequest {serverId:number;receiptId:string} export interface ServerPollPlan {version:1|2|3|4;approvalProof?:string;maintenanceGroupSize?:number;requiresHuman:true;scope:"global";request:ServerPollRequest;around:string;beforeProof:string;authorityProof:string;bindingProof:string;maintenance:ServerPollEffect[];affectedServerIds:number[];mutationCost:number} export interface ServerPollExecutionStart {fullProof:string;approvalProof:string} export interface ServerPollReceipt {receiptId:string;configurationVerified:boolean;complete:boolean;maintenanceComplete:boolean;nativeReads:number;confirmedPublications:number;completedPublications:number;afterProof:string;observation:"online"|"offline";snapshot:PollRow} export class ServerPollExecutionError extends ServerPollProblem { constructor(code:"MUTATION_OUTCOME_UNKNOWN"|"VERIFICATION_FAILED",readonly phase:string,readonly nativeReads:number,readonly confirmedPublications:number,readonly completedPublications:number){super(code,"Server poll завершён не полностью; повтор без новой сверки запрещён")} } export interface ServerPollDependencies { /** Private opt-in candidate; never derived from an external request. */ maintenanceGroup?:(group:AtomicMaintenanceGroup,options:{manifest:ServerPollEffect[];receiptId:string;stage:number;deadlineAt:number;beforeDispatch:()=>Promise;verifyBeforeCommit:(observe:(context:ServerPollContext)=>AtomicMaintenanceAck)=>Promise})=>Promise /** Fixed private session coordinator; never obtained from request data. */ boundary?:ServerPollBoundaryCoordinator now:()=>number /** Checks owner/policy/generation/global+all-server admission/protections and * cancellation. Must not read privileged sources before validating access. */ authorize:(request:ServerPollRequest,affectedServerIds:number[],signal?:AbortSignal)=>Promise /** Bounded complete authoritative reader; every internal SQL read must invoke * beforeRead. First read only bounded catalogue IDs and refuse any ID outside * permittedServerIds BEFORE loading private server/settings/partition rows. * No implicit empty catalog/fallback after a provider failure. */ readContext:(request:ServerPollRequest,beforeRead:()=>Promise,permittedServerIds:readonly number[],options:{signal?:AbortSignal;deadlineAt:number})=>Promise /** A private, purpose-separated HMAC over canonical complete input. */ proof:(value:unknown)=>Promise /** Fixed authoritative reader capability; never supplied by plan JSON. */ approvalProof?:(context:ServerPollContext)=>Promise /** Future production implementation must own a cross-replica, DB-wide * maintenance exclusion covering affected collectors/background writers and * drain owned work. An in-memory mutex alone does not satisfy this contract. */ withGlobalLock:(work:()=>Promise,options:{signal?:AbortSignal;deadlineAt:number})=>Promise native:(read:ServerPollNativeRead,server:PollRow,options:{signal?:AbortSignal;deadlineAt:number;beforeDispatch:()=>Promise})=>Promise /** Persist intent before invoking beforeDispatch; every actual publication, * including individual DDL and cache invalidation, must invoke it immediately * at dispatch. Return only after ACK; unknown dispatch outcomes must throw. */ effect:(effect:ServerPollEffect,options:{receiptId:string;stage:number;deadlineAt:number;beforeDispatch:()=>Promise})=>Promise intent:(value:{receiptId:string;stage:string;beforeProof:string;effectProof:string})=>Promise acknowledge:(value:{receiptId:string;stage:string;resultProof:string;outcome:"confirmed"|"read-failed"})=>Promise claim:(request:ServerPollRequest,bindingProof:string,start?:ServerPollExecutionStart)=>Promise<{state:"new"}|{state:"known";bindingProof:string;receipt:ServerPollReceipt}|{state:"unresolved";bindingProof:string}> complete:(receipt:ServerPollReceipt,bindingProof:string)=>Promise } const input=(request:ServerPollRequest)=>{if(!Number.isSafeInteger(request.serverId)||request.serverId<=0||!/^[a-f\d]{8}(?:-[a-f\d]{4}){3}-[a-f\d]{12}$/i.test(request.receiptId))pollProblem("INVALID_INPUT","Некорректный server poll request")} function grantScope(grant:ServerPollGrant,ids:number[]){if(!grant.global||ids.some(id=>!grant.serverIds.includes(id)))pollProblem("PERMISSION_DENIED","Полный server poll требует global scope и все затронутые серверы")} function safeSnapshotReceipt(row:PollRow,privateSources:unknown):PollRow{ const secrets=new Set() const visit=(value:unknown,key="")=>{if(typeof value==="string"&&value.length&&/password|secret|private.?key|token|authorization/i.test(key))secrets.add(value);else if(Array.isArray(value))for(const item of value)visit(item,key);else if(value&&typeof value==="object")for(const [name,item]of Object.entries(value))visit(item,name)} visit(privateSources) return Object.fromEntries(Object.entries(row).filter(([key])=>["id","serverId","polledAt","status","latencyMs","rosVersion","boardName","uptime","cpuLoad","freeMemory","totalMemory","identityName"].includes(key)).map(([key,value])=>[key,typeof value==="string"?[...secrets].reduce((text,secret)=>text.replaceAll(secret,"[redacted]"),value):value])) } /** Inert injectable foundation ONLY. No production reader, SQL, native client, * partition executor, route or MCP registration is wired by this module. * Maintenance is awaited under the same authority instead of escaping into a * detached background task; the snapshot alone is never a full-run receipt. */ export function createServerPollExecution(deps:ServerPollDependencies){ const capture=async(request:ServerPollRequest,ids:number[],signal?:AbortSignal,deadline=Infinity,forApproval=false)=>{ const required=[...new Set([request.serverId,...ids])],initial=await deps.authorize(request,required,signal) grantScope(initial,required) if(typeof initial.authorityProof!=="string"||!/^[a-f0-9]{64}$/.test(initial.authorityProof))pollProblem("PERMISSION_DENIED","Не подтверждён private authority proof server poll") const local=()=>{signal?.throwIfAborted();if(deps.now()>=deadline)pollProblem("ABORTED","Истёк deadline server poll")} const access=deps.boundary?.fence(required,initial.authorityProof,{signal,deadlineAt:deadline})??(async()=>{local();const grant=await deps.authorize(request,required,signal);grantScope(grant,required);if(grant.authorityProof!==initial.authorityProof)pollProblem("STATE_CONFLICT","Допуск poll изменился");local()}) local();const context=await deps.readContext(request,access,initial.serverIds,{signal,deadlineAt:deadline});local();validateServerPollContext(context,request.serverId) const all=context.servers.map(row=>Number(row.id)).sort((a,b)=>a-b),allGrant=await deps.authorize(request,all,signal);grantScope(allGrant,all) if(allGrant.authorityProof!==initial.authorityProof)pollProblem("STATE_CONFLICT","Допуск poll изменился после полного каталога") local() const approvalProof=forApproval?await deps.approvalProof?.(context):undefined if(forApproval&&(typeof approvalProof!=="string"||!/^[a-f0-9]{64}$/.test(approvalProof)))pollProblem("STATE_UNAVAILABLE","Не подтверждена версия конфигурации server poll") local() return {context,ids:all,proof:await deps.proof(context),authorityProof:initial.authorityProof,approvalProof} } const prepare=async(request:ServerPollRequest,signal?:AbortSignal):Promise=>{ input(request);const around=new Date(deps.now()).toISOString(),before=await capture(request,[],signal,deps.now()+45000,!!deps.approvalProof),maintenance=planServerPollMaintenanceV2(before.context,around) if(deps.maintenanceGroup&&!before.approvalProof)pollProblem("STATE_UNAVAILABLE","Atomic maintenance requires a current configuration approval proof") const core={version:(deps.maintenanceGroup?4:before.approvalProof?3:2) as 2|3|4,...(deps.maintenanceGroup?{maintenanceGroupSize:8}:{}),...(before.approvalProof?{approvalProof:before.approvalProof}:{}),requiresHuman:true as const,scope:"global" as const,request:{...request},around,beforeProof:before.proof,authorityProof:before.authorityProof,maintenance,affectedServerIds:before.ids,mutationCost:maintenance.length+4+4} return {...core,bindingProof:await deps.proof(core)} } const execute=async(plan:ServerPollPlan,signal?:AbortSignal):Promise=>{ const executionDeadline=deps.now()+SERVER_POLL_WALL_BUDGET_MS signal=AbortSignal.any([...(signal?[signal]:[]),AbortSignal.timeout(SERVER_POLL_WALL_BUDGET_MS)]) return deps.withGlobalLock(async()=>{ input(plan.request) if(!Array.isArray(plan.maintenance)||plan.maintenance.length>2000||!Array.isArray(plan.affectedServerIds)||plan.affectedServerIds.length>1000)pollProblem("INVALID_INPUT","Превышен полный бюджет плана server poll") const {bindingProof,...core}=plan const epochPlan=plan.version===3||plan.version===4 if((plan.version!==1&&plan.version!==2&&plan.version!==3&&plan.version!==4)||(epochPlan?typeof plan.approvalProof!=="string"||!/^[a-f0-9]{64}$/.test(plan.approvalProof)||!deps.approvalProof:Object.hasOwn(plan,"approvalProof"))||plan.requiresHuman!==true||plan.scope!=="global"||bindingProof!==await deps.proof(core)||plan.mutationCost!==plan.maintenance.length+8)pollProblem("STATE_CONFLICT","Изменён полный план server poll") publicationLayout(plan) if(plan.version===4&&!deps.maintenanceGroup)pollProblem("STATE_UNAVAILABLE","Atomic maintenance executor is unavailable") const executionAround=new Date(deps.now()).toISOString(),deadline=executionDeadline,read=(forApproval=false)=>capture(plan.request,plan.affectedServerIds,signal,deadline,forApproval) let current=await read(epochPlan) if(current.authorityProof!==plan.authorityProof)pollProblem("STATE_CONFLICT","Допуск server poll изменился после подготовки") if(epochPlan&&(current.approvalProof!==plan.approvalProof||await deps.proof(current.ids)!==await deps.proof(plan.affectedServerIds)||await deps.proof(planServerPollMaintenanceV2(current.context,executionAround))!==await deps.proof(plan.maintenance)))pollProblem("STATE_CONFLICT","Изменилась конфигурация или окно обслуживания после подтверждения poll") const claim=await deps.claim(plan.request,bindingProof,epochPlan?{fullProof:current.proof,approvalProof:current.approvalProof!}:undefined) if(claim.state!=="new"){ if(claim.bindingProof!==bindingProof)pollProblem("STATE_CONFLICT","Receipt poll принадлежит другому плану") if(claim.state==="known"){if(claim.receipt.receiptId!==plan.request.receiptId||!claim.receipt.complete||!claim.receipt.maintenanceComplete)pollProblem("STATE_CONFLICT","Повреждён сохранённый receipt poll");return {...claim.receipt,configurationVerified:claim.receipt.afterProof===current.proof}} pollProblem("MUTATION_OUTCOME_UNKNOWN","Незавершённый server poll не повторяется автоматически") } if((!epochPlan&¤t.proof!==plan.beforeProof)||await deps.proof(current.ids)!==await deps.proof(plan.affectedServerIds)||await deps.proof((plan.version===1?planServerPollMaintenance:planServerPollMaintenanceV2)(current.context,executionAround))!==await deps.proof(plan.maintenance))pollProblem("STATE_CONFLICT","Изменились server/settings/partition источники или maintenance window poll") let nativeReads=0,confirmedPublications=0,completedPublications=0,dispatchStarted=false let phase: PollExecutionProgress["phase"] = "native" let boundary: PollExecutionProgress["boundary"] = "gate" const access=async()=>{signal?.throwIfAborted();if(deps.now()>=deadline)pollProblem("ABORTED","Истёк deadline server poll");const grant=await deps.authorize(plan.request,plan.affectedServerIds,signal);grantScope(grant,plan.affectedServerIds);if(grant.authorityProof!==plan.authorityProof)pollProblem("STATE_CONFLICT","Допуск server poll изменился");signal?.throwIfAborted()} const gate=async()=>{const fresh=await read();if(fresh.proof!==current.proof)pollProblem("STATE_CONFLICT","Изменился полный контекст server poll перед эффектом");await access()} const nativeGate=async()=>{await gate()} try{ const start=deps.now(),server=current.context.servers.find(row=>row.id===plan.request.serverId)! // Native reads can overlap. Their durable journal writes share one storage lock. let journalTail=Promise.resolve() const writeNativeJournal=(write:()=>Promise)=>{ journalTail=journalTail.then(async()=>{ await access() await write() }) return journalTail } const observations=await Promise.allSettled(SERVER_POLL_NATIVE_READS.map(async key=>{ await writeNativeJournal(async()=>deps.intent({receiptId:plan.request.receiptId,stage:"native."+key,beforeProof:current.proof,effectProof:await deps.proof({read:key,source:server})}));await nativeGate() let dispatched=false let result:unknown,readFailed=false,nativeDiagnostic:ReturnType|undefined try{result=await deps.native(key,structuredClone(server),{signal,deadlineAt:deadline,beforeDispatch:async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повторный dispatch native poll запрещён");await nativeGate();if(dispatched)pollProblem("STATE_CONFLICT","Повторный dispatch native poll запрещён");dispatched=true;nativeReads++}})}catch(error){if(error instanceof ServerPollProblem||signal?.aborted)throw error;nativeDiagnostic=capturePollError(error);readFailed=true} if(!dispatched)throw carryCapturedPollError(new ServerPollProblem("STATE_UNAVAILABLE","Native transport пропустил dispatch fence"),nativeDiagnostic,"native.fence") await writeNativeJournal(async()=>{ if(!readFailed){let bytes:number;try{bytes=Buffer.byteLength(JSON.stringify(result))}catch{pollProblem("STATE_UNAVAILABLE","Некорректный native ответ poll")};if(bytes>1048576)pollProblem("STATE_UNAVAILABLE","Превышен native budget poll")} await deps.acknowledge({receiptId:plan.request.receiptId,stage:"native."+key,resultProof:await deps.proof(readFailed?{unavailable:true}:result),outcome:readFailed?"read-failed":"confirmed"}) }) return readFailed?undefined:result })) const failed=observations.find(result=>result.status==="rejected");if(failed?.status==="rejected")throw failed.reason await gate() const values=Object.fromEntries(SERVER_POLL_NATIVE_READS.map((key,index)=>[key,(observations[index]as PromiseFulfilledResult).value]))as Record let snapshot:PollRow={serverId:plan.request.serverId,polledAt:executionAround,status:"offline",latencyMs:null} if(Object.values(values).every(value=>value!==undefined))snapshot=serverPollSnapshot(plan.request.serverId,executionAround,values,Math.max(0,deps.now()-start)) let stage=0,inserted:PollRow|undefined const publish=async(effect:ServerPollEffect|AtomicMaintenanceGroup)=>{ phase=effect.kind;boundary="gate";await gate() boundary="journal-intent";await deps.intent({receiptId:plan.request.receiptId,stage:"publish."+stage,beforeProof:current.proof,effectProof:await deps.proof(effect)}) let sent=false const atomic=effect.kind==="partition.atomic" const project=(ack:ServerPollAck|AtomicMaintenanceAck)=>atomic?projectAtomicMaintenance(current.context,plan.maintenance,effect as AtomicMaintenanceGroup,ack as AtomicMaintenanceAck):projectServerPollEffect(current.context,effect as ServerPollEffect,ack as ServerPollAck) const options={receiptId:plan.request.receiptId,stage,deadlineAt:deadline,beforeDispatch:async()=>{ boundary="dispatch-fence" if(sent)pollProblem("STATE_CONFLICT","Повторный dispatch publication poll запрещён");await gate();if(sent)pollProblem("STATE_CONFLICT","Повторный dispatch publication poll запрещён") sent=true;dispatchStarted=true;boundary="effect" }} let precommitStarted=false,verifiedBeforeCommit=false boundary="dispatch-fence" const ack:ServerPollAck|AtomicMaintenanceAck=atomic?await deps.maintenanceGroup!(structuredClone(effect as AtomicMaintenanceGroup),{...options,manifest:structuredClone(plan.maintenance),verifyBeforeCommit:async observe=>{ boundary="precommit" if(precommitStarted||!sent)pollProblem("STATE_CONFLICT","Invalid atomic precommit verification boundary") precommitStarted=true const fresh=await read() const value=structuredClone(observe(structuredClone(fresh.context))) const expected=project(value) if(await deps.proof(expected)!==fresh.proof)pollProblem("VERIFICATION_FAILED","Atomic full source changed before COMMIT") await access();verifiedBeforeCommit=true;boundary="after-precommit";return value }}):await deps.effect(structuredClone(effect as ServerPollEffect),options) if(!sent)pollProblem("STATE_UNAVAILABLE","Publication пропустила dispatch fence") if(atomic&&!verifiedBeforeCommit)pollProblem("STATE_UNAVAILABLE","Atomic executor skipped mandatory precommit full source verification") confirmedPublications++ boundary="independent-readback" const projected=project(ack),after=await read() if(await deps.proof(projected)!==after.proof)throw new ServerPollExecutionError("VERIFICATION_FAILED",phase,nativeReads,confirmedPublications,completedPublications) boundary="journal-ack" await access();await deps.acknowledge({receiptId:plan.request.receiptId,stage:"publish."+stage,resultProof:after.proof,outcome:"confirmed"}) current=after;completedPublications++;stage++ if(effect.kind==="snapshot.insert")inserted=(ack as ServerPollAck).row } if(snapshot.status==="online"&&snapshot.identityName!==server.name){await publish({kind:"server.name",serverId:plan.request.serverId,name:String(snapshot.identityName),updatedAt:executionAround});await publish({kind:"flow-cache.invalidate",serverId:plan.request.serverId})} await publish({kind:"snapshot.insert",snapshot}) if(!inserted)pollProblem("VERIFICATION_FAILED","Не подтверждён snapshot receipt") const old=current.context.snapshots.filter(row=>row.serverId===plan.request.serverId&&String(row.polledAt)row.id)}) if(plan.version===4){for(const group of groupAtomicMaintenance(plan.maintenance))await publish(group)}else for(const effect of plan.maintenance)await publish(effect) phase="receipt";boundary="gate";await gate() boundary="receipt-write" const safeSnapshot=safeSnapshotReceipt(inserted,{context:current.context,observations:values}) const receipt:ServerPollReceipt={receiptId:plan.request.receiptId,configurationVerified:true,complete:true,maintenanceComplete:true,nativeReads,confirmedPublications,completedPublications,afterProof:current.proof,observation:snapshot.status==="online"?"online":"offline",snapshot:safeSnapshot} await deps.complete(receipt,bindingProof);boundary="receipt-authority";await access();return receipt }catch(error){ const progress = { phase, boundary, confirmedPublications, completedPublications } if(error instanceof ServerPollExecutionError)throw recordPollExecutionError(error,error,progress) if(dispatchStarted)throw recordPollExecutionError(new ServerPollExecutionError("MUTATION_OUTCOME_UNKNOWN",phase,nativeReads,confirmedPublications,completedPublications),error,progress) throw error } },{signal,deadlineAt:executionDeadline}) } return {prepare,execute} }