fix(mcp): очередь архивов POLL и проверка прав опроса раз на этап

Опрос держит исключительную блокировку записи разделов на всё время
работы, поэтому воркер успевал архивировать только один источник между
опросами. Теперь воркер разбирает очередь, пока писатель свободен
(каждый архив — прежний цикл из двух транзакций, уступает ждущему
опросу).

Права опроса проверяются свежими в начале этапа, перед каждой
отправкой на RouterOS и перед COMMIT каждой пишущей транзакции данных
или DDL; журнал и чтения используют доказательство не старше 2 с.
Холодный опрос: проверок допуска ~3700 -> 32, операция 19,9 -> 3,2 с.

Тесты: разбор очереди, уступка ждущему опросу, отзыв прав при
удержанных чтениях RouterOS, граница числа проверок; исправлено
устаревшее ожидание двух соединений в варианте с потерей ACK.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
DenozordecandClaude Opus 5.5 committed 2026-11-20 02:54:37 +07:00
1 parent b9f950c233
commit bd48368929
10 files changed
+151 -19

No files matched your search

+2 -2
View File
@@ -7,7 +7,7 @@ import { randomUUID } from "node:crypto"
import { retainServerProbeStorage } from "../../services/server-probe-retention.js"
import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js"
import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js"
import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
import { drainCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js"
import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js"
import { pool } from "../../db/index.js"
import { sharedDatabaseGate } from "../../services/shared-database-gate.js"
@@ -165,7 +165,7 @@ async function tickAdmittedMcpWorkers() {
await recoverScopedRecoveryIntents()
await retainMcpData()
await retainServerProbeStorage()
await compactCompletedServerPollStorage()
await drainCompletedServerPollStorage()
await retireCompletedServerPollOperations()
await collectExpiredServerPollStorage()
await retireOrdinaryMcpOperations()
@@ -5,7 +5,7 @@ import type { ServerPollSqlConnection } from "./server-poll-production.js"
import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js"
import type { ProbeStorageOptions, ProbeStorageSql } from "./server-probe-journal-maintenance.js"
import { createProbeStorageConnectionConnector } from "./server-probe-storage-connection.js"
import { runPollStorageTransaction } from "./server-poll-storage.js"
import { freshPollAuthority, runPollStorageTransaction } from "./server-poll-storage.js"
export const SERVER_PARTITION_WRITER_LOCK=8723110
export interface ServerPartitionLeaseOptions {signal?:AbortSignal;deadlineAt:number;serverIds:number[]}
@@ -72,18 +72,19 @@ export function createServerPartitionMaintenanceLease(deps:ServerPartitionMainte
try{await deps.assertAuthority({...holder.options,serverIds:[...holder.options.serverIds]})}catch{fail()}
if(!holder.active)fail();checkOptions(options);checkOptions(holder.options)
}
const raw=async(holder:Holder,text:string,values:unknown[],options:ServerPartitionTransactionOptions,fence:()=>Promise<void>,scopeLive:()=>boolean=()=>true)=>{
const raw=async(holder:Holder,text:string,values:unknown[],options:ServerPartitionTransactionOptions,fence:()=>Promise<void>,scopeLive:()=>boolean=()=>true,durableCommit=false)=>{
const complete=deps.boundary?.combine([fence],holder.options.serverIds,holder.options)
const boundary=async()=>{
const check=async()=>{
local(holder,options);if(!scopeLive())fail()
if(complete)await complete()
else{await authority(holder,options);await fence();await authority(holder,options)}
local(holder,options);if(!scopeLive())fail()
}
await boundary()
// Only the COMMIT of a write transaction makes poll data or DDL durable.
await (durableCommit?freshPollAuthority(check):check())
let result!:Awaited<ReturnType<ServerPollSqlConnection["query"]>>
try{result=await holder.connection.query(text,values,{signal:options.signal,timeoutMs:Math.max(1,Math.min(5000,options.deadlineAt-now(),holder.options.deadlineAt-now()))})}catch{holder.destroy();fail(true)}
await boundary();return result
await check();return result
}
const enqueue=async<T>(owner:{tail:Promise<void>},work:()=>Promise<T>):Promise<T>=>{
const previous=owner.tail;let unlock!:()=>void;owner.tail=new Promise<void>(resolve=>{unlock=resolve});await previous
@@ -119,7 +120,7 @@ export function createServerPartitionMaintenanceLease(deps:ServerPartitionMainte
await scope.tail;scope.active=false
// Any failure after sending COMMIT immediately destroys the owner; there
// is no ROLLBACK/retry after the uncertain boundary.
await raw(holder,"COMMIT",[],options,fence);return result
await raw(holder,"COMMIT",[],options,fence,undefined,!readOnly);return result
}catch(error){holder.destroy();throw error}finally{scope.active=false}
}
const transaction=async<T>(readOnly:boolean,fence:()=>Promise<void>,options:ServerPartitionTransactionOptions,work:(query:ServerPartitionQuery)=>Promise<T>):Promise<T>=>{
@@ -31,10 +31,12 @@ let releaseNative!:()=>void,firstNative!:()=>void
const nativeGate=new Promise<void>(resolve=>{releaseNative=()=>{releasedNative=true;resolve()}})
const started=new Promise<void>(resolve=>{firstNative=resolve})
const paths:string[]=[]
let holdAll:{reached:()=>void;release:Promise<void>}|undefined
const listener=createServer(async(req,res)=>{
requests++;paths.push(req.url??"")
// Keep the lease observable without exhausting RouterOS's two HTTP sockets.
if(requests===1){firstNative();await nativeGate}
else if(holdAll){holdAll.reached();await holdAll.release}
res.setHeader("content-type","application/json")
const value=req.url==="/rest/system/identity"?{name:"poll-owned-renamed"}:req.url==="/rest/system/resource"?{version:"7.synthetic","board-name":"owned","uptime":"1h","cpu-load":"3","free-memory":"1024","total-memory":"2048"}:req.url==="/rest/interface"?[{name:"owned-interface",type:"ether"}]:req.url==="/rest/ip/address"?[{address:"192.0.2.1/24",interface:"owned-interface"}]:null
res.statusCode=value===null?404:200;res.end(JSON.stringify(value))
@@ -101,6 +103,7 @@ try{
assert.equal(requests,0)
assert.equal(new Set(plan.maintenance.map(effect=>"parent"in effect?effect.parent:"")).size,PARTITION_SPECS.length)
assert(plan.maintenance.some(effect=>effect.kind==="partition.drop"&&effect.name===expired))
const readsBeforeRun=authorityReads
const pending=session.execute(plan);void pending.catch(()=>{})
await Promise.race([started,pending.then(()=>assert.fail("Native request must occur before completion"))])
const {rows:[holder]}=await pool.query(`SELECT pid FROM pg_catalog.pg_locks WHERE locktype='advisory' AND classid=0
@@ -155,6 +158,9 @@ try{
assert.equal((await pool.query("SELECT count(*)::int AS n FROM pg_locks l JOIN pg_class c ON c.oid=l.relation WHERE l.pid=$1 AND l.locktype='relation' AND (c.relname='traffic_samples' OR c.relname LIKE 'traffic_samples_w_%')",[writerPid])).rows[0].n,0)
}finally{controller.abort();await assert.rejects(blocked,(error:unknown)=>error instanceof PartitionWriteError&&!error.outcomeUnknown);releaseNative()}
const receipt=await pending
const runAuthorityReads=authorityReads-readsBeforeRun
// 17108 complete authority reads before stage coalescing (2026-11-19), 152 after.
assert(runAuthorityReads<=300,"Stage-coalesced authority regressed: "+runAuthorityReads)
await tickMcpWorkers()
assert.equal(await hasExpiredLease(),false,"The next tick collects the expired lease after POLL releases the writer lock")
assert.equal((await compactCompletedServerPollStorage()).compacted,0)
@@ -237,6 +243,36 @@ try{
await assert.rejects(session.execute({...plan,bindingProof:"0".repeat(64)}));assert.equal(requests,4)
await withPartitionWrite(async owner=>{await owner.client.query("INSERT INTO traffic_samples(server_id,interface_name,sampled_at,rx_bytes) VALUES($1,'after-full-poll',now(),1)",[serverId])})
revoked=true;await assert.rejects(session.execute(plan));assert.equal(requests,4)
// Stage-coalesced authority contract: rights are fresh before every native
// dispatch and before the COMMIT of every data/DDL write, so a revocation while
// native reads are held allows no later dispatch, effect or receipt. Journal
// bookkeeping of the reads may still land on a grant younger than 2 s.
revoked=false
const revokeRequest={serverId:serverId!,receiptId:randomUUID()}
const revokeSession=createDurableServerPollSession({request:revokeRequest,
identity:{kind:"rest",authentication:"local",ownerId,executionId:randomUUID(),requestId:randomUUID(),issuedAt,expiresAt},
authorize:async(request,ids,signal)=>{assert.deepEqual(request,revokeRequest);return originalAuthority({serverId:serverId!,receiptId},ids,signal)}})
const revokePlan=await revokeSession.prepare()
let reached!:()=>void,releaseHeld!:()=>void
const firstHeld=new Promise<void>(resolve=>{reached=resolve})
holdAll={reached,release:new Promise<void>(resolve=>{releaseHeld=resolve})}
const revokedRun=revokeSession.execute(revokePlan);void revokedRun.catch(()=>{})
await firstHeld;await new Promise(resolve=>setTimeout(resolve,500))
const journalOf=async()=>(await pool.query(`SELECT (SELECT count(*)::int FROM server_poll_intents WHERE receipt_id=$1) AS intents,
(SELECT count(*)::int FROM server_poll_intents WHERE receipt_id=$1 AND stage LIKE 'native.%') AS native_intents,
(SELECT count(*)::int FROM server_poll_acks WHERE receipt_id=$1 AND stage NOT LIKE 'native.%') AS effect_acks,
(SELECT count(*)::int FROM server_poll_receipts WHERE receipt_id=$1) AS receipts`,[revokeRequest.receiptId])).rows[0]
const atRevocation=await journalOf()
revoked=true;releaseHeld()
await assert.rejects(revokedRun);holdAll=undefined
const afterRevocation=await journalOf()
assert.equal(atRevocation.receipts,0);assert.equal(afterRevocation.receipts,0,"No receipt after the revocation")
assert.equal(afterRevocation.effect_acks,atRevocation.effect_acks,"No maintenance or publication commits after the revocation")
// HTTP arrival can trail an authorized dispatch (two RouterOS sockets), so the
// proof is the intent: every request that reached RouterOS was journaled first.
assert(requests-4<=atRevocation.native_intents,"Every native request was authorized and journaled before the revocation")
console.log("Full durable poll authority:",JSON.stringify({runAuthorityReads,nativeArrived:requests-4,journalAtRevocation:atRevocation}))
console.log("Full durable poll actual SQL/HTTP acceptance:",JSON.stringify({fourNativeGets:requests,maintenanceEffects:plan.maintenance.length,parents:PARTITION_SPECS.length,completedPublications:receipt.completedPublications,sameBackendLeaseAndTransactions:true,guardedSameHolderJournal:true,separateDurableJournalCommits:true,independentQuarantineAfterRealLostCommitAck:true,ordinaryWriterBlockedBeforeRelationSql:true,backgroundTickDeferredAndResumed:true,realJournal:true,repeatedApplyNoNewNative:true,outerInputMutationCannotReplaceCapturedIdentityOrAuthority:true,changedPlanBindingRefused:true,lateRevocationRefused:true,actualHttpAuthenticationMiddleware:false,productionRegistration:false,externalNetwork:false,authorityReads}))
}finally{
if(!releasedNative)releaseNative()
@@ -7,6 +7,8 @@ import { pollProblem } from "./server-poll-execution-model.js"
import type { ServerPollDependencies, ServerPollPlan, ServerPollRequest } from "./server-poll-execution.js"
import { createHolderBoundServerPollStorageJournal, createServerPollStorageJournal } from "./server-poll-storage-journal.js"
import { createServerPollBoundaryCoordinator } from "./server-poll-boundary.js"
import { freshPollAuthorityScope } from "./server-poll-storage.js"
import { coalesceAuthorization } from "../modules/mcp/authorization-stage.js"
export interface DurableServerPollSessionInput {
request:ServerPollRequest
@@ -36,12 +38,31 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean
signer??=security.privateFingerprinter("server-poll-production-v1")
return (await signer)(security.fingerprint(value))
}
// Rights are proven fresh at every stage start (prepare/execute/read reset the
// stages), before every RouterOS dispatch and right before the COMMIT of every
// write that changes application data or DDL; one proof serves a whole marked
// boundary. Journal and read boundaries reuse a grant whose check began < 2 s ago;
// a refusal is never reused.
type Grant=Awaited<ReturnType<ServerPollDependencies["authorize"]>>
type Stage={signal?:AbortSignal;proof:ReturnType<typeof coalesceAuthorization<Grant>>}
let stages=new Map<string,Stage>()
const grantFor=(required:number[],signal?:AbortSignal):Promise<Grant>=>{
const key=[...required].sort((a,b)=>a-b).join(",")
let stage=stages.get(key)
if(!stage){const created={signal} as Stage;created.proof=coalesceAuthorization(()=>authorityProvider({...request},[...required],created.signal));stages.set(key,stage=created)}
stage.signal=signal
const boundary=freshPollAuthorityScope()
if(!boundary)return stage.proof.coalesced()
let proven=boundary.get(key) as Promise<Grant>|undefined
if(!proven)boundary.set(key,proven=stage.proof.fresh())
return proven
}
const authorize:ServerPollDependencies["authorize"]=async(candidate,ids,signal)=>{
if(candidate.serverId!==request.serverId||candidate.receiptId!==request.receiptId)pollProblem("STATE_CONFLICT","Запрос не принадлежит durable poll session")
signal?.throwIfAborted()
if(Date.now()<identity.issuedAt||Date.now()>=identity.expiresAt)pollProblem("PERMISSION_DENIED","Исходная identity poll истекла")
const required=[...new Set([request.serverId,...ids])]
const grant=await authorityProvider({...request},required,signal)
const grant=await grantFor(required,signal)
if(!grant.global||required.some(id=>!grant.serverIds.includes(id))||!/^[a-f0-9]{64}$/.test(grant.authorityProof))pollProblem("PERMISSION_DENIED","Требуется полный живой global допуск poll")
signal?.throwIfAborted()
if(Date.now()>=identity.expiresAt)pollProblem("PERMISSION_DENIED","Исходная identity poll истекла")
@@ -63,7 +84,7 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean
// Preparation has no journal effects. These throwing boundaries make an
// accidental unbound execution impossible; they never invent an ACK/claim.
const unavailable=async():Promise<never>=>{throw new ServerPollJournalConflict()}
const prepare=async(signal?:AbortSignal)=>build({claim:unavailable,intent:unavailable,acknowledge:unavailable,complete:unavailable}).prepare(request,signal)
const prepare=async(signal?:AbortSignal)=>{stages=new Map();return build({claim:unavailable,intent:unavailable,acknowledge:unavailable,complete:unavailable}).prepare(request,signal)}
const journalFor=async(raw:ServerPollPlan,mode:"execute"|"read",signal?:AbortSignal)=>{
const plan=structuredClone(raw)
if(plan.request.serverId!==request.serverId||plan.request.receiptId!==request.receiptId)pollProblem("STATE_CONFLICT","План не принадлежит durable poll session")
@@ -83,8 +104,8 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean
})
return {journal,plan}
}
const execute=async(raw:ServerPollPlan,signal?:AbortSignal)=>{const {journal,plan}=await journalFor(raw,"execute",signal);return build(journal).execute(plan,signal)}
const readExisting=async(raw:ServerPollPlan,signal?:AbortSignal)=>{const {journal}=await journalFor(raw,"read",signal);return journal.readExisting()}
const execute=async(raw:ServerPollPlan,signal?:AbortSignal)=>{stages=new Map();const {journal,plan}=await journalFor(raw,"execute",signal);return build(journal).execute(plan,signal)}
const readExisting=async(raw:ServerPollPlan,signal?:AbortSignal)=>{stages=new Map();const {journal}=await journalFor(raw,"read",signal);return journal.readExisting()}
return {prepare,execute,readExisting}
}
/** Historical private assembly for the enrolled 0038 fixture; public adapters
@@ -15,7 +15,7 @@ const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
const { invokeMcpTool } = await import("../modules/mcp/runtime.js")
const { encryptSecret,privateSecretFingerprint,fingerprint,redact } = await import("../modules/mcp/security.js")
const { encodeServerPollJournalAnchor } = await import("./server-poll-journal-anchor.js")
const { compactCompletedServerPollStorage,compactCompletedServerPollStorageForTests,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js")
const { compactCompletedServerPollStorage,compactCompletedServerPollStorageForTests,drainCompletedServerPollStorage,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js")
const { openServerPollTerminalArchive } = await import("./server-poll-terminal-archive.js")
const { retireCompletedServerPollOperation, retireCompletedServerPollOperations } = await import("./server-poll-operation-retirement.js")
const { PARTITION_WRITER_ADVISORY_KEY } = await import("./partition-write-transaction.js")
@@ -330,6 +330,39 @@ try{
assert.deepEqual(await compactCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false})
console.log("Owned PostgreSQL: guarded compaction and mixed V1/V2 GC, physical claim/publish/refund ACK losses, retained history, and actual worker skips held/spent rows")
// A backlog of naturally expired sources leaves in one drain and in one worker tick.
const drainOwner=await historical(2,Number(server.id))
const more=async()=>[await historical(2,Number(server.id),drainOwner),await historical(2,Number(server.id),drainOwner)]
const archivedOf=async(sources:{receiptId:string}[])=>(await pool.query("SELECT count(*)::int AS n FROM server_poll_archives WHERE receipt_id=ANY($1::uuid[])",[sources.map(source=>source.receiptId)])).rows[0].n
const drained=[drainOwner,...await more()]
assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:3,compacted:3,skipped:0,automaticReplay:false})
assert.equal(await archivedOf(drained),3)
const ticked=[await historical(2,Number(server.id),drainOwner),...await more()]
await tickMcpWorkers()
assert.equal(await archivedOf(ticked),3)
assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false})
assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true)
// A POLL already waiting for its exclusive lease stops the drain: every cycle
// re-admits the writer with a try-lock that cannot queue ahead of that waiter.
const queued=[await historical(2,Number(server.id),drainOwner),await historical(2,Number(server.id),drainOwner)]
const sharedHolder=await pool.connect(),exclusiveWaiter=await pool.connect()
try{
await sharedHolder.query("SELECT pg_advisory_lock_shared($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY])
const waiterPid=(await exclusiveWaiter.query("SELECT pg_backend_pid() AS pid")).rows[0].pid
const exclusive=exclusiveWaiter.query("SELECT pg_advisory_lock($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY])
for(let attempt=0;!(await pool.query("SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory' AND mode='ExclusiveLock' AND NOT granted",[waiterPid])).rows.length;attempt++){
assert(attempt<200,"exclusive request did not start waiting");await new Promise(resolve=>setTimeout(resolve,25))
}
assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false})
assert.equal(await archivedOf(queued),0)
await sharedHolder.query("SELECT pg_advisory_unlock_shared($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY])
await exclusive
await exclusiveWaiter.query("SELECT pg_advisory_unlock($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY])
}finally{sharedHolder.release(true);exclusiveWaiter.release(true)}
assert.equal((await drainCompletedServerPollStorage()).compacted,2)
assert.equal(await archivedOf(queued),2)
console.log("Owned PostgreSQL: worker drain archives a three-source backlog in one call and in one tick, and yields to a waiting exclusive POLL lease")
if (process.env.MMAPP_OWN_SKIP_POLL_CAPACITY === "true") {
console.log("Focused POLL retirement: 336-archive capacity scenario skipped")
} else {
@@ -7,6 +7,7 @@ import type { QueryConfig } from "pg"
import type { ServerPartitionTransaction } from "./server-partition-maintenance-lease.js"
import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js"
import { PARTITION_SPECS } from "../db/partitions.js"
import { freshPollAuthority } from "./server-poll-storage.js"
import { createServerPollExecution,type ServerPollDependencies,type ServerPollGrant,type ServerPollRequest } from "./server-poll-execution.js"
import { ServerPollProblem,pollProblem,validateServerPollContext,planServerPollMaintenance,type ServerPollContext,type PollRow,type ServerPollAck } from "./server-poll-execution-model.js"
@@ -178,7 +179,7 @@ export function createServerPollProductionDependencies(input:ServerPollProductio
const transaction=async<T>(readOnly:boolean,fence:()=>Promise<void>,options:{signal?:AbortSignal;deadlineAt:number},work:(query:(sql:string,values?:unknown[])=>Promise<{rows:Record<string,unknown>[];rowCount?:number|null}>)=>Promise<T>):Promise<T>=>{
if(io.transaction)return io.transaction(readOnly,fence,options,work)
await fence();let connection:ServerPollSqlConnection;try{connection=await io.connect(options)}catch(error){throw carryPollError(new ServerPollProblem("STATE_UNAVAILABLE","SQL connection server poll недоступен"),error,"poll.connect")};let unsafe=false,committed=false
const query=async(sql:string,values:unknown[]=[])=>{await fence();const timeoutMs=Math.max(1,Math.min(10000,options.deadlineAt-now()));let result:{rows:Record<string,unknown>[];rowCount?:number|null};try{result=await connection.query(sql,values,{signal:options.signal,timeoutMs})}catch(error){unsafe=true;throw carryPollError(Error("SQL boundary unavailable"),error,"poll.sql")};await fence();return result}
const query=async(sql:string,values:unknown[]=[])=>{await (sql==="COMMIT"&&!readOnly?freshPollAuthority(fence):fence());const timeoutMs=Math.max(1,Math.min(10000,options.deadlineAt-now()));let result:{rows:Record<string,unknown>[];rowCount?:number|null};try{result=await connection.query(sql,values,{signal:options.signal,timeoutMs})}catch(error){unsafe=true;throw carryPollError(Error("SQL boundary unavailable"),error,"poll.sql")};await fence();return result}
try{await query(readOnly?"BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY":"BEGIN");await query("SET LOCAL TIME ZONE 'UTC'");await query(`SET LOCAL statement_timeout='${Math.max(1,Math.min(10000,options.deadlineAt-now()))}ms'`);const result=await work(query);await query("COMMIT");committed=true;return result}
finally{if(!committed)unsafe=true;connection.release(unsafe)}
}
@@ -276,7 +277,7 @@ export function createServerPollProductionDependencies(input:ServerPollProductio
},{...options,serverIds:ids}))
})
},
native:async(read,server,options)=>{const {lease,fence}=requireLease(options);let dispatched=false;await fence();return io.native(read,server,{...options,beforeDispatch:async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повтор native dispatch poll запрещён");await fence();await options.beforeDispatch();await fence();const current=lease.context?.servers.find(row=>row.id===input.request.serverId);if(server.id!==input.request.serverId||!current||await proof(current)!==await proof(server)||dispatched)pollProblem("STATE_CONFLICT","Native endpoint/dispatch не совпадает с private source poll");dispatched=true}})},
native:async(read,server,options)=>{const {lease,fence}=requireLease(options);let dispatched=false;await fence();return io.native(read,server,{...options,beforeDispatch:()=>freshPollAuthority(async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повтор native dispatch poll запрещён");await fence();await options.beforeDispatch();await fence();const current=lease.context?.servers.find(row=>row.id===input.request.serverId);if(server.id!==input.request.serverId||!current||await proof(current)!==await proof(server)||dispatched)pollProblem("STATE_CONFLICT","Native endpoint/dispatch не совпадает с private source poll");dispatched=true})})},
effect,intent:input.journal.intent,acknowledge:input.journal.acknowledge,claim:input.journal.claim,complete:input.journal.complete,
}
}
@@ -2,6 +2,7 @@ import { carryPollError } from "./server-poll-error-provenance.js"
/** Private opt-in poll storage. No import/constructor performs I/O or enables
* a route. The separately reviewed storage schema must already be installed. */
import { randomUUID } from "node:crypto"
import { AsyncLocalStorage } from "node:async_hooks"
import { z } from "zod"
import { connectProbeStorage } from "./server-probe-storage-connection.js"
import type { ProbeStorageConnection, ProbeStorageOptions, ProbeStorageSql } from "./server-probe-journal-maintenance.js"
@@ -30,6 +31,13 @@ export interface PollStorageDependencies {
}
const production:PollStorageDependencies={connect:connectProbeStorage,now:Date.now}
const fail=():never=>{throw new PollStorageRefused()}
/** Marks the boundaries that need fresh rights: every RouterOS dispatch and the
* COMMIT of every write transaction that changes application data or DDL. Inside one
* marked boundary each scope is proven once; journal and read boundaries may reuse
* a recent grant (MCP journal rows also re-check the connection in their own SQL). */
const commitAuthority=new AsyncLocalStorage<Map<string,Promise<unknown>>>()
export const freshPollAuthority=<T>(run:()=>Promise<T>):Promise<T>=>commitAuthority.run(new Map(),run)
export const freshPollAuthorityScope=()=>commitAuthority.getStore()
const lock="SELECT pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0))"
/** The writer fence precedes POLL's storage, operation and row locks. Restore
@@ -278,7 +278,8 @@ async function main() {
assert.equal(error.message.includes("PRIVATE_OWNED_ARCHIVE_COMMIT_ACK"), false)
return true
})
assert.equal(commits, 2); assert.deepEqual(releases, [false, true])
// One maintenance-writer backend carries both transactions; the unknown COMMIT destroys it.
assert.equal(commits, 2); assert.deepEqual(releases, [true])
assert.equal(commands.includes("ROLLBACK"), false, "Unknown COMMIT ACK cannot trigger rollback or replay")
commitAckLossAccepted = true
groups.push("after natural original expiry, real claim COMMIT ACK is known and real archive COMMIT succeeds but loses caller ACK; generic unknown destroys backend without rollback/retry")
@@ -1,6 +1,7 @@
import assert from "node:assert/strict"
import { readFileSync } from "node:fs"
import { randomUUID } from "node:crypto"
import { AsyncLocalStorage } from "node:async_hooks"
import { test } from "node:test"
import { runInNewContext } from "node:vm"
import ts from "typescript"
@@ -21,7 +22,8 @@ const ACCOUNTING = "SELECT component_accounting"
type Row = Record<string, unknown>
type State = { claim: string | null; archive: Row | null; audits: Row[] }
type Command = { tx: number; text: string; values: unknown[] }
type Api = { compactCompletedServerPollStorageForTests(deps: PollStorageDependencies): Promise<{ scanned: number; compacted: number; skipped: number; automaticReplay: false }> }
type Result = Promise<{ scanned: number; compacted: number; skipped: number; automaticReplay: false }>
type Api = { compactCompletedServerPollStorageForTests(deps: PollStorageDependencies): Result; drainCompletedServerPollStorageForTests(deps: PollStorageDependencies): Result }
function load(code: string, modules: Record<string, unknown>) {
const exports: Record<string, unknown> = {}
runInNewContext(code, { exports, require(name: string) { assert(Object.hasOwn(modules, name), "Unapproved production import: " + name); return modules[name] },
@@ -41,7 +43,7 @@ function fixture(quarantined = false) {
const forbiddenConnector = async (): Promise<never> => { throw Error("Production connector must never run") }
const accounting = async (sql: { query(text: string): Promise<unknown> }) => { accountCalls++; await sql.query(ACCOUNTING); if (accountingFails) throw new model.PollStorageRefused() }
const storage = load(storageCode, {
"node:crypto": { randomUUID }, zod: { z }, "./server-probe-storage-connection.js": { connectProbeStorage: forbiddenConnector },
"node:crypto": { randomUUID }, "node:async_hooks": { AsyncLocalStorage }, zod: { z }, "./server-probe-storage-connection.js": { connectProbeStorage: forbiddenConnector },
"./server-poll-storage-model.js": model, "./server-poll-archive-accounting.js": { assertServerPollArchiveAccounting: accounting },
"./server-poll-error-provenance.js": errorProvenance,
"./partition-write-transaction.js": { admitPartitionWriter, PARTITION_WRITER_ADVISORY_KEY },
@@ -139,7 +141,7 @@ function fixture(quarantined = false) {
throw Error("Unapproved fixture SQL: " + text)
}, release(unsafe = false) { assert(!closed); closed = true; transaction = undefined; releases.push({ tx:owner, unsafe }) } }
} }
return { run: () => api.compactCompletedServerPollStorageForTests(deps), commands, releases, events, state: () => structuredClone(state),
return { run: () => api.compactCompletedServerPollStorageForTests(deps), drain: () => api.drainCompletedServerPollStorageForTests(deps), commands, releases, events, state: () => structuredClone(state),
counts: () => ({ connects, commits, createCalls, openCalls, accountCalls }), Unknown: storage.PollStorageUnknown as new () => Error,setWriter:(value:boolean|Error)=>{writer=value},
configure(options: { eligible?: boolean; sourceMissing?: boolean; accountingFails?: boolean; preflight?: typeof preflight; secondOversize?: boolean; lostCommit?: number; commitPersists?: boolean; badPublishedSize?: boolean; badPublishedText?: boolean; badPublishedId?: boolean; auditFails?: boolean; skippedPrefix?: typeof skippedPrefix; removeLastSlowReceipt?: boolean }) {
eligible = options.eligible ?? eligible; sourceMissing = options.sourceMissing ?? sourceMissing; accountingFails = options.accountingFails ?? accountingFails
@@ -265,6 +267,18 @@ test("phase2 size growth preserves spent full charge and subsequent candidate ex
assert(!f.commands.some(command => /^(?:DELETE|UPDATE server_poll_storage_usage)/.test(command.text)))
})
test("worker drain re-admits the writer for every cycle and stops on the first cycle without an archive", async () => {
const f = fixture()
assert.deepEqual({ ...await f.drain() }, { scanned: 1, compacted: 1, skipped: 0, automaticReplay: false })
const tries = f.commands.filter(command => command.text.includes("pg_try_advisory_lock_shared")), unlocks = f.commands.filter(command => command.text.includes("pg_advisory_unlock_shared"))
assert.equal(tries.length, 2); assert.equal(unlocks.length, 2); assert.equal(f.counts().connects, 2)
assert.deepEqual(f.releases, [{ tx: 1, unsafe: false }, { tx: 2, unsafe: false }])
assert.equal(claimed(f).length, 1); assert.equal(published(f).length, 1)
const busy = fixture(); busy.setWriter(false)
assert.deepEqual({ ...await busy.drain() }, { scanned: 0, compacted: 0, skipped: 0, automaticReplay: false })
assert.equal(busy.counts().connects, 1); assert.equal(claimed(busy).length, 0)
})
test("published size, text or identity mismatch or audit failure rolls back archive but keeps acknowledged claim", async () => {
for (const options of [{ badPublishedSize: true }, { badPublishedText: true }, { badPublishedId: true }, { auditFails: true }]) {
const f = fixture(); f.configure(options); await assert.rejects(f.run(), model.PollStorageRefused)
@@ -126,6 +126,23 @@ async function compactOnWriter(writer:ReturnType<typeof createPollMaintenanceWri
await writer.finish(secondDeadline)
return { scanned, compacted, skipped: skipped + (compacted ? 0 : 1), automaticReplay: false as const }
}
// A POLL holds the exclusive writer lease for its whole run, so the worker only
// reaches compaction between polls; one archive per tick then falls behind. The
// drain repeats the unchanged single-archive cycle while the writer is free. Each
// cycle re-admits the writer with a try-lock, so a POLL already waiting for its
// exclusive lease stops the drain after at most the cycle in progress.
const DRAIN_BUDGET_MS = 20000, DRAIN_MAX_ARCHIVES = 64
async function drain(deps: PollStorageDependencies) {
const until = deps.now() + DRAIN_BUDGET_MS, total = { scanned: 0, compacted: 0, skipped: 0, automaticReplay: false as const }
for (let cycle = 0; cycle < DRAIN_MAX_ARCHIVES && deps.now() < until; cycle++) {
const result = await compact(deps)
total.scanned += result.scanned; total.compacted += result.compacted; total.skipped += result.skipped
if (result.compacted !== 1) break
}
return total
}
export const compactCompletedServerPollStorage = () => sharedDatabaseGate.request(() => compact(production))
export const drainCompletedServerPollStorage = () => sharedDatabaseGate.request(() => drain(production))
/** Actual owned-SQL fault injection only; not a registered maintenance API. */
export const compactCompletedServerPollStorageForTests = (deps: PollStorageDependencies) => sharedDatabaseGate.request(() => compact(deps))
export const drainCompletedServerPollStorageForTests = (deps: PollStorageDependencies) => sharedDatabaseGate.request(() => drain(deps))