From bd483689291424950379f3c8e66d2250f7584edf Mon Sep 17 00:00:00 2001 From: Denozordec Date: Fri, 20 Nov 2026 02:54:37 +0700 Subject: [PATCH] =?UTF-8?q?fix(mcp):=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=B4=D1=8C=20=D0=B0=D1=80=D1=85=D0=B8=D0=B2=D0=BE=D0=B2=20POL?= =?UTF-8?q?L=20=D0=B8=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BA=D0=B0?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B0=D0=B2=20=D0=BE=D0=BF=D1=80=D0=BE=D1=81?= =?UTF-8?q?=D0=B0=20=D1=80=D0=B0=D0=B7=20=D0=BD=D0=B0=20=D1=8D=D1=82=D0=B0?= =?UTF-8?q?=D0=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Опрос держит исключительную блокировку записи разделов на всё время работы, поэтому воркер успевал архивировать только один источник между опросами. Теперь воркер разбирает очередь, пока писатель свободен (каждый архив — прежний цикл из двух транзакций, уступает ждущему опросу). Права опроса проверяются свежими в начале этапа, перед каждой отправкой на RouterOS и перед COMMIT каждой пишущей транзакции данных или DDL; журнал и чтения используют доказательство не старше 2 с. Холодный опрос: проверок допуска ~3700 -> 32, операция 19,9 -> 3,2 с. Тесты: разбор очереди, уступка ждущему опросу, отзыв прав при удержанных чтениях RouterOS, граница числа проверок; исправлено устаревшее ожидание двух соединений в варианте с потерей ACK. Co-Authored-By: Claude Opus 5.5 --- backend/src/modules/mcp/workers.ts | 4 +-- .../server-partition-maintenance-lease.ts | 13 +++---- .../server-poll-durable-session.sql.test.ts | 36 +++++++++++++++++++ .../services/server-poll-durable-session.ts | 29 ++++++++++++--- ...er-poll-operation-retirement.owned.test.ts | 35 +++++++++++++++++- .../src/services/server-poll-production.ts | 5 +-- backend/src/services/server-poll-storage.ts | 8 +++++ ...server-poll-terminal-archive.owned.test.ts | 3 +- .../server-poll-terminal-compaction.test.ts | 20 +++++++++-- .../server-poll-terminal-compaction.ts | 17 +++++++++ 10 files changed, 151 insertions(+), 19 deletions(-) diff --git a/backend/src/modules/mcp/workers.ts b/backend/src/modules/mcp/workers.ts index 59dfb523..9bffe330 100644 --- a/backend/src/modules/mcp/workers.ts +++ b/backend/src/modules/mcp/workers.ts @@ -7,7 +7,7 @@ import { randomUUID } from "node:crypto" import { retainServerProbeStorage } from "../../services/server-probe-retention.js" import { maintainServerProbeOperationRetirement } from "../../services/server-probe-operation-retirement.js" import { collectExpiredServerPollStorage } from "../../services/server-poll-storage-retention.js" -import { compactCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js" +import { drainCompletedServerPollStorage } from "../../services/server-poll-terminal-compaction.js" import { retireCompletedServerPollOperations } from "../../services/server-poll-operation-retirement.js" import { pool } from "../../db/index.js" import { sharedDatabaseGate } from "../../services/shared-database-gate.js" @@ -165,7 +165,7 @@ async function tickAdmittedMcpWorkers() { await recoverScopedRecoveryIntents() await retainMcpData() await retainServerProbeStorage() - await compactCompletedServerPollStorage() + await drainCompletedServerPollStorage() await retireCompletedServerPollOperations() await collectExpiredServerPollStorage() await retireOrdinaryMcpOperations() diff --git a/backend/src/services/server-partition-maintenance-lease.ts b/backend/src/services/server-partition-maintenance-lease.ts index 1736afb0..055388d5 100644 --- a/backend/src/services/server-partition-maintenance-lease.ts +++ b/backend/src/services/server-partition-maintenance-lease.ts @@ -5,7 +5,7 @@ import type { ServerPollSqlConnection } from "./server-poll-production.js" import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js" import type { ProbeStorageOptions, ProbeStorageSql } from "./server-probe-journal-maintenance.js" import { createProbeStorageConnectionConnector } from "./server-probe-storage-connection.js" -import { runPollStorageTransaction } from "./server-poll-storage.js" +import { freshPollAuthority, runPollStorageTransaction } from "./server-poll-storage.js" export const SERVER_PARTITION_WRITER_LOCK=8723110 export interface ServerPartitionLeaseOptions {signal?:AbortSignal;deadlineAt:number;serverIds:number[]} @@ -72,18 +72,19 @@ export function createServerPartitionMaintenanceLease(deps:ServerPartitionMainte try{await deps.assertAuthority({...holder.options,serverIds:[...holder.options.serverIds]})}catch{fail()} if(!holder.active)fail();checkOptions(options);checkOptions(holder.options) } - const raw=async(holder:Holder,text:string,values:unknown[],options:ServerPartitionTransactionOptions,fence:()=>Promise,scopeLive:()=>boolean=()=>true)=>{ + const raw=async(holder:Holder,text:string,values:unknown[],options:ServerPartitionTransactionOptions,fence:()=>Promise,scopeLive:()=>boolean=()=>true,durableCommit=false)=>{ const complete=deps.boundary?.combine([fence],holder.options.serverIds,holder.options) - const boundary=async()=>{ + const check=async()=>{ local(holder,options);if(!scopeLive())fail() if(complete)await complete() else{await authority(holder,options);await fence();await authority(holder,options)} local(holder,options);if(!scopeLive())fail() } - await boundary() + // Only the COMMIT of a write transaction makes poll data or DDL durable. + await (durableCommit?freshPollAuthority(check):check()) let result!:Awaited> try{result=await holder.connection.query(text,values,{signal:options.signal,timeoutMs:Math.max(1,Math.min(5000,options.deadlineAt-now(),holder.options.deadlineAt-now()))})}catch{holder.destroy();fail(true)} - await boundary();return result + await check();return result } const enqueue=async(owner:{tail:Promise},work:()=>Promise):Promise=>{ const previous=owner.tail;let unlock!:()=>void;owner.tail=new Promise(resolve=>{unlock=resolve});await previous @@ -119,7 +120,7 @@ export function createServerPartitionMaintenanceLease(deps:ServerPartitionMainte await scope.tail;scope.active=false // Any failure after sending COMMIT immediately destroys the owner; there // is no ROLLBACK/retry after the uncertain boundary. - await raw(holder,"COMMIT",[],options,fence);return result + await raw(holder,"COMMIT",[],options,fence,undefined,!readOnly);return result }catch(error){holder.destroy();throw error}finally{scope.active=false} } const transaction=async(readOnly:boolean,fence:()=>Promise,options:ServerPartitionTransactionOptions,work:(query:ServerPartitionQuery)=>Promise):Promise=>{ diff --git a/backend/src/services/server-poll-durable-session.sql.test.ts b/backend/src/services/server-poll-durable-session.sql.test.ts index 9a2ae641..4a39d709 100644 --- a/backend/src/services/server-poll-durable-session.sql.test.ts +++ b/backend/src/services/server-poll-durable-session.sql.test.ts @@ -31,10 +31,12 @@ let releaseNative!:()=>void,firstNative!:()=>void const nativeGate=new Promise(resolve=>{releaseNative=()=>{releasedNative=true;resolve()}}) const started=new Promise(resolve=>{firstNative=resolve}) const paths:string[]=[] +let holdAll:{reached:()=>void;release:Promise}|undefined const listener=createServer(async(req,res)=>{ requests++;paths.push(req.url??"") // Keep the lease observable without exhausting RouterOS's two HTTP sockets. if(requests===1){firstNative();await nativeGate} + else if(holdAll){holdAll.reached();await holdAll.release} res.setHeader("content-type","application/json") const value=req.url==="/rest/system/identity"?{name:"poll-owned-renamed"}:req.url==="/rest/system/resource"?{version:"7.synthetic","board-name":"owned","uptime":"1h","cpu-load":"3","free-memory":"1024","total-memory":"2048"}:req.url==="/rest/interface"?[{name:"owned-interface",type:"ether"}]:req.url==="/rest/ip/address"?[{address:"192.0.2.1/24",interface:"owned-interface"}]:null res.statusCode=value===null?404:200;res.end(JSON.stringify(value)) @@ -101,6 +103,7 @@ try{ assert.equal(requests,0) assert.equal(new Set(plan.maintenance.map(effect=>"parent"in effect?effect.parent:"")).size,PARTITION_SPECS.length) assert(plan.maintenance.some(effect=>effect.kind==="partition.drop"&&effect.name===expired)) + const readsBeforeRun=authorityReads const pending=session.execute(plan);void pending.catch(()=>{}) await Promise.race([started,pending.then(()=>assert.fail("Native request must occur before completion"))]) const {rows:[holder]}=await pool.query(`SELECT pid FROM pg_catalog.pg_locks WHERE locktype='advisory' AND classid=0 @@ -155,6 +158,9 @@ try{ assert.equal((await pool.query("SELECT count(*)::int AS n FROM pg_locks l JOIN pg_class c ON c.oid=l.relation WHERE l.pid=$1 AND l.locktype='relation' AND (c.relname='traffic_samples' OR c.relname LIKE 'traffic_samples_w_%')",[writerPid])).rows[0].n,0) }finally{controller.abort();await assert.rejects(blocked,(error:unknown)=>error instanceof PartitionWriteError&&!error.outcomeUnknown);releaseNative()} const receipt=await pending + const runAuthorityReads=authorityReads-readsBeforeRun + // 17108 complete authority reads before stage coalescing (2026-11-19), 152 after. + assert(runAuthorityReads<=300,"Stage-coalesced authority regressed: "+runAuthorityReads) await tickMcpWorkers() assert.equal(await hasExpiredLease(),false,"The next tick collects the expired lease after POLL releases the writer lock") assert.equal((await compactCompletedServerPollStorage()).compacted,0) @@ -237,6 +243,36 @@ try{ await assert.rejects(session.execute({...plan,bindingProof:"0".repeat(64)}));assert.equal(requests,4) await withPartitionWrite(async owner=>{await owner.client.query("INSERT INTO traffic_samples(server_id,interface_name,sampled_at,rx_bytes) VALUES($1,'after-full-poll',now(),1)",[serverId])}) revoked=true;await assert.rejects(session.execute(plan));assert.equal(requests,4) + + // Stage-coalesced authority contract: rights are fresh before every native + // dispatch and before the COMMIT of every data/DDL write, so a revocation while + // native reads are held allows no later dispatch, effect or receipt. Journal + // bookkeeping of the reads may still land on a grant younger than 2 s. + revoked=false + const revokeRequest={serverId:serverId!,receiptId:randomUUID()} + const revokeSession=createDurableServerPollSession({request:revokeRequest, + identity:{kind:"rest",authentication:"local",ownerId,executionId:randomUUID(),requestId:randomUUID(),issuedAt,expiresAt}, + authorize:async(request,ids,signal)=>{assert.deepEqual(request,revokeRequest);return originalAuthority({serverId:serverId!,receiptId},ids,signal)}}) + const revokePlan=await revokeSession.prepare() + let reached!:()=>void,releaseHeld!:()=>void + const firstHeld=new Promise(resolve=>{reached=resolve}) + holdAll={reached,release:new Promise(resolve=>{releaseHeld=resolve})} + const revokedRun=revokeSession.execute(revokePlan);void revokedRun.catch(()=>{}) + await firstHeld;await new Promise(resolve=>setTimeout(resolve,500)) + const journalOf=async()=>(await pool.query(`SELECT (SELECT count(*)::int FROM server_poll_intents WHERE receipt_id=$1) AS intents, + (SELECT count(*)::int FROM server_poll_intents WHERE receipt_id=$1 AND stage LIKE 'native.%') AS native_intents, + (SELECT count(*)::int FROM server_poll_acks WHERE receipt_id=$1 AND stage NOT LIKE 'native.%') AS effect_acks, + (SELECT count(*)::int FROM server_poll_receipts WHERE receipt_id=$1) AS receipts`,[revokeRequest.receiptId])).rows[0] + const atRevocation=await journalOf() + revoked=true;releaseHeld() + await assert.rejects(revokedRun);holdAll=undefined + const afterRevocation=await journalOf() + assert.equal(atRevocation.receipts,0);assert.equal(afterRevocation.receipts,0,"No receipt after the revocation") + assert.equal(afterRevocation.effect_acks,atRevocation.effect_acks,"No maintenance or publication commits after the revocation") + // HTTP arrival can trail an authorized dispatch (two RouterOS sockets), so the + // proof is the intent: every request that reached RouterOS was journaled first. + assert(requests-4<=atRevocation.native_intents,"Every native request was authorized and journaled before the revocation") + console.log("Full durable poll authority:",JSON.stringify({runAuthorityReads,nativeArrived:requests-4,journalAtRevocation:atRevocation})) console.log("Full durable poll actual SQL/HTTP acceptance:",JSON.stringify({fourNativeGets:requests,maintenanceEffects:plan.maintenance.length,parents:PARTITION_SPECS.length,completedPublications:receipt.completedPublications,sameBackendLeaseAndTransactions:true,guardedSameHolderJournal:true,separateDurableJournalCommits:true,independentQuarantineAfterRealLostCommitAck:true,ordinaryWriterBlockedBeforeRelationSql:true,backgroundTickDeferredAndResumed:true,realJournal:true,repeatedApplyNoNewNative:true,outerInputMutationCannotReplaceCapturedIdentityOrAuthority:true,changedPlanBindingRefused:true,lateRevocationRefused:true,actualHttpAuthenticationMiddleware:false,productionRegistration:false,externalNetwork:false,authorityReads})) }finally{ if(!releasedNative)releaseNative() diff --git a/backend/src/services/server-poll-durable-session.ts b/backend/src/services/server-poll-durable-session.ts index dfe96b9a..00334ced 100644 --- a/backend/src/services/server-poll-durable-session.ts +++ b/backend/src/services/server-poll-durable-session.ts @@ -7,6 +7,8 @@ import { pollProblem } from "./server-poll-execution-model.js" import type { ServerPollDependencies, ServerPollPlan, ServerPollRequest } from "./server-poll-execution.js" import { createHolderBoundServerPollStorageJournal, createServerPollStorageJournal } from "./server-poll-storage-journal.js" import { createServerPollBoundaryCoordinator } from "./server-poll-boundary.js" +import { freshPollAuthorityScope } from "./server-poll-storage.js" +import { coalesceAuthorization } from "../modules/mcp/authorization-stage.js" export interface DurableServerPollSessionInput { request:ServerPollRequest @@ -36,12 +38,31 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean signer??=security.privateFingerprinter("server-poll-production-v1") return (await signer)(security.fingerprint(value)) } + // Rights are proven fresh at every stage start (prepare/execute/read reset the + // stages), before every RouterOS dispatch and right before the COMMIT of every + // write that changes application data or DDL; one proof serves a whole marked + // boundary. Journal and read boundaries reuse a grant whose check began < 2 s ago; + // a refusal is never reused. + type Grant=Awaited> + type Stage={signal?:AbortSignal;proof:ReturnType>} + let stages=new Map() + const grantFor=(required:number[],signal?:AbortSignal):Promise=>{ + const key=[...required].sort((a,b)=>a-b).join(",") + let stage=stages.get(key) + if(!stage){const created={signal} as Stage;created.proof=coalesceAuthorization(()=>authorityProvider({...request},[...required],created.signal));stages.set(key,stage=created)} + stage.signal=signal + const boundary=freshPollAuthorityScope() + if(!boundary)return stage.proof.coalesced() + let proven=boundary.get(key) as Promise|undefined + if(!proven)boundary.set(key,proven=stage.proof.fresh()) + return proven + } const authorize:ServerPollDependencies["authorize"]=async(candidate,ids,signal)=>{ if(candidate.serverId!==request.serverId||candidate.receiptId!==request.receiptId)pollProblem("STATE_CONFLICT","Запрос не принадлежит durable poll session") signal?.throwIfAborted() if(Date.now()=identity.expiresAt)pollProblem("PERMISSION_DENIED","Исходная identity poll истекла") const required=[...new Set([request.serverId,...ids])] - const grant=await authorityProvider({...request},required,signal) + const grant=await grantFor(required,signal) if(!grant.global||required.some(id=>!grant.serverIds.includes(id))||!/^[a-f0-9]{64}$/.test(grant.authorityProof))pollProblem("PERMISSION_DENIED","Требуется полный живой global допуск poll") signal?.throwIfAborted() if(Date.now()>=identity.expiresAt)pollProblem("PERMISSION_DENIED","Исходная identity poll истекла") @@ -63,7 +84,7 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean // Preparation has no journal effects. These throwing boundaries make an // accidental unbound execution impossible; they never invent an ACK/claim. const unavailable=async():Promise=>{throw new ServerPollJournalConflict()} - const prepare=async(signal?:AbortSignal)=>build({claim:unavailable,intent:unavailable,acknowledge:unavailable,complete:unavailable}).prepare(request,signal) + const prepare=async(signal?:AbortSignal)=>{stages=new Map();return build({claim:unavailable,intent:unavailable,acknowledge:unavailable,complete:unavailable}).prepare(request,signal)} const journalFor=async(raw:ServerPollPlan,mode:"execute"|"read",signal?:AbortSignal)=>{ const plan=structuredClone(raw) if(plan.request.serverId!==request.serverId||plan.request.receiptId!==request.receiptId)pollProblem("STATE_CONFLICT","План не принадлежит durable poll session") @@ -83,8 +104,8 @@ function durableSession(input:DurableServerPollSessionInput,storageBound:boolean }) return {journal,plan} } - const execute=async(raw:ServerPollPlan,signal?:AbortSignal)=>{const {journal,plan}=await journalFor(raw,"execute",signal);return build(journal).execute(plan,signal)} - const readExisting=async(raw:ServerPollPlan,signal?:AbortSignal)=>{const {journal}=await journalFor(raw,"read",signal);return journal.readExisting()} + const execute=async(raw:ServerPollPlan,signal?:AbortSignal)=>{stages=new Map();const {journal,plan}=await journalFor(raw,"execute",signal);return build(journal).execute(plan,signal)} + const readExisting=async(raw:ServerPollPlan,signal?:AbortSignal)=>{stages=new Map();const {journal}=await journalFor(raw,"read",signal);return journal.readExisting()} return {prepare,execute,readExisting} } /** Historical private assembly for the enrolled 0038 fixture; public adapters diff --git a/backend/src/services/server-poll-operation-retirement.owned.test.ts b/backend/src/services/server-poll-operation-retirement.owned.test.ts index ab1d3794..08e85bc1 100644 --- a/backend/src/services/server-poll-operation-retirement.owned.test.ts +++ b/backend/src/services/server-poll-operation-retirement.owned.test.ts @@ -15,7 +15,7 @@ const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp") const { invokeMcpTool } = await import("../modules/mcp/runtime.js") const { encryptSecret,privateSecretFingerprint,fingerprint,redact } = await import("../modules/mcp/security.js") const { encodeServerPollJournalAnchor } = await import("./server-poll-journal-anchor.js") -const { compactCompletedServerPollStorage,compactCompletedServerPollStorageForTests,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js") +const { compactCompletedServerPollStorage,compactCompletedServerPollStorageForTests,drainCompletedServerPollStorage,SERVER_POLL_ARCHIVE_READ_SQL } = await import("./server-poll-terminal-compaction.js") const { openServerPollTerminalArchive } = await import("./server-poll-terminal-archive.js") const { retireCompletedServerPollOperation, retireCompletedServerPollOperations } = await import("./server-poll-operation-retirement.js") const { PARTITION_WRITER_ADVISORY_KEY } = await import("./partition-write-transaction.js") @@ -330,6 +330,39 @@ try{ assert.deepEqual(await compactCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false}) console.log("Owned PostgreSQL: guarded compaction and mixed V1/V2 GC, physical claim/publish/refund ACK losses, retained history, and actual worker skips held/spent rows") + // A backlog of naturally expired sources leaves in one drain and in one worker tick. + const drainOwner=await historical(2,Number(server.id)) + const more=async()=>[await historical(2,Number(server.id),drainOwner),await historical(2,Number(server.id),drainOwner)] + const archivedOf=async(sources:{receiptId:string}[])=>(await pool.query("SELECT count(*)::int AS n FROM server_poll_archives WHERE receipt_id=ANY($1::uuid[])",[sources.map(source=>source.receiptId)])).rows[0].n + const drained=[drainOwner,...await more()] + assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:3,compacted:3,skipped:0,automaticReplay:false}) + assert.equal(await archivedOf(drained),3) + const ticked=[await historical(2,Number(server.id),drainOwner),...await more()] + await tickMcpWorkers() + assert.equal(await archivedOf(ticked),3) + assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false}) + assert.equal((await pool.query("SELECT server_poll_storage_accounting_valid() AS valid")).rows[0].valid,true) + // A POLL already waiting for its exclusive lease stops the drain: every cycle + // re-admits the writer with a try-lock that cannot queue ahead of that waiter. + const queued=[await historical(2,Number(server.id),drainOwner),await historical(2,Number(server.id),drainOwner)] + const sharedHolder=await pool.connect(),exclusiveWaiter=await pool.connect() + try{ + await sharedHolder.query("SELECT pg_advisory_lock_shared($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY]) + const waiterPid=(await exclusiveWaiter.query("SELECT pg_backend_pid() AS pid")).rows[0].pid + const exclusive=exclusiveWaiter.query("SELECT pg_advisory_lock($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY]) + for(let attempt=0;!(await pool.query("SELECT 1 FROM pg_locks WHERE pid=$1 AND locktype='advisory' AND mode='ExclusiveLock' AND NOT granted",[waiterPid])).rows.length;attempt++){ + assert(attempt<200,"exclusive request did not start waiting");await new Promise(resolve=>setTimeout(resolve,25)) + } + assert.deepEqual(await drainCompletedServerPollStorage(),{scanned:0,compacted:0,skipped:0,automaticReplay:false}) + assert.equal(await archivedOf(queued),0) + await sharedHolder.query("SELECT pg_advisory_unlock_shared($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY]) + await exclusive + await exclusiveWaiter.query("SELECT pg_advisory_unlock($1::bigint)",[PARTITION_WRITER_ADVISORY_KEY]) + }finally{sharedHolder.release(true);exclusiveWaiter.release(true)} + assert.equal((await drainCompletedServerPollStorage()).compacted,2) + assert.equal(await archivedOf(queued),2) + console.log("Owned PostgreSQL: worker drain archives a three-source backlog in one call and in one tick, and yields to a waiting exclusive POLL lease") + if (process.env.MMAPP_OWN_SKIP_POLL_CAPACITY === "true") { console.log("Focused POLL retirement: 336-archive capacity scenario skipped") } else { diff --git a/backend/src/services/server-poll-production.ts b/backend/src/services/server-poll-production.ts index 6861e383..fa608afa 100644 --- a/backend/src/services/server-poll-production.ts +++ b/backend/src/services/server-poll-production.ts @@ -7,6 +7,7 @@ import type { QueryConfig } from "pg" import type { ServerPartitionTransaction } from "./server-partition-maintenance-lease.js" import type { ServerPollBoundaryCoordinator } from "./server-poll-boundary.js" import { PARTITION_SPECS } from "../db/partitions.js" +import { freshPollAuthority } from "./server-poll-storage.js" import { createServerPollExecution,type ServerPollDependencies,type ServerPollGrant,type ServerPollRequest } from "./server-poll-execution.js" import { ServerPollProblem,pollProblem,validateServerPollContext,planServerPollMaintenance,type ServerPollContext,type PollRow,type ServerPollAck } from "./server-poll-execution-model.js" @@ -178,7 +179,7 @@ export function createServerPollProductionDependencies(input:ServerPollProductio const transaction=async(readOnly:boolean,fence:()=>Promise,options:{signal?:AbortSignal;deadlineAt:number},work:(query:(sql:string,values?:unknown[])=>Promise<{rows:Record[];rowCount?:number|null}>)=>Promise):Promise=>{ if(io.transaction)return io.transaction(readOnly,fence,options,work) await fence();let connection:ServerPollSqlConnection;try{connection=await io.connect(options)}catch(error){throw carryPollError(new ServerPollProblem("STATE_UNAVAILABLE","SQL connection server poll недоступен"),error,"poll.connect")};let unsafe=false,committed=false - const query=async(sql:string,values:unknown[]=[])=>{await fence();const timeoutMs=Math.max(1,Math.min(10000,options.deadlineAt-now()));let result:{rows:Record[];rowCount?:number|null};try{result=await connection.query(sql,values,{signal:options.signal,timeoutMs})}catch(error){unsafe=true;throw carryPollError(Error("SQL boundary unavailable"),error,"poll.sql")};await fence();return result} + const query=async(sql:string,values:unknown[]=[])=>{await (sql==="COMMIT"&&!readOnly?freshPollAuthority(fence):fence());const timeoutMs=Math.max(1,Math.min(10000,options.deadlineAt-now()));let result:{rows:Record[];rowCount?:number|null};try{result=await connection.query(sql,values,{signal:options.signal,timeoutMs})}catch(error){unsafe=true;throw carryPollError(Error("SQL boundary unavailable"),error,"poll.sql")};await fence();return result} try{await query(readOnly?"BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY":"BEGIN");await query("SET LOCAL TIME ZONE 'UTC'");await query(`SET LOCAL statement_timeout='${Math.max(1,Math.min(10000,options.deadlineAt-now()))}ms'`);const result=await work(query);await query("COMMIT");committed=true;return result} finally{if(!committed)unsafe=true;connection.release(unsafe)} } @@ -276,7 +277,7 @@ export function createServerPollProductionDependencies(input:ServerPollProductio },{...options,serverIds:ids})) }) }, - native:async(read,server,options)=>{const {lease,fence}=requireLease(options);let dispatched=false;await fence();return io.native(read,server,{...options,beforeDispatch:async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повтор native dispatch poll запрещён");await fence();await options.beforeDispatch();await fence();const current=lease.context?.servers.find(row=>row.id===input.request.serverId);if(server.id!==input.request.serverId||!current||await proof(current)!==await proof(server)||dispatched)pollProblem("STATE_CONFLICT","Native endpoint/dispatch не совпадает с private source poll");dispatched=true}})}, + native:async(read,server,options)=>{const {lease,fence}=requireLease(options);let dispatched=false;await fence();return io.native(read,server,{...options,beforeDispatch:()=>freshPollAuthority(async()=>{if(dispatched)pollProblem("STATE_CONFLICT","Повтор native dispatch poll запрещён");await fence();await options.beforeDispatch();await fence();const current=lease.context?.servers.find(row=>row.id===input.request.serverId);if(server.id!==input.request.serverId||!current||await proof(current)!==await proof(server)||dispatched)pollProblem("STATE_CONFLICT","Native endpoint/dispatch не совпадает с private source poll");dispatched=true})})}, effect,intent:input.journal.intent,acknowledge:input.journal.acknowledge,claim:input.journal.claim,complete:input.journal.complete, } } diff --git a/backend/src/services/server-poll-storage.ts b/backend/src/services/server-poll-storage.ts index 3c9c4d4d..ff8b49a8 100644 --- a/backend/src/services/server-poll-storage.ts +++ b/backend/src/services/server-poll-storage.ts @@ -2,6 +2,7 @@ import { carryPollError } from "./server-poll-error-provenance.js" /** Private opt-in poll storage. No import/constructor performs I/O or enables * a route. The separately reviewed storage schema must already be installed. */ import { randomUUID } from "node:crypto" +import { AsyncLocalStorage } from "node:async_hooks" import { z } from "zod" import { connectProbeStorage } from "./server-probe-storage-connection.js" import type { ProbeStorageConnection, ProbeStorageOptions, ProbeStorageSql } from "./server-probe-journal-maintenance.js" @@ -30,6 +31,13 @@ export interface PollStorageDependencies { } const production:PollStorageDependencies={connect:connectProbeStorage,now:Date.now} const fail=():never=>{throw new PollStorageRefused()} +/** Marks the boundaries that need fresh rights: every RouterOS dispatch and the + * COMMIT of every write transaction that changes application data or DDL. Inside one + * marked boundary each scope is proven once; journal and read boundaries may reuse + * a recent grant (MCP journal rows also re-check the connection in their own SQL). */ +const commitAuthority=new AsyncLocalStorage>>() +export const freshPollAuthority=(run:()=>Promise):Promise=>commitAuthority.run(new Map(),run) +export const freshPollAuthorityScope=()=>commitAuthority.getStore() const lock="SELECT pg_advisory_xact_lock(hashtextextended('server-poll-storage-v1',0))" /** The writer fence precedes POLL's storage, operation and row locks. Restore diff --git a/backend/src/services/server-poll-terminal-archive.owned.test.ts b/backend/src/services/server-poll-terminal-archive.owned.test.ts index f6e2bc86..c260b2b8 100644 --- a/backend/src/services/server-poll-terminal-archive.owned.test.ts +++ b/backend/src/services/server-poll-terminal-archive.owned.test.ts @@ -278,7 +278,8 @@ async function main() { assert.equal(error.message.includes("PRIVATE_OWNED_ARCHIVE_COMMIT_ACK"), false) return true }) - assert.equal(commits, 2); assert.deepEqual(releases, [false, true]) + // One maintenance-writer backend carries both transactions; the unknown COMMIT destroys it. + assert.equal(commits, 2); assert.deepEqual(releases, [true]) assert.equal(commands.includes("ROLLBACK"), false, "Unknown COMMIT ACK cannot trigger rollback or replay") commitAckLossAccepted = true groups.push("after natural original expiry, real claim COMMIT ACK is known and real archive COMMIT succeeds but loses caller ACK; generic unknown destroys backend without rollback/retry") diff --git a/backend/src/services/server-poll-terminal-compaction.test.ts b/backend/src/services/server-poll-terminal-compaction.test.ts index d50b56d0..638e89a3 100644 --- a/backend/src/services/server-poll-terminal-compaction.test.ts +++ b/backend/src/services/server-poll-terminal-compaction.test.ts @@ -1,6 +1,7 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" import { randomUUID } from "node:crypto" +import { AsyncLocalStorage } from "node:async_hooks" import { test } from "node:test" import { runInNewContext } from "node:vm" import ts from "typescript" @@ -21,7 +22,8 @@ const ACCOUNTING = "SELECT component_accounting" type Row = Record type State = { claim: string | null; archive: Row | null; audits: Row[] } type Command = { tx: number; text: string; values: unknown[] } -type Api = { compactCompletedServerPollStorageForTests(deps: PollStorageDependencies): Promise<{ scanned: number; compacted: number; skipped: number; automaticReplay: false }> } +type Result = Promise<{ scanned: number; compacted: number; skipped: number; automaticReplay: false }> +type Api = { compactCompletedServerPollStorageForTests(deps: PollStorageDependencies): Result; drainCompletedServerPollStorageForTests(deps: PollStorageDependencies): Result } function load(code: string, modules: Record) { const exports: Record = {} runInNewContext(code, { exports, require(name: string) { assert(Object.hasOwn(modules, name), "Unapproved production import: " + name); return modules[name] }, @@ -41,7 +43,7 @@ function fixture(quarantined = false) { const forbiddenConnector = async (): Promise => { throw Error("Production connector must never run") } const accounting = async (sql: { query(text: string): Promise }) => { accountCalls++; await sql.query(ACCOUNTING); if (accountingFails) throw new model.PollStorageRefused() } const storage = load(storageCode, { - "node:crypto": { randomUUID }, zod: { z }, "./server-probe-storage-connection.js": { connectProbeStorage: forbiddenConnector }, + "node:crypto": { randomUUID }, "node:async_hooks": { AsyncLocalStorage }, zod: { z }, "./server-probe-storage-connection.js": { connectProbeStorage: forbiddenConnector }, "./server-poll-storage-model.js": model, "./server-poll-archive-accounting.js": { assertServerPollArchiveAccounting: accounting }, "./server-poll-error-provenance.js": errorProvenance, "./partition-write-transaction.js": { admitPartitionWriter, PARTITION_WRITER_ADVISORY_KEY }, @@ -139,7 +141,7 @@ function fixture(quarantined = false) { throw Error("Unapproved fixture SQL: " + text) }, release(unsafe = false) { assert(!closed); closed = true; transaction = undefined; releases.push({ tx:owner, unsafe }) } } } } - return { run: () => api.compactCompletedServerPollStorageForTests(deps), commands, releases, events, state: () => structuredClone(state), + return { run: () => api.compactCompletedServerPollStorageForTests(deps), drain: () => api.drainCompletedServerPollStorageForTests(deps), commands, releases, events, state: () => structuredClone(state), counts: () => ({ connects, commits, createCalls, openCalls, accountCalls }), Unknown: storage.PollStorageUnknown as new () => Error,setWriter:(value:boolean|Error)=>{writer=value}, configure(options: { eligible?: boolean; sourceMissing?: boolean; accountingFails?: boolean; preflight?: typeof preflight; secondOversize?: boolean; lostCommit?: number; commitPersists?: boolean; badPublishedSize?: boolean; badPublishedText?: boolean; badPublishedId?: boolean; auditFails?: boolean; skippedPrefix?: typeof skippedPrefix; removeLastSlowReceipt?: boolean }) { eligible = options.eligible ?? eligible; sourceMissing = options.sourceMissing ?? sourceMissing; accountingFails = options.accountingFails ?? accountingFails @@ -265,6 +267,18 @@ test("phase2 size growth preserves spent full charge and subsequent candidate ex assert(!f.commands.some(command => /^(?:DELETE|UPDATE server_poll_storage_usage)/.test(command.text))) }) +test("worker drain re-admits the writer for every cycle and stops on the first cycle without an archive", async () => { + const f = fixture() + assert.deepEqual({ ...await f.drain() }, { scanned: 1, compacted: 1, skipped: 0, automaticReplay: false }) + const tries = f.commands.filter(command => command.text.includes("pg_try_advisory_lock_shared")), unlocks = f.commands.filter(command => command.text.includes("pg_advisory_unlock_shared")) + assert.equal(tries.length, 2); assert.equal(unlocks.length, 2); assert.equal(f.counts().connects, 2) + assert.deepEqual(f.releases, [{ tx: 1, unsafe: false }, { tx: 2, unsafe: false }]) + assert.equal(claimed(f).length, 1); assert.equal(published(f).length, 1) + const busy = fixture(); busy.setWriter(false) + assert.deepEqual({ ...await busy.drain() }, { scanned: 0, compacted: 0, skipped: 0, automaticReplay: false }) + assert.equal(busy.counts().connects, 1); assert.equal(claimed(busy).length, 0) +}) + test("published size, text or identity mismatch or audit failure rolls back archive but keeps acknowledged claim", async () => { for (const options of [{ badPublishedSize: true }, { badPublishedText: true }, { badPublishedId: true }, { auditFails: true }]) { const f = fixture(); f.configure(options); await assert.rejects(f.run(), model.PollStorageRefused) diff --git a/backend/src/services/server-poll-terminal-compaction.ts b/backend/src/services/server-poll-terminal-compaction.ts index 0b406862..e40315f2 100644 --- a/backend/src/services/server-poll-terminal-compaction.ts +++ b/backend/src/services/server-poll-terminal-compaction.ts @@ -126,6 +126,23 @@ async function compactOnWriter(writer:ReturnType sharedDatabaseGate.request(() => compact(production)) +export const drainCompletedServerPollStorage = () => sharedDatabaseGate.request(() => drain(production)) /** Actual owned-SQL fault injection only; not a registered maintenance API. */ export const compactCompletedServerPollStorageForTests = (deps: PollStorageDependencies) => sharedDatabaseGate.request(() => compact(deps)) +export const drainCompletedServerPollStorageForTests = (deps: PollStorageDependencies) => sharedDatabaseGate.request(() => drain(deps))