Files
MikrotikManager/backend/src/services/traffic-flow-engine-owned.test.ts
T

325 lines
22 KiB
TypeScript

import assert from "node:assert/strict"
import { readFile } from "node:fs/promises"
import { runInNewContext } from "node:vm"
import * as crypto from "node:crypto"
import { isDeepStrictEqual } from "node:util"
import { createFlowIngestQuality, type FlowIngestQuality } from "./traffic-flow-ingest-quality.js"
import ts from "typescript"
import { DatabaseBackgroundWork } from "./database-background-work.js"
import * as owned from "./traffic-flow-owned-batch.js"
import * as proof from "./traffic-flow-batch-proof.js"
import * as accumulator from "./traffic-flow-fact-accumulator.js"
import * as serial from "./serial-operation.js"
import * as parse from "./traffic-flow-parse.js"
import * as ip from "./traffic-flow-ip.js"
import { pruneRipeSqlite } from "./traffic-flow-ripe.js"
import { PARTITION_WRITER_ADVISORY_KEY } from "./partition-write-transaction.js"
// Exercise the actual connected engine with explicit fake I/O boundaries.
const source=await readFile(new URL("./traffic-flow-engine.ts",import.meta.url),"utf8")
assert.ok(source.includes("ownedEngineBatch?.assertResolved()"))
assert.ok(!source.includes("privateSecretFingerprint"))
const live=accumulator.createFlowFactAccumulator()
let mode="normal",commits=0,dispatches=0,inTransaction=false,receiptPresent=false,factsResetAllowed=true,mutatePayload=false,ripePruneFails=false
let identified:{rows:{bytes:number}[];ingestQuality:FlowIngestQuality}|undefined
let heldStarted:()=>void=()=>{},releaseHeld:()=>void=()=>{},heldGate:Promise<void>=Promise.resolve()
const capQueries:Array<{sql:string;params:unknown}>=[]
class Unknown extends Error {}
const dependencies:Record<string,unknown>={
"node:crypto":crypto,
"node:util":{isDeepStrictEqual},"./traffic-flow-ingest-quality.js":{createFlowIngestQuality},
"./traffic-flow-owned-batch.js":owned,"./traffic-flow-batch-proof.js":{flowBatchProof:(payload:unknown,domain:"ingest"|"facts")=>{identified=payload as typeof identified;return proof.flowBatchProof(payload,domain)}},
"./traffic-flow-fact-accumulator.js":accumulator,"./serial-operation.js":serial,
"./traffic-flow-parse.js":parse,"./traffic-flow-ip.js":ip,
"./database-background-work.js":{databaseBackgroundWork:{track:()=>{throw Error("unexpected background load")}}},
"./traffic-flow-write-transaction.js":{
FlowCommitUnknown:Unknown,flowWriteContext:()=>inTransaction?{partitions:new Set()}:undefined,
flowWritePool:{query:async()=>{assert.ok(inTransaction);dispatches++;return {rows:[]}}},
flowWriteAll:async(sql:string,params:unknown)=>{assert.ok(inTransaction);dispatches++;if(sql.includes("FROM flow_daily_dims")&&sql.includes("dim = 'asn'"))capQueries.push({sql,params});return []},
flowWriteQuery:async(sql:string)=>{assert.ok(inTransaction);dispatches++;if(ripePruneFails&&sql.includes("DELETE FROM flow_ip_meta"))throw Error("RIPE prune SQL failure");return {rows:[]}},
commitFlowWrite:async(_input:unknown,work:()=>Promise<unknown>,authorize:()=>Promise<unknown>)=>{
commits++;await authorize();inTransaction=true
if(mutatePayload&&identified?.rows[0])identified.rows[0].bytes++
try{const result=await work();await authorize();if(mode==="held"){heldStarted();await heldGate}if(mode==="unknown")throw new Unknown("lost COMMIT");if(mode==="rollback")throw Error("known rollback");return result}finally{inTransaction=false}
},
reconcileFlowWrite:async()=>({committed:receiptPresent,result:receiptPresent?{buckets:1,ingestQuality:identified?.ingestQuality}:null}),
},
"../db/partitions.js":{specForParent:()=>null,ensurePartitionFor:()=>{throw Error("unexpected DDL")}},
"./traffic-flow-planes.js":{classifyFlowPlaneLite:()=>"internet"},
"./traffic-flow-map-exporter.js":{pickServerIdForExporter:()=>1},
"./traffic-flow-apps.js":{applicationName:()=>"https"},
"./traffic-flow-ripe.js":{enqueueRipeMisses:()=>{throw Error("unexpected RIPE enqueue")},pruneRipeSqlite},
"./traffic-flow-settings.js":{invalidateTrafficFlowSettingsCache:()=>{}},
"./traffic-flow-ifaces.js":{maybeRefreshIfaces:async()=>{}},
"./traffic-flow-ifindex.js":{canonicalFactIface:(_server:number,iface:string)=>iface},
"./traffic-flow-facts-filter.js":{shouldWriteFlowFact:()=>true},
"./traffic-flow-dest.js":{resolveInternetDest:()=>({dest:"",ripe:null,classified:{category:"web",service:"https"},country:"XX",asn:0})},
"./traffic-flow-topology.js":{peekFlowTopology:()=>({}),peekServerCatalog:()=>({byId:new Map([[1,{type:"router"}]])}),getServerCatalog:()=>{throw Error("unexpected catalog SQL")},loadFlowTopology:()=>{throw Error("unexpected topology SQL")}},
"./traffic-flow-facts.js":{
bumpFlowFact:live.bump,hasPendingFlowFact:live.has,factsPendingSize:()=>live.size,
takePendingFlowFacts:()=>{const rows=live.snapshot();live.clear();return rows},
persistFlowFactRows:async(rows:unknown[])=>{assert.ok(inTransaction);return rows.length},
hourBucketIso:accumulator.hourBucketIso,resetFactsAfterDatabaseRestore:()=>live.clear(),assertFactsResetAllowed:()=>{if(!factsResetAllowed)throw Error("unresolved standalone facts")},
},
}
const output=ts.transpileModule(source,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022}}).outputText
type ReadOptions={serverId?:number;sinceIso?:string;untilIso?:string}
type ReadRow={serverId:number;bucketAt:string;src:string;bytes:number;packets:number}
type Engine={peekPendingFlows:(options?:ReadOptions)=>ReadRow[];listLiveFlowRows:(sinceIso:string,options?:Omit<ReadOptions,"sinceIso">)=>ReadRow[];queueParsedFlows:(id:number,flows:unknown[])=>void;strictDrainEngine:(authorize?:()=>Promise<unknown>,serverIds?:readonly number[])=>Promise<unknown>;getEngineStats:()=>{pendingSize:number;rowsStored:number};resetEngineAfterDatabaseRestore:()=>void;reconcileEnginePendingWrite:()=>Promise<{state:string}>}
const exports:Record<string,unknown>={}
runInNewContext(output,{exports,require:(name:string)=>{if(!(name in dependencies))throw Error("Unexpected engine dependency: "+name);return dependencies[name]},Buffer,console,setTimeout,clearTimeout})
const engine=exports as unknown as Engine
assert.equal(await engine.strictDrainEngine(async()=>{},[]),null,"unobserved maintenance drain must not overwrite restored stats or run a global cap")
assert.equal(commits,0);assert.equal(dispatches,0)
engine.resetEngineAfterDatabaseRestore()
assert.equal(await engine.strictDrainEngine(async()=>{},[]),null)
assert.equal(commits,0);assert.equal(dispatches,0)
const strict=engine.strictDrainEngine
engine.strictDrainEngine=(authorize=async()=>{},serverIds=[1])=>strict(authorize,serverIds)
const flow={src:"192.0.2.1",dst:"198.51.100.1",proto:6,srcPort:12345,dstPort:443,bytes:100,packets:1,inIface:"ether1",outIface:"ether2"}
engine.queueParsedFlows(1,[flow]);mode="rollback"
await assert.rejects(engine.strictDrainEngine(async()=>{},[2]),/outside authorized drain scope/)
assert.equal(commits,0);assert.equal(dispatches,0)
await assert.rejects(engine.strictDrainEngine(),/known rollback/)
assert.equal(engine.getEngineStats().pendingSize,1)
assert.throws(()=>engine.resetEngineAfterDatabaseRestore(),/resolved|unresolved/i)
engine.queueParsedFlows(1,[{...flow,src:"192.0.2.2"}]);mode="unknown"
await assert.rejects(engine.strictDrainEngine(),/lost COMMIT/)
const attempted=commits,sql=dispatches
await assert.rejects(engine.strictDrainEngine(),/unknown/i)
assert.equal(commits,attempted);assert.equal(dispatches,sql)
assert.equal(engine.getEngineStats().pendingSize,2)
assert.throws(()=>engine.resetEngineAfterDatabaseRestore(),/resolved|unresolved/i)
receiptPresent=true
assert.equal((await engine.reconcileEnginePendingWrite()).state,"committed")
assert.equal(engine.getEngineStats().pendingSize,1)
assert.equal(engine.getEngineStats().rowsStored,1)
mode="normal";await engine.strictDrainEngine()
assert.ok(capQueries.length>0&&capQueries.every(query=>query.sql.includes("server_id=ANY(?::integer[])")))
assert.equal(engine.getEngineStats().pendingSize,0)
engine.resetEngineAfterDatabaseRestore()
engine.queueParsedFlows(1,[flow]);const before=commits
await assert.rejects(engine.strictDrainEngine(async()=>{throw Error("revoked")}),/revoked/)
assert.equal(commits,before);assert.equal(engine.getEngineStats().pendingSize,1)
factsResetAllowed=false
assert.throws(()=>engine.resetEngineAfterDatabaseRestore(),/standalone facts/)
assert.equal(engine.getEngineStats().pendingSize,1,"fact-owner reset refusal precedes every engine clear")
factsResetAllowed=true;mutatePayload=true
await assert.rejects(engine.strictDrainEngine(),/payload changed/)
assert.equal(engine.getEngineStats().pendingSize,1)
mutatePayload=false
const retryExports:Record<string,unknown>={}
runInNewContext(output,{exports:retryExports,require:(name:string)=>{if(!(name in dependencies))throw Error("Unexpected engine dependency: "+name);return dependencies[name]},Buffer,console,setTimeout,clearTimeout})
const normalEngine=retryExports as unknown as Engine&{flushPending:(opts:{force:boolean;prune?:boolean})=>Promise<void>}
normalEngine.queueParsedFlows(1,[flow]);mode="rollback"
await assert.rejects(normalEngine.flushPending({force:true}),/known rollback/)
const priorNormalAttempts=commits,priorNormalSql=dispatches
await assert.rejects(normalEngine.strictDrainEngine(async()=>{},[1]),/global prune/)
assert.equal(commits,priorNormalAttempts);assert.equal(dispatches,priorNormalSql)
assert.equal(normalEngine.getEngineStats().pendingSize,1)
assert.throws(()=>normalEngine.resetEngineAfterDatabaseRestore(),/unresolved/)
mode="normal";ripePruneFails=true
await assert.rejects(normalEngine.flushPending({force:true}),/RIPE prune SQL failure/)
assert.equal(normalEngine.getEngineStats().pendingSize,1)
ripePruneFails=false
const scopedExports:Record<string,unknown>={}
runInNewContext(output,{exports:scopedExports,require:(name:string)=>{if(!(name in dependencies))throw Error("Unexpected engine dependency: "+name);return dependencies[name]},Buffer,console,setTimeout,clearTimeout})
const scopedEngine=scopedExports as unknown as Engine&{configureEngine:(input:{topN:number})=>void;peekPendingFlows:()=>unknown[]}
scopedEngine.configureEngine({topN:1})
scopedEngine.queueParsedFlows(1,[{...flow,bytes:1000}])
scopedEngine.queueParsedFlows(999,Array.from({length:21},(_,index)=>({...flow,src:`203.0.113.${index+1}`,bytes:index===20?1:1000})))
const scopedAttempts=commits
await assert.rejects(scopedEngine.strictDrainEngine(async()=>{},[1]),/Pending flow source/)
assert.equal(commits,scopedAttempts);assert.equal(scopedEngine.getEngineStats().pendingSize,22,"foreign below per-server minimum topN=20 input stays pending without detachment or discard")
assert.ok(source.indexOf("serverIds&&[...pending.values()]")<source.indexOf("for(const row of topNPending("),"complete source-scope check precedes topN truncation")
// Execute the actual purge code too: drain must precede the exclusive
// writer lease; partition work stays fenced; restart follows unlock on all paths.
const purgeSource=await readFile(new URL("./traffic-flow-purge-execution.ts",import.meta.url),"utf8")
const purgeExports:Record<string,unknown>={}
runInNewContext(ts.transpileModule(purgeSource,{compilerOptions:{module:ts.ModuleKind.CommonJS,target:ts.ScriptTarget.ES2022}}).outputText,{exports:purgeExports,Error,process:{env:{}},require:(name:string)=>{if(name==="./partition-write-transaction.js")return {PARTITION_WRITER_ADVISORY_KEY};if(name==="../db/database-restore-quarantine.js")return {databaseRestoreQuarantineEnabled:()=>false};throw Error("Unexpected purge dependency: "+name)}})
type PurgeDeps={stop:()=>Promise<void>;deleteData:()=>Promise<Record<string,number>>;invalidate:()=>void;maintain:()=>Promise<void>;restart:()=>Promise<void>;withDataFence:<T>(work:()=>Promise<T>)=>Promise<T>}
const executePurge=purgeExports.executeTrafficFlowPurge as (deps:PurgeDeps)=>Promise<{partitionError?:string}>
const order:string[]=[];let held=false,failDelete=false
const purgeDeps:PurgeDeps={
stop:async()=>{order.push("drain");assert.equal(held,false)},
withDataFence:async work=>{held=true;order.push("lock");try{return await work()}finally{held=false;order.push("unlock")}},
deleteData:async()=>{assert.equal(held,true);order.push("delete");if(failDelete)throw Error("delete failed");return {buckets:1,minuteStats:1,minuteDims:1,dailyDims:1}},
invalidate:()=>{assert.equal(held,true);order.push("invalidate")},
maintain:async()=>{assert.equal(held,true);order.push("maintain");throw Error("partition partial")},
restart:async()=>{assert.equal(held,false);order.push("restart")},
}
assert.equal((await executePurge(purgeDeps)).partitionError,"partition partial")
assert.deepEqual(order,["drain","lock","delete","invalidate","maintain","unlock","restart"])
order.length=0;failDelete=true
await assert.rejects(executePurge(purgeDeps),/delete failed/)
assert.deepEqual(order,["drain","lock","delete","unlock","restart"])
console.log("traffic-flow-engine-owned.test.ts: actual connected engine source; known rollback retains batch, unknown prevents replay/reset, new arrivals isolated, explicit receipt reconcile acknowledges once, revocation blocks capture/dispatch; fake dependencies only")
// Read the exact detached candidate batch through the real engine exports.
// SQL-backed callers can merge only known-uncommitted rows; all uncertain
// intersections fail explicitly and unrelated server/windows remain readable.
const readNow=Date.now()
class ReadModelDate extends Date {
constructor(value?:string|number){super(value??readNow)}
static now(){return readNow}
}
function freshReadEngine():Engine {
const exports:Record<string,unknown>={}
runInNewContext(output,{exports,require:(name:string)=>{if(!(name in dependencies))throw Error("Unexpected engine dependency: "+name);return dependencies[name]},Buffer,console,setTimeout,clearTimeout,Date:ReadModelDate})
const next=exports as unknown as Engine
factsResetAllowed=true;mutatePayload=false;ripePruneFails=false
next.resetEngineAfterDatabaseRestore()
return next
}
const since=new Date(Date.now()-60000).toISOString()
const knownRead=freshReadEngine()
knownRead.queueParsedFlows(1,[flow]);mode="rollback"
await assert.rejects(knownRead.strictDrainEngine(async()=>{},[1,2]),/known rollback/)
assert.equal(knownRead.getEngineStats().pendingSize,1)
assert.equal(knownRead.peekPendingFlows({serverId:1})[0].bytes,100)
knownRead.queueParsedFlows(1,[{...flow,bytes:40,packets:2}])
knownRead.queueParsedFlows(2,[{...flow,src:"192.0.2.2",bytes:25}])
assert.equal(knownRead.peekPendingFlows().length,2)
assert.equal(knownRead.peekPendingFlows({serverId:1})[0].bytes,140)
assert.equal(knownRead.peekPendingFlows({serverId:1})[0].packets,3)
assert.equal(knownRead.listLiveFlowRows(since,{serverId:1})[0].bytes,140)
const detachedView=knownRead.peekPendingFlows({serverId:1});detachedView[0].bytes=99999
assert.equal(knownRead.peekPendingFlows({serverId:1})[0].bytes,140,"view mutation cannot alter either detached proof or new arrivals")
mode="normal";await knownRead.strictDrainEngine(async()=>{},[1,2])
assert.equal(knownRead.peekPendingFlows({serverId:1})[0].bytes,40)
assert.equal(knownRead.listLiveFlowRows(since,{serverId:1})[0].bytes,140,"ACK moves detached rows to recent exactly once")
await knownRead.strictDrainEngine(async()=>{},[1,2])
assert.equal(knownRead.peekPendingFlows().length,0)
assert.equal(knownRead.listLiveFlowRows(since,{serverId:1})[0].bytes,140)
assert.equal(knownRead.listLiveFlowRows(since,{serverId:2})[0].bytes,25)
const unknownRead=freshReadEngine()
unknownRead.queueParsedFlows(1,[flow]);mode="unknown"
await assert.rejects(unknownRead.strictDrainEngine(async()=>{},[1,2]),/lost COMMIT/)
unknownRead.queueParsedFlows(1,[{...flow,bytes:40}]);unknownRead.queueParsedFlows(2,[{...flow,src:"192.0.2.2",bytes:25}])
const readAttempts=commits,readDispatches=dispatches
const uncertain=(state:"running"|"unknown")=>(error:unknown)=>Boolean(error&&typeof error==="object"&&(error as {code?:string}).code==="FLOW_PENDING_UNCONFIRMED"&&(error as {batchState?:string}).batchState===state)
assert.throws(()=>unknownRead.peekPendingFlows({serverId:1}),uncertain("unknown"))
assert.throws(()=>unknownRead.listLiveFlowRows(since,{serverId:1}),uncertain("unknown"))
assert.throws(()=>unknownRead.peekPendingFlows(),uncertain("unknown"))
assert.equal(unknownRead.peekPendingFlows({serverId:2})[0].bytes,25)
assert.equal(unknownRead.listLiveFlowRows(since,{serverId:2})[0].bytes,25)
assert.equal(unknownRead.peekPendingFlows({serverId:1,sinceIso:new Date(Date.now()+3600000).toISOString()}).length,0)
assert.equal(unknownRead.peekPendingFlows({serverId:1,untilIso:new Date(Date.now()-3600000).toISOString()}).length,0)
assert.equal(commits,readAttempts);assert.equal(dispatches,readDispatches,"reads never replay/reconcile or dispatch SQL")
receiptPresent=false
assert.equal((await unknownRead.reconcileEnginePendingWrite()).state,"pending")
assert.equal(unknownRead.peekPendingFlows({serverId:1})[0].bytes,140,"barrier-proven absence restores known pending visibility")
const committedUnknown=freshReadEngine()
committedUnknown.queueParsedFlows(1,[flow]);mode="unknown"
await assert.rejects(committedUnknown.strictDrainEngine(async()=>{},[1]),/lost COMMIT/)
committedUnknown.queueParsedFlows(1,[{...flow,bytes:40}]);receiptPresent=true
assert.equal((await committedUnknown.reconcileEnginePendingWrite()).state,"committed")
assert.equal(committedUnknown.peekPendingFlows({serverId:1})[0].bytes,40,"proven committed batch is not added to SQL pending merge")
assert.equal(committedUnknown.listLiveFlowRows(since,{serverId:1})[0].bytes,140)
const runningRead=freshReadEngine()
runningRead.queueParsedFlows(1,[flow]);mode="held"
const started=new Promise<void>(resolve=>{heldStarted=resolve})
heldGate=new Promise<void>(resolve=>{releaseHeld=resolve})
const inFlight=runningRead.strictDrainEngine(async()=>{},[1,2])
await started
runningRead.queueParsedFlows(2,[{...flow,src:"192.0.2.2",bytes:25}])
assert.throws(()=>runningRead.peekPendingFlows({serverId:1}),uncertain("running"))
assert.throws(()=>runningRead.listLiveFlowRows(since,{serverId:1}),uncertain("running"))
assert.equal(runningRead.peekPendingFlows({serverId:2})[0].bytes,25)
mode="normal";releaseHeld();await inFlight
assert.equal(runningRead.peekPendingFlows({serverId:1}).length,0)
assert.equal(runningRead.listLiveFlowRows(since,{serverId:1})[0].bytes,100)
for(const options of [{serverId:0},{serverId:-1},{sinceIso:"invalid"},{untilIso:"invalid"},{sinceIso:"2026-02-02T00:00:00Z",untilIso:"2026-01-01T00:00:00Z"}])assert.throws(()=>runningRead.peekPendingFlows(options),/Invalid pending flow read scope/)
console.log("actual engine read boundary: known rollback+same-key arrivals merge once, detached view immutable, scoped running/unknown refusal, unrelated scope readable, explicit reconciliation resolves ambiguity without double count")
const cappedRead=freshReadEngine() as Engine&{configureEngine:(options:{topN:number})=>void}
cappedRead.configureEngine({topN:1})
cappedRead.queueParsedFlows(1,Array.from({length:21},(_,index)=>({...flow,src:`203.0.113.${index+1}`,bytes:index===20?1:1000})))
mode="rollback"
await assert.rejects(cappedRead.strictDrainEngine(async()=>{},[1]),/known rollback/)
assert.equal(cappedRead.peekPendingFlows({serverId:1}).length,21,"known rollback retains all source rows, including the row outside persisted topN")
assert.equal(cappedRead.peekPendingFlows({serverId:1}).find(row=>row.src==="203.0.113.21")!.bytes,1)
assert.equal(cappedRead.listLiveFlowRows(since,{serverId:1}).length,21)
console.log("actual read topN model: all detached source rows remain visible until ACK; fixed model clock avoids minute-boundary races")
// A maintenance no-op must never hide error-only evidence or an uncertain
// owned batch. Explicit normal retention work must still run on a fresh engine.
const errorOnly=freshReadEngine() as Engine&{setEngineError:(message:string)=>void}
errorOnly.setEngineError("synthetic collector error")
mode="unknown";receiptPresent=false
const errorAttempts=commits
await assert.rejects(errorOnly.strictDrainEngine(async()=>{},[]),/lost COMMIT/)
assert.equal(commits,errorAttempts+1)
await assert.rejects(errorOnly.strictDrainEngine(async()=>{},[]),/unknown/i)
assert.equal(commits,errorAttempts+1,"empty unknown batch cannot be skipped or replayed")
receiptPresent=true
assert.equal((await errorOnly.reconcileEnginePendingWrite()).state,"committed")
mode="normal"
const explicitPrune=freshReadEngine() as Engine&{flushPending:(opts:{force:boolean;prune:boolean})=>Promise<unknown>}
const pruneAttempts=commits,pruneDispatches=dispatches
await explicitPrune.flushPending({force:true,prune:true})
assert.equal(commits,pruneAttempts+1)
assert.ok(dispatches>pruneDispatches,"explicit retention/prune still dispatches on a fresh engine")
console.log("maintenance no-op: pristine/reset skip only; error-only unknown batch remains owned without replay; explicit prune preserved")
// A paused cold cache must not enqueue one pair of SQL loaders per datagram.
const coldWork = new DatabaseBackgroundWork()
let coldClock = 0
let topologyLoads = 0
let catalogLoads = 0
let catalogReady = false
let topologyReady = false
let rejectTopology!: (error: Error) => void
let releaseCatalog!: () => void
const firstTopology = new Promise<void>((_resolve, reject) => { rejectTopology = reject })
const firstCatalog = new Promise<void>(resolve => { releaseCatalog = resolve })
const coldDependencies = {
...dependencies,
"./database-background-work.js": { databaseBackgroundWork: coldWork },
"./traffic-flow-topology.js": {
peekFlowTopology: () => topologyReady ? {} : null,
peekServerCatalog: () => catalogReady ? { byId: new Map() } : null,
loadFlowTopology: async () => {
topologyLoads++
if (topologyLoads === 1) await firstTopology
topologyReady = true
},
getServerCatalog: async () => {
catalogLoads++
await firstCatalog
catalogReady = true
},
},
}
const coldExports: Record<string, unknown> = {}
runInNewContext(output, {
exports: coldExports,
require: (name: keyof typeof coldDependencies) => coldDependencies[name],
performance: { now: () => coldClock },
Buffer, console, setTimeout, clearTimeout,
})
const coldEngine = coldExports as unknown as Engine
for (let i = 0; i < 128; i++) coldEngine.queueParsedFlows(1, [])
assert.deepEqual([topologyLoads, catalogLoads], [1, 1])
let drained = false
const drain = coldWork.drain().then(() => { drained = true })
await Promise.resolve()
assert.equal(drained, false)
rejectTopology(new Error("Owned topology load refused"))
releaseCatalog()
await drain
assert.equal(drained, true)
coldClock = 999
for (let i = 0; i < 128; i++) coldEngine.queueParsedFlows(1, [])
assert.deepEqual([topologyLoads, catalogLoads], [1, 1])
coldClock = 1000
coldEngine.queueParsedFlows(1, [])
await coldWork.drain()
assert.deepEqual([topologyLoads, catalogLoads], [2, 1])
assert.equal(topologyReady, true)