Files
MikrotikManager/backend/src/services/installation-co-load.owned.test.ts
T

220 lines
17 KiB
TypeScript

/** Backend co-load fixture for the full-installation resource matrix.
* Real app bootstrap/scheduler, public MCP poll, main-owned UDP worker and HTTP.
* External frontend/portal are separate phases, not implied by a local PASS. */
import assert from "node:assert/strict"
import { randomUUID } from "node:crypto"
import { createSocket } from "node:dgram"
import { existsSync } from "node:fs"
import { createServer } from "node:http"
import { basename, dirname, resolve } from "node:path"
import { monitorEventLoopDelay, performance } from "node:perf_hooks"
import { setTimeout as delay } from "node:timers/promises"
assert.equal(process.env.MMAPP_OWN_INSTALLATION_COLOAD, "true")
assert.equal(process.env.NODE_ENV, "test")
const endpoint = new URL(process.env.DATABASE_URL ?? "")
assert(["postgres:", "postgresql:"].includes(endpoint.protocol))
assert.equal(endpoint.hostname, "127.0.0.1"); assert.equal(endpoint.username, "mcp_test"); assert(endpoint.password)
assert.equal(endpoint.pathname, "/postgres"); assert.equal(endpoint.search, ""); assert.equal(endpoint.hash, "")
assert(Number.isInteger(Number(endpoint.port)) && Number(endpoint.port) > 0 && Number(endpoint.port) <= 65535)
const state = resolve(process.env.MCP_STATE_DIR ?? "")
assert.equal(basename(state), "state"); assert.match(basename(dirname(state)), /^mm-mcp-test-[A-Za-z0-9-]+$/)
assert.equal(resolve(process.cwd()), dirname(state))
assert.match(basename(dirname(dirname(new URL(import.meta.url).pathname))), /^dist-installation-memory(?:-[a-z0-9-]{3,64})?$/)
assert(!/tsx|--import|--require|--loader/.test(process.execArgv.join(" ")))
assert(process.execArgv.includes("--max-old-space-size=192") && process.execArgv.includes("--max-semi-space-size=16"))
assert.equal(process.env.AUTH_REQUIRED, "true"); assert(process.env.AUTH_JWT_SECRET)
assert.equal(process.env.MM_FLOW_READ_RESOURCE_PROFILE, "constrained")
assert.equal(existsSync(resolve(process.env.DATABASE_PATH ?? "")), false)
process.env.FLOW_LISTEN_HOST = "127.0.0.1"
process.env.MMAPP_OWN_SERVER_POLL_STORAGE_FRESH = "true"
process.env.MMAPP_OWN_SQL_DB = "postgres"
const { pool, closePool } = await import("../db/index.js")
const { initDatabase } = await import("../db/bootstrap.js")
const { buildApp } = await import("../index.js")
const ingest = await import("./traffic-flow-ingest.js")
const settings = await import("./traffic-flow-settings.js")
const { pauseRipeBackgroundWork } = await import("./traffic-flow-ripe.js")
const { acceptServerPollPublicRuntimeSql } = await import("../modules/mcp/server-poll-http.sql-helper.js")
const { createMcpConnection, authenticateMcpToken } = await import("../modules/mcp/store.js")
const { invokeMcpTool } = await import("../modules/mcp/runtime.js")
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
const uptime = await import("./uptime-collector.js")
const traffic = await import("./traffic-collector.js")
const internetPath = await import("./internet-path-collector.js")
const { isAnyAlertSnapshotSourceJobRunning } = await import("./scheduler-running.js")
const { withApplicationCollectorLock, withApplicationMutationLock, captureConfigurationLockHandoff, assertNoConfigurationLockContext } = await import("./config-mutation-lock.js")
let nativeRequests = 0
const sender = createSocket("udp4"), native = createServer((request, response) => {
nativeRequests++
if (request.method !== "GET" || request.url !== "/rest/interface") { response.writeHead(404).end(); return }
response.setHeader("content-type", "application/json"); response.end("[]")
})
const controller = new AbortController(), loop = monitorEventLoopDelay({ resolution: 20 })
const metrics = { sent: 0, reads: 0, readFailures: 0, maxReadMs: 0, readerRefusals: 0, samples: 0,
peakRss: 0, peakHeap: 0, peakExternal: 0, maxPoolWaiting: 0, maxPoolTotal: 0, maxFlowPending: 0 }
let app: Awaited<ReturnType<typeof buildApp>> | undefined, flowServerId: number | undefined
let background: Promise<unknown>[] = [], backgroundError: unknown, sampler: ReturnType<typeof setInterval> | undefined
const sample = () => {
const memory = process.memoryUsage(), health = ingest.getFlowWorkerHealth()
metrics.samples++; metrics.peakRss = Math.max(metrics.peakRss, memory.rss); metrics.peakHeap = Math.max(metrics.peakHeap, memory.heapUsed)
metrics.peakExternal = Math.max(metrics.peakExternal, memory.external); metrics.maxFlowPending = Math.max(metrics.maxFlowPending, health.pendingSize)
metrics.maxPoolWaiting = Math.max(metrics.maxPoolWaiting, pool.waitingCount); metrics.maxPoolTotal = Math.max(metrics.maxPoolTotal, pool.totalCount)
}
const until = async (work: () => boolean | Promise<boolean>, description: string) => {
const deadline = performance.now() + 15000
while (!await work()) { if (backgroundError) throw backgroundError; assert(performance.now() < deadline, description); await delay(25) }
}
function packet(sequence: number) {
const bytes = Buffer.alloc(72), identity = sequence % 1024
bytes.writeUInt16BE(5,0); bytes.writeUInt16BE(1,2); bytes.writeUInt32BE(Math.floor(Date.now()/1000),8); bytes.writeUInt32BE(sequence,16)
bytes.set([10,144,identity>>8,identity&255],24); bytes.set([10,145,0,1],28)
bytes.writeUInt16BE(1,36); bytes.writeUInt32BE(3,40); bytes.writeUInt32BE(1500,44)
bytes.writeUInt16BE(49152+identity,56); bytes.writeUInt16BE(443,58); bytes.writeUInt8(6,62)
return bytes
}
try {
loop.enable(); sampler = setInterval(sample,250); sampler.unref()
await initDatabase()
assert.equal(existsSync(resolve(process.env.DATABASE_PATH!)), false)
await new Promise<void>(done => native.listen(0,"127.0.0.2",done))
flowServerId = Number((await pool.query(`INSERT INTO servers(name,host,port,use_ssl,verify_ssl,username,password,enabled)
VALUES('owned-coload-flow','127.0.0.2',$1,false,false,'owned-flow','owned-flow',false) RETURNING id`, [(native.address() as {port:number}).port])).rows[0].id)
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,event_at,received_at,effective_at,topics,level,message,generation)
SELECT $1,'owned-coload-flow','127.0.0.1','owned-'||n::text,'owned-time',now()-n*interval '1 millisecond',now(),now()-n*interval '1 millisecond',
ARRAY['info','routing'],'info',repeat('owned log ',52),1 FROM generate_series(1,2048) AS n`,[flowServerId])
const portSocket = createSocket("udp4")
await new Promise<void>(done => portSocket.bind(0,"127.0.0.1",done))
const port = portSocket.address().port; await new Promise<void>(done => portSocket.close(done))
await new Promise<void>(done=>sender.bind(0,"127.0.0.2",done))
await settings.getTrafficFlowSettingsRow()
await pool.query(`UPDATE traffic_flow_settings SET enabled=true,collector_ip='127.0.0.1',flow_listen_port=$1,
host_private_key='owned-coload-private',host_public_key='owned-coload-public',peers_json=$2::jsonb WHERE id=1`,
[port,JSON.stringify([{serverId:flowServerId,name:"Owned co-load",publicKey:"owned-coload-peer",address:"127.0.0.2",allowedIps:["127.0.0.2/32"]}])])
settings.invalidateTrafficFlowSettingsCache()
await pool.query("UPDATE mcp_settings SET enabled=true WHERE id=1")
const settingsSnapshot = async () => (await pool.query(`SELECT jsonb_build_object(
'uptime',(SELECT to_jsonb(t) FROM uptime_settings t WHERE id=1),
'traffic',(SELECT to_jsonb(t) FROM traffic_settings t WHERE id=1),
'internetPath',(SELECT to_jsonb(t) FROM internet_path_settings t WHERE id=1)) AS value`)).rows[0].value
const beforeBusy = await settingsSnapshot(), owner = await pool.connect()
let unsafeOwner = false
try {
await owner.query("SELECT pg_advisory_lock(hashtextextended('mm-config:application',0))")
for (const collect of [uptime.collectResourceSamplesOnce,uptime.collectPingProbesOnce,traffic.collectTrafficOnce,internetPath.collectInternetPathSnapshotOnce])
await assert.rejects(collect(),error=>(error as {statusCode?:number}).statusCode===409)
assert.equal(nativeRequests,0,"Busy collector must refuse before native work")
assert.deepEqual(await settingsSnapshot(),beforeBusy,"Busy collector must not publish success or failure settings")
assert.equal(uptime.getUptimeCollectorsState().resources,false)
assert.equal(traffic.isTrafficCollecting(),false); assert.equal(internetPath.isInternetPathCollecting(),false)
} finally {
try { assert.equal((await owner.query("SELECT pg_advisory_unlock(hashtextextended('mm-config:application',0)) AS released")).rows[0].released,true) }
catch { unsafeOwner=true }
owner.release(unsafeOwner)
}
let entered = 0, releaseReaders!: () => void
const readerGate = new Promise<void>(done => { releaseReaders=done }), handoffs: Promise<void>[] = []
const readerTasks = [0,1].map(()=>withApplicationCollectorLock(async()=>{
entered++; assert.throws(assertNoConfigurationLockContext)
const handoff=captureConfigurationLockHandoff(); assert(handoff); handoffs.push(handoff)
await withApplicationCollectorLock(async()=>{})
await assert.rejects(withApplicationMutationLock(async()=>assert.fail("Shared collector cannot upgrade to configuration mutation")),error=>(error as {statusCode?:number}).statusCode===409)
await readerGate
}))
for(const task of readerTasks)void task.catch(()=>{})
try {
await until(()=>entered===2,"Independent collector readers must coexist")
await assert.rejects(withApplicationMutationLock(async()=>assert.fail("Mutation must exclude active collectors")),error=>(error as {statusCode?:number}).statusCode===409)
} finally { releaseReaders(); await Promise.all(readerTasks) }
await Promise.all(handoffs)
assert.equal(await withApplicationMutationLock(()=>withApplicationCollectorLock(async()=>"exclusive-owner-reentry")),"exclusive-owner-reentry")
app = await buildApp({logger:false,startScheduler:true})
await app.listen({port:0,host:"127.0.0.1"})
await until(()=>ingest.getFlowListenerState().bound,"Production main-owned flow worker did not bind")
await until(()=>!isAnyAlertSnapshotSourceJobRunning(),"Initial automatic source jobs did not settle before the measured phase")
const base = app.server.address(); assert(base && typeof base !== "string")
const origin = `http://127.0.0.1:${base.port}`
const actor = {id:"owned-coload-"+randomUUID(),email:"[email protected]",name:"Owned co-load",apps:["mm"],
permissions:["mm:servers:read","mm:servers:write","mm:network:read","mm:traffic:read","mm:mcp:read","mm:mcp:manage","mm:mcp:admin"],issuedAt:Date.now()-1000,expiresAt:Date.now()+3600000}
const jwt = app.jwt.sign({iss:process.env.AUTH_ISSUER??"https://auth.shnt.top",sub:actor.id,email:actor.email,name:actor.name,apps:actor.apps,permissions:actor.permissions,iat:Math.floor(actor.issuedAt/1000),exp:Math.floor(actor.expiresAt/1000)})
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({name:"Owned co-load observer",mode:"observer",expiresAt:new Date(actor.expiresAt).toISOString(),permissions:["mm:servers:read"],scope:{serverIds:[flowServerId],operationIds:["servers.list"],allowGlobal:false},limits:{callsPerMinute:120,mutationsPerHour:1,serversPerPlan:1,objectsPerPlan:1}}),actor)
const observer = await authenticateMcpToken(access.token)
const watch = (work: Promise<unknown>) => work.catch(error => { backgroundError ??= error; controller.abort() })
background = [watch((async()=>{ while(!controller.signal.aborted) {
await new Promise<void>((done,reject)=>sender.send(packet(metrics.sent+1),port,"127.0.0.1",error=>error?reject(error):done()))
metrics.sent++; await delay(5)
} })())]
const read = async (path: string) => {
const began = performance.now(), response = await fetch(origin+path,{headers:{authorization:"Bearer "+jwt},signal:AbortSignal.timeout(10000)})
assert.equal(response.status,200,`HTTP co-load read failed: ${path}`)
const reader = response.body?.getReader(); assert(reader); let bytes = 0
for (;;) { const part=await reader.read(); if(part.done)break; bytes+=part.value.byteLength; assert(bytes<=2*1024*1024,"Bounded HTTP co-load response overflow") }
metrics.reads++; metrics.maxReadMs=Math.max(metrics.maxReadMs,performance.now()-began)
}
for (const path of ["/api/mcp/overview",`/api/logs?serverId=${flowServerId}`]) background.push(watch((async()=>{
while(!controller.signal.aborted) { try { await read(path) } catch(error) { metrics.readFailures++; throw error }; await delay(1000) }
})()))
background.push(watch((async()=>{ while(!controller.signal.aborted) {
try { await invokeMcpTool(observer,"mm_servers_list",{}); metrics.reads++ } catch(error) {
if ((error as {code?:string}).code!=="RATE_LIMITED") throw error
metrics.readerRefusals++
}
await delay(2000)
} })()))
await until(()=>ingest.getFlowWorkerHealth().packetsReceived>0,"No genuine UDP arrived")
const delayedApproval = process.env.MMAPP_OWN_COLOAD_DELAYED_APPROVAL === "true"
let beforeApprovalRuntimeChanged = false
const poll = await acceptServerPollPublicRuntimeSql({concurrentApplicationReads:true,...(delayedApproval ? {
beforeHumanApproval: async () => {
const before = await settingsSnapshot()
await delay(20000)
await until(()=>!uptime.getUptimeCollectorsState().resources,"Resource collector did not settle before delayed approval")
const sample = await uptime.collectResourceSamplesOnce()
assert.equal(sample.skipped, undefined)
beforeApprovalRuntimeChanged = JSON.stringify(before) !== JSON.stringify(await settingsSnapshot())
assert(beforeApprovalRuntimeChanged,"Delayed approval must observe actual collector runtime writes after planning")
console.log("Owned delayed approval:",JSON.stringify({waitMs:20000,actualCollectorRuntimeChanged:true}))
}
} : {})})
if(delayedApproval){assert.equal(poll.delayedApprovalUsabilityAccepted,true);assert("innerPlanVersion" in poll);assert.equal(poll.innerPlanVersion,3)}
controller.abort(); await Promise.all(background)
if(backgroundError)throw backgroundError
await until(()=>ingest.getFlowWorkerHealth().packetsReceived===metrics.sent,"UDP receive count did not catch up before drain")
const drain = await ingest.drainExistingTrafficFlowWriters(async()=>{},[flowServerId])
assert(drain.existingWorker.workerPresent)
const aggregateTotals = async () => (await pool.query("SELECT coalesce(sum(bytes),0)::text AS bytes,coalesce(sum(packets),0)::text AS packets FROM flow_minute_stats WHERE server_id=$1",[flowServerId])).rows[0]
await until(async()=>{
const totals=await aggregateTotals()
return Number(totals.bytes)===metrics.sent*1500&&Number(totals.packets)===metrics.sent*3
},"Lossless minute counters diverged during MCP maintenance")
const totals=await aggregateTotals(), detail=(await pool.query("SELECT coalesce(sum(bytes),0)::text AS bytes,coalesce(sum(packets),0)::text AS packets,count(*)::int AS buckets FROM flow_buckets WHERE server_id=$1",[flowServerId])).rows[0]
assert(Number(detail.bytes)>0&&Number(detail.bytes)<=Number(totals.bytes))
assert(Number(detail.packets)>0&&Number(detail.packets)<=Number(totals.packets))
const health=ingest.getFlowWorkerHealth(); assert.equal(health.dropped,0)
assert(metrics.reads>=4 && metrics.sent>=10)
sample()
console.log("Owned installation backend co-load:",JSON.stringify({backendCoLoadAccepted:true,staleApprovalSafetyAccepted:false,delayedApprovalUsabilityAccepted:delayedApproval&&beforeApprovalRuntimeChanged&&poll.delayedApprovalUsabilityAccepted,wholeInstallation2GbAccepted:false,
delayedHumanApprovalScenario:delayedApproval,beforeApprovalRuntimeChanged,schedulerStarted:true,genuineMainWorker:true,collectorBusyChecks:4,collectorReadersConcurrent:true,collectorUpgradeRejected:true,busyRefusedBeforeNativeAndSettings:true,udpTargetPerSecond:200,flowCardinality:1024,distinctSyntheticExporterAddress:true,seededRouterLogEntries:2048,metrics,poll,health,
losslessCounterSource:"flow_minute_stats",exactAggregateTotals:totals,boundedTopNDetail:detail,
eventLoopP99Ms:loop.percentile(99)/1e6,eventLoopMaxMs:loop.max/1e6,frontendExercised:false,portalExercised:false,
geoipDatabaseLoaded:false,sqlObserverInstalled:false,productionDatabaseTouched:false,realRouterTouched:false}))
} catch(error) {
let storedTotalsByServer:unknown="unavailable"
try { storedTotalsByServer=(await pool.query("SELECT server_id,sum(bytes)::text AS bytes,sum(packets)::text AS packets FROM flow_minute_stats GROUP BY server_id ORDER BY server_id LIMIT 10")).rows } catch { /* Keep original failure; diagnostic queries are never admission or recovery. */ }
console.log("Owned installation co-load failure:",JSON.stringify({metrics,health:ingest.getFlowWorkerHealth(),flowServerId,storedTotalsByServer,wholeInstallation2GbAccepted:false}))
throw error
} finally {
controller.abort(); await Promise.allSettled(background)
if(sampler)clearInterval(sampler);loop.disable()
try {
await ingest.stopTrafficFlowListenerForDatabaseOperation()
await pauseRipeBackgroundWork()
await app?.close()
} finally {
try { sender.close() } catch { /* setup may precede socket use */ }
native.closeAllConnections(); if(native.listening)await new Promise<void>(done=>native.close(()=>done()))
await closePool()
}
}