220 lines
17 KiB
TypeScript
220 lines
17 KiB
TypeScript
/** Backend co-load fixture for the full-installation resource matrix.
|
|
* Real app bootstrap/scheduler, public MCP poll, main-owned UDP worker and HTTP.
|
|
* External frontend/portal are separate phases, not implied by a local PASS. */
|
|
import assert from "node:assert/strict"
|
|
import { randomUUID } from "node:crypto"
|
|
import { createSocket } from "node:dgram"
|
|
import { existsSync } from "node:fs"
|
|
import { createServer } from "node:http"
|
|
import { basename, dirname, resolve } from "node:path"
|
|
import { monitorEventLoopDelay, performance } from "node:perf_hooks"
|
|
import { setTimeout as delay } from "node:timers/promises"
|
|
|
|
assert.equal(process.env.MMAPP_OWN_INSTALLATION_COLOAD, "true")
|
|
assert.equal(process.env.NODE_ENV, "test")
|
|
const endpoint = new URL(process.env.DATABASE_URL ?? "")
|
|
assert(["postgres:", "postgresql:"].includes(endpoint.protocol))
|
|
assert.equal(endpoint.hostname, "127.0.0.1"); assert.equal(endpoint.username, "mcp_test"); assert(endpoint.password)
|
|
assert.equal(endpoint.pathname, "/postgres"); assert.equal(endpoint.search, ""); assert.equal(endpoint.hash, "")
|
|
assert(Number.isInteger(Number(endpoint.port)) && Number(endpoint.port) > 0 && Number(endpoint.port) <= 65535)
|
|
const state = resolve(process.env.MCP_STATE_DIR ?? "")
|
|
assert.equal(basename(state), "state"); assert.match(basename(dirname(state)), /^mm-mcp-test-[A-Za-z0-9-]+$/)
|
|
assert.equal(resolve(process.cwd()), dirname(state))
|
|
assert.match(basename(dirname(dirname(new URL(import.meta.url).pathname))), /^dist-installation-memory(?:-[a-z0-9-]{3,64})?$/)
|
|
assert(!/tsx|--import|--require|--loader/.test(process.execArgv.join(" ")))
|
|
assert(process.execArgv.includes("--max-old-space-size=192") && process.execArgv.includes("--max-semi-space-size=16"))
|
|
assert.equal(process.env.AUTH_REQUIRED, "true"); assert(process.env.AUTH_JWT_SECRET)
|
|
assert.equal(process.env.MM_FLOW_READ_RESOURCE_PROFILE, "constrained")
|
|
assert.equal(existsSync(resolve(process.env.DATABASE_PATH ?? "")), false)
|
|
process.env.FLOW_LISTEN_HOST = "127.0.0.1"
|
|
process.env.MMAPP_OWN_SERVER_POLL_STORAGE_FRESH = "true"
|
|
process.env.MMAPP_OWN_SQL_DB = "postgres"
|
|
|
|
const { pool, closePool } = await import("../db/index.js")
|
|
const { initDatabase } = await import("../db/bootstrap.js")
|
|
const { buildApp } = await import("../index.js")
|
|
const ingest = await import("./traffic-flow-ingest.js")
|
|
const settings = await import("./traffic-flow-settings.js")
|
|
const { pauseRipeBackgroundWork } = await import("./traffic-flow-ripe.js")
|
|
const { acceptServerPollPublicRuntimeSql } = await import("../modules/mcp/server-poll-http.sql-helper.js")
|
|
const { createMcpConnection, authenticateMcpToken } = await import("../modules/mcp/store.js")
|
|
const { invokeMcpTool } = await import("../modules/mcp/runtime.js")
|
|
const { mcpConnectionCreateSchema } = await import("@mmapp/contracts/mcp")
|
|
const uptime = await import("./uptime-collector.js")
|
|
const traffic = await import("./traffic-collector.js")
|
|
const internetPath = await import("./internet-path-collector.js")
|
|
const { isAnyAlertSnapshotSourceJobRunning } = await import("./scheduler-running.js")
|
|
const { withApplicationCollectorLock, withApplicationMutationLock, captureConfigurationLockHandoff, assertNoConfigurationLockContext } = await import("./config-mutation-lock.js")
|
|
let nativeRequests = 0
|
|
const sender = createSocket("udp4"), native = createServer((request, response) => {
|
|
nativeRequests++
|
|
if (request.method !== "GET" || request.url !== "/rest/interface") { response.writeHead(404).end(); return }
|
|
response.setHeader("content-type", "application/json"); response.end("[]")
|
|
})
|
|
const controller = new AbortController(), loop = monitorEventLoopDelay({ resolution: 20 })
|
|
const metrics = { sent: 0, reads: 0, readFailures: 0, maxReadMs: 0, readerRefusals: 0, samples: 0,
|
|
peakRss: 0, peakHeap: 0, peakExternal: 0, maxPoolWaiting: 0, maxPoolTotal: 0, maxFlowPending: 0 }
|
|
let app: Awaited<ReturnType<typeof buildApp>> | undefined, flowServerId: number | undefined
|
|
let background: Promise<unknown>[] = [], backgroundError: unknown, sampler: ReturnType<typeof setInterval> | undefined
|
|
const sample = () => {
|
|
const memory = process.memoryUsage(), health = ingest.getFlowWorkerHealth()
|
|
metrics.samples++; metrics.peakRss = Math.max(metrics.peakRss, memory.rss); metrics.peakHeap = Math.max(metrics.peakHeap, memory.heapUsed)
|
|
metrics.peakExternal = Math.max(metrics.peakExternal, memory.external); metrics.maxFlowPending = Math.max(metrics.maxFlowPending, health.pendingSize)
|
|
metrics.maxPoolWaiting = Math.max(metrics.maxPoolWaiting, pool.waitingCount); metrics.maxPoolTotal = Math.max(metrics.maxPoolTotal, pool.totalCount)
|
|
}
|
|
const until = async (work: () => boolean | Promise<boolean>, description: string) => {
|
|
const deadline = performance.now() + 15000
|
|
while (!await work()) { if (backgroundError) throw backgroundError; assert(performance.now() < deadline, description); await delay(25) }
|
|
}
|
|
function packet(sequence: number) {
|
|
const bytes = Buffer.alloc(72), identity = sequence % 1024
|
|
bytes.writeUInt16BE(5,0); bytes.writeUInt16BE(1,2); bytes.writeUInt32BE(Math.floor(Date.now()/1000),8); bytes.writeUInt32BE(sequence,16)
|
|
bytes.set([10,144,identity>>8,identity&255],24); bytes.set([10,145,0,1],28)
|
|
bytes.writeUInt16BE(1,36); bytes.writeUInt32BE(3,40); bytes.writeUInt32BE(1500,44)
|
|
bytes.writeUInt16BE(49152+identity,56); bytes.writeUInt16BE(443,58); bytes.writeUInt8(6,62)
|
|
return bytes
|
|
}
|
|
try {
|
|
loop.enable(); sampler = setInterval(sample,250); sampler.unref()
|
|
await initDatabase()
|
|
assert.equal(existsSync(resolve(process.env.DATABASE_PATH!)), false)
|
|
await new Promise<void>(done => native.listen(0,"127.0.0.2",done))
|
|
flowServerId = Number((await pool.query(`INSERT INTO servers(name,host,port,use_ssl,verify_ssl,username,password,enabled)
|
|
VALUES('owned-coload-flow','127.0.0.2',$1,false,false,'owned-flow','owned-flow',false) RETURNING id`, [(native.address() as {port:number}).port])).rows[0].id)
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,event_at,received_at,effective_at,topics,level,message,generation)
|
|
SELECT $1,'owned-coload-flow','127.0.0.1','owned-'||n::text,'owned-time',now()-n*interval '1 millisecond',now(),now()-n*interval '1 millisecond',
|
|
ARRAY['info','routing'],'info',repeat('owned log ',52),1 FROM generate_series(1,2048) AS n`,[flowServerId])
|
|
const portSocket = createSocket("udp4")
|
|
await new Promise<void>(done => portSocket.bind(0,"127.0.0.1",done))
|
|
const port = portSocket.address().port; await new Promise<void>(done => portSocket.close(done))
|
|
await new Promise<void>(done=>sender.bind(0,"127.0.0.2",done))
|
|
await settings.getTrafficFlowSettingsRow()
|
|
await pool.query(`UPDATE traffic_flow_settings SET enabled=true,collector_ip='127.0.0.1',flow_listen_port=$1,
|
|
host_private_key='owned-coload-private',host_public_key='owned-coload-public',peers_json=$2::jsonb WHERE id=1`,
|
|
[port,JSON.stringify([{serverId:flowServerId,name:"Owned co-load",publicKey:"owned-coload-peer",address:"127.0.0.2",allowedIps:["127.0.0.2/32"]}])])
|
|
settings.invalidateTrafficFlowSettingsCache()
|
|
await pool.query("UPDATE mcp_settings SET enabled=true WHERE id=1")
|
|
const settingsSnapshot = async () => (await pool.query(`SELECT jsonb_build_object(
|
|
'uptime',(SELECT to_jsonb(t) FROM uptime_settings t WHERE id=1),
|
|
'traffic',(SELECT to_jsonb(t) FROM traffic_settings t WHERE id=1),
|
|
'internetPath',(SELECT to_jsonb(t) FROM internet_path_settings t WHERE id=1)) AS value`)).rows[0].value
|
|
const beforeBusy = await settingsSnapshot(), owner = await pool.connect()
|
|
let unsafeOwner = false
|
|
try {
|
|
await owner.query("SELECT pg_advisory_lock(hashtextextended('mm-config:application',0))")
|
|
for (const collect of [uptime.collectResourceSamplesOnce,uptime.collectPingProbesOnce,traffic.collectTrafficOnce,internetPath.collectInternetPathSnapshotOnce])
|
|
await assert.rejects(collect(),error=>(error as {statusCode?:number}).statusCode===409)
|
|
assert.equal(nativeRequests,0,"Busy collector must refuse before native work")
|
|
assert.deepEqual(await settingsSnapshot(),beforeBusy,"Busy collector must not publish success or failure settings")
|
|
assert.equal(uptime.getUptimeCollectorsState().resources,false)
|
|
assert.equal(traffic.isTrafficCollecting(),false); assert.equal(internetPath.isInternetPathCollecting(),false)
|
|
} finally {
|
|
try { assert.equal((await owner.query("SELECT pg_advisory_unlock(hashtextextended('mm-config:application',0)) AS released")).rows[0].released,true) }
|
|
catch { unsafeOwner=true }
|
|
owner.release(unsafeOwner)
|
|
}
|
|
let entered = 0, releaseReaders!: () => void
|
|
const readerGate = new Promise<void>(done => { releaseReaders=done }), handoffs: Promise<void>[] = []
|
|
const readerTasks = [0,1].map(()=>withApplicationCollectorLock(async()=>{
|
|
entered++; assert.throws(assertNoConfigurationLockContext)
|
|
const handoff=captureConfigurationLockHandoff(); assert(handoff); handoffs.push(handoff)
|
|
await withApplicationCollectorLock(async()=>{})
|
|
await assert.rejects(withApplicationMutationLock(async()=>assert.fail("Shared collector cannot upgrade to configuration mutation")),error=>(error as {statusCode?:number}).statusCode===409)
|
|
await readerGate
|
|
}))
|
|
for(const task of readerTasks)void task.catch(()=>{})
|
|
try {
|
|
await until(()=>entered===2,"Independent collector readers must coexist")
|
|
await assert.rejects(withApplicationMutationLock(async()=>assert.fail("Mutation must exclude active collectors")),error=>(error as {statusCode?:number}).statusCode===409)
|
|
} finally { releaseReaders(); await Promise.all(readerTasks) }
|
|
await Promise.all(handoffs)
|
|
assert.equal(await withApplicationMutationLock(()=>withApplicationCollectorLock(async()=>"exclusive-owner-reentry")),"exclusive-owner-reentry")
|
|
app = await buildApp({logger:false,startScheduler:true})
|
|
await app.listen({port:0,host:"127.0.0.1"})
|
|
await until(()=>ingest.getFlowListenerState().bound,"Production main-owned flow worker did not bind")
|
|
await until(()=>!isAnyAlertSnapshotSourceJobRunning(),"Initial automatic source jobs did not settle before the measured phase")
|
|
const base = app.server.address(); assert(base && typeof base !== "string")
|
|
const origin = `http://127.0.0.1:${base.port}`
|
|
const actor = {id:"owned-coload-"+randomUUID(),email:"[email protected]",name:"Owned co-load",apps:["mm"],
|
|
permissions:["mm:servers:read","mm:servers:write","mm:network:read","mm:traffic:read","mm:mcp:read","mm:mcp:manage","mm:mcp:admin"],issuedAt:Date.now()-1000,expiresAt:Date.now()+3600000}
|
|
const jwt = app.jwt.sign({iss:process.env.AUTH_ISSUER??"https://auth.shnt.top",sub:actor.id,email:actor.email,name:actor.name,apps:actor.apps,permissions:actor.permissions,iat:Math.floor(actor.issuedAt/1000),exp:Math.floor(actor.expiresAt/1000)})
|
|
const access = await createMcpConnection(mcpConnectionCreateSchema.parse({name:"Owned co-load observer",mode:"observer",expiresAt:new Date(actor.expiresAt).toISOString(),permissions:["mm:servers:read"],scope:{serverIds:[flowServerId],operationIds:["servers.list"],allowGlobal:false},limits:{callsPerMinute:120,mutationsPerHour:1,serversPerPlan:1,objectsPerPlan:1}}),actor)
|
|
const observer = await authenticateMcpToken(access.token)
|
|
const watch = (work: Promise<unknown>) => work.catch(error => { backgroundError ??= error; controller.abort() })
|
|
background = [watch((async()=>{ while(!controller.signal.aborted) {
|
|
await new Promise<void>((done,reject)=>sender.send(packet(metrics.sent+1),port,"127.0.0.1",error=>error?reject(error):done()))
|
|
metrics.sent++; await delay(5)
|
|
} })())]
|
|
const read = async (path: string) => {
|
|
const began = performance.now(), response = await fetch(origin+path,{headers:{authorization:"Bearer "+jwt},signal:AbortSignal.timeout(10000)})
|
|
assert.equal(response.status,200,`HTTP co-load read failed: ${path}`)
|
|
const reader = response.body?.getReader(); assert(reader); let bytes = 0
|
|
for (;;) { const part=await reader.read(); if(part.done)break; bytes+=part.value.byteLength; assert(bytes<=2*1024*1024,"Bounded HTTP co-load response overflow") }
|
|
metrics.reads++; metrics.maxReadMs=Math.max(metrics.maxReadMs,performance.now()-began)
|
|
}
|
|
for (const path of ["/api/mcp/overview",`/api/logs?serverId=${flowServerId}`]) background.push(watch((async()=>{
|
|
while(!controller.signal.aborted) { try { await read(path) } catch(error) { metrics.readFailures++; throw error }; await delay(1000) }
|
|
})()))
|
|
background.push(watch((async()=>{ while(!controller.signal.aborted) {
|
|
try { await invokeMcpTool(observer,"mm_servers_list",{}); metrics.reads++ } catch(error) {
|
|
if ((error as {code?:string}).code!=="RATE_LIMITED") throw error
|
|
metrics.readerRefusals++
|
|
}
|
|
await delay(2000)
|
|
} })()))
|
|
await until(()=>ingest.getFlowWorkerHealth().packetsReceived>0,"No genuine UDP arrived")
|
|
const delayedApproval = process.env.MMAPP_OWN_COLOAD_DELAYED_APPROVAL === "true"
|
|
let beforeApprovalRuntimeChanged = false
|
|
const poll = await acceptServerPollPublicRuntimeSql({concurrentApplicationReads:true,...(delayedApproval ? {
|
|
beforeHumanApproval: async () => {
|
|
const before = await settingsSnapshot()
|
|
await delay(20000)
|
|
await until(()=>!uptime.getUptimeCollectorsState().resources,"Resource collector did not settle before delayed approval")
|
|
const sample = await uptime.collectResourceSamplesOnce()
|
|
assert.equal(sample.skipped, undefined)
|
|
beforeApprovalRuntimeChanged = JSON.stringify(before) !== JSON.stringify(await settingsSnapshot())
|
|
assert(beforeApprovalRuntimeChanged,"Delayed approval must observe actual collector runtime writes after planning")
|
|
console.log("Owned delayed approval:",JSON.stringify({waitMs:20000,actualCollectorRuntimeChanged:true}))
|
|
}
|
|
} : {})})
|
|
if(delayedApproval){assert.equal(poll.delayedApprovalUsabilityAccepted,true);assert("innerPlanVersion" in poll);assert.equal(poll.innerPlanVersion,3)}
|
|
controller.abort(); await Promise.all(background)
|
|
if(backgroundError)throw backgroundError
|
|
await until(()=>ingest.getFlowWorkerHealth().packetsReceived===metrics.sent,"UDP receive count did not catch up before drain")
|
|
const drain = await ingest.drainExistingTrafficFlowWriters(async()=>{},[flowServerId])
|
|
assert(drain.existingWorker.workerPresent)
|
|
const aggregateTotals = async () => (await pool.query("SELECT coalesce(sum(bytes),0)::text AS bytes,coalesce(sum(packets),0)::text AS packets FROM flow_minute_stats WHERE server_id=$1",[flowServerId])).rows[0]
|
|
await until(async()=>{
|
|
const totals=await aggregateTotals()
|
|
return Number(totals.bytes)===metrics.sent*1500&&Number(totals.packets)===metrics.sent*3
|
|
},"Lossless minute counters diverged during MCP maintenance")
|
|
const totals=await aggregateTotals(), detail=(await pool.query("SELECT coalesce(sum(bytes),0)::text AS bytes,coalesce(sum(packets),0)::text AS packets,count(*)::int AS buckets FROM flow_buckets WHERE server_id=$1",[flowServerId])).rows[0]
|
|
assert(Number(detail.bytes)>0&&Number(detail.bytes)<=Number(totals.bytes))
|
|
assert(Number(detail.packets)>0&&Number(detail.packets)<=Number(totals.packets))
|
|
const health=ingest.getFlowWorkerHealth(); assert.equal(health.dropped,0)
|
|
assert(metrics.reads>=4 && metrics.sent>=10)
|
|
sample()
|
|
console.log("Owned installation backend co-load:",JSON.stringify({backendCoLoadAccepted:true,staleApprovalSafetyAccepted:false,delayedApprovalUsabilityAccepted:delayedApproval&&beforeApprovalRuntimeChanged&&poll.delayedApprovalUsabilityAccepted,wholeInstallation2GbAccepted:false,
|
|
delayedHumanApprovalScenario:delayedApproval,beforeApprovalRuntimeChanged,schedulerStarted:true,genuineMainWorker:true,collectorBusyChecks:4,collectorReadersConcurrent:true,collectorUpgradeRejected:true,busyRefusedBeforeNativeAndSettings:true,udpTargetPerSecond:200,flowCardinality:1024,distinctSyntheticExporterAddress:true,seededRouterLogEntries:2048,metrics,poll,health,
|
|
losslessCounterSource:"flow_minute_stats",exactAggregateTotals:totals,boundedTopNDetail:detail,
|
|
eventLoopP99Ms:loop.percentile(99)/1e6,eventLoopMaxMs:loop.max/1e6,frontendExercised:false,portalExercised:false,
|
|
geoipDatabaseLoaded:false,sqlObserverInstalled:false,productionDatabaseTouched:false,realRouterTouched:false}))
|
|
} catch(error) {
|
|
let storedTotalsByServer:unknown="unavailable"
|
|
try { storedTotalsByServer=(await pool.query("SELECT server_id,sum(bytes)::text AS bytes,sum(packets)::text AS packets FROM flow_minute_stats GROUP BY server_id ORDER BY server_id LIMIT 10")).rows } catch { /* Keep original failure; diagnostic queries are never admission or recovery. */ }
|
|
console.log("Owned installation co-load failure:",JSON.stringify({metrics,health:ingest.getFlowWorkerHealth(),flowServerId,storedTotalsByServer,wholeInstallation2GbAccepted:false}))
|
|
throw error
|
|
} finally {
|
|
controller.abort(); await Promise.allSettled(background)
|
|
if(sampler)clearInterval(sampler);loop.disable()
|
|
try {
|
|
await ingest.stopTrafficFlowListenerForDatabaseOperation()
|
|
await pauseRipeBackgroundWork()
|
|
await app?.close()
|
|
} finally {
|
|
try { sender.close() } catch { /* setup may precede socket use */ }
|
|
native.closeAllConnections(); if(native.listening)await new Promise<void>(done=>native.close(()=>done()))
|
|
await closePool()
|
|
}
|
|
}
|