227 lines
16 KiB
TypeScript
227 lines
16 KiB
TypeScript
/** Integration/performance tests use a fresh, disposable schema, never application tables. */
|
|
import assert from "node:assert/strict"
|
|
import { readFile } from "node:fs/promises"
|
|
import { createServer } from "node:http"
|
|
import { randomBytes } from "node:crypto"
|
|
import { setTimeout as sleep } from "node:timers/promises"
|
|
import pg from "pg"
|
|
import { logEntrySchema, logFiltersSchema, logPageSchema, logSummarySchema } from "@mmapp/contracts"
|
|
import { pool } from "../db/index.js"
|
|
import { env } from "../config.js"
|
|
import { collectRouterLogsOnce, getLogStates, subscribeLogDevice, updateLogSettings } from "./router-logs-collector.js"
|
|
import { readLogContext, readLogEntry, readLogPage, readLogStream, readLogSummary } from "./router-logs-query.js"
|
|
import { PARTITION_WRITER_ADVISORY_KEY } from "./partition-write-transaction.js"
|
|
|
|
const schema = "test_router_logs_" + randomBytes(6).toString("hex")
|
|
const admin = new pg.Pool({ connectionString: env.DATABASE_URL, connectionTimeoutMillis: 3000 })
|
|
try { await admin.query("SELECT 1") } catch { await admin.end(); console.error("PostgreSQL недоступен; SQL/live/performance проверки не выполнены"); process.exit(1) }
|
|
if (!/^[a-z_0-9]+$/.test(schema)) throw new Error("Unsafe test schema")
|
|
await admin.query(`CREATE SCHEMA ${schema}`)
|
|
pool.options.options = `-c search_path=${schema}`
|
|
let entries = Array.from({ length: 301 }, (_, i) => ({ ".id": `*${i}`, time: "12:00:00", topics: "system,info", message: "entry " + i }))
|
|
let requests = 0, denied = false
|
|
let beforeLogResponse: (() => Promise<unknown>) | undefined
|
|
const router = createServer(async (req, res) => {
|
|
requests++
|
|
const url = new URL(req.url!, "http://127.0.0.1")
|
|
assert.equal(url.pathname, "/rest/log")
|
|
assert.equal(url.searchParams.get("buffer"), "memory")
|
|
const beforeResponse = beforeLogResponse
|
|
beforeLogResponse = undefined
|
|
try { await beforeResponse?.() }
|
|
catch (error) { res.destroy(error as Error); return }
|
|
res.writeHead(denied ? 401 : 200, { "content-type": "application/json" }); res.end(JSON.stringify(denied ? { error: "denied" } : entries))
|
|
})
|
|
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
|
const address = router.address()
|
|
assert.ok(address && typeof address !== "string")
|
|
const routerPort = address.port
|
|
|
|
async function verifyMultiServerHistory() {
|
|
const addServer = async (name: string) => Number((await pool.query<{ id: number }>(
|
|
"INSERT INTO servers(name,host,port,use_ssl) VALUES($1,'127.0.0.1',$2,false) RETURNING id", [name, routerPort],
|
|
)).rows[0].id)
|
|
const firstId = await addServer("logs-commit-first")
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
SELECT $1,'logs-commit-first','127.0.0.1',i::text,'12:00:00',now(),now()-(i * interval '1 second'),ARRAY['system'],'info','history sequence baseline '||i,0
|
|
FROM generate_series(1,2) AS i`, [firstId])
|
|
const filters = logFiltersSchema.parse({ q: "history sequence" })
|
|
const initial = await readLogPage(filters, { limit: 1 })
|
|
const continuation = { limit: 1, cursor: initial.nextCursor!, watermark: initial.watermark }
|
|
const expected = await readLogPage(filters, continuation)
|
|
await pool.query(`CREATE FUNCTION hold_first_log_append() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN PERFORM pg_advisory_xact_lock(872316,1); RETURN NEW; END $$;
|
|
CREATE TRIGGER hold_first_log_append AFTER INSERT ON router_log_entries FOR EACH ROW
|
|
WHEN (NEW.server_id = ${firstId}) EXECUTE FUNCTION hold_first_log_append()`)
|
|
const blocker = await pool.connect()
|
|
const pending: ReturnType<typeof collectRouterLogsOnce>[] = []
|
|
let secondId!: number
|
|
const waitForLock = async (classid: number, objid: number, granted: boolean) => {
|
|
for (let i = 0; i < 200; i++) {
|
|
const { rows: [waiter] } = await pool.query<{ pid: number }>(`SELECT pid FROM pg_locks
|
|
WHERE locktype='advisory' AND database=(SELECT oid FROM pg_database WHERE datname=current_database())
|
|
AND classid=$1::oid AND objid=$2::oid AND objsubid=2 AND granted=$3`, [classid, objid, granted])
|
|
if (waiter) return waiter.pid
|
|
if (secondId !== undefined && (await pool.query("SELECT 1 FROM router_log_entries WHERE server_id=$1", [secondId])).rowCount) {
|
|
assert.fail("The later collector committed while an earlier reserved log ID was still invisible")
|
|
}
|
|
await sleep(25)
|
|
}
|
|
assert.fail(`Timed out waiting for test collector ${classid}/${objid}`)
|
|
}
|
|
try {
|
|
await blocker.query("BEGIN")
|
|
await blocker.query("SELECT pg_advisory_xact_lock(872316,1)")
|
|
entries = [{ ".id": "*1", time: new Date(Date.now() - 1500).toISOString(), topics: "system,info", message: "history sequence concurrent" }]
|
|
pending.push(collectRouterLogsOnce())
|
|
const firstPid = await waitForLock(872316, 1, false)
|
|
secondId = await addServer("logs-commit-second")
|
|
pending.push(collectRouterLogsOnce())
|
|
const secondPid = await waitForLock(872315, 1, false)
|
|
assert.ok((await pool.query<{ blocks: boolean }>("SELECT $1=ANY(pg_blocking_pids($2)) AS blocks", [firstPid, secondPid])).rows[0].blocks)
|
|
assert.deepEqual(await readLogPage(filters, { limit: 1 }), initial)
|
|
} finally {
|
|
await blocker.query("ROLLBACK")
|
|
blocker.release()
|
|
await Promise.all(pending)
|
|
await pool.query("DROP TRIGGER hold_first_log_append ON router_log_entries")
|
|
await pool.query("DROP FUNCTION hold_first_log_append()")
|
|
}
|
|
const visible = (await readLogPage(filters, { limit: 200 })).entries.filter(row => row.message === "history sequence concurrent")
|
|
assert.equal(visible.length, 2)
|
|
assert.ok(BigInt(visible.find(row => row.serverId === firstId)!.id) < BigInt(visible.find(row => row.serverId === secondId)!.id))
|
|
assert.deepEqual(await readLogPage(filters, continuation), expected)
|
|
console.log("PASS multi-CHR history: later write waits for prior commit, and the existing page excludes both later entries")
|
|
}
|
|
|
|
try {
|
|
const initial = await readFile(new URL("../../drizzle/0000_postgresql.sql", import.meta.url), "utf8")
|
|
const serverSql = initial.match(/CREATE TABLE IF NOT EXISTS servers \([\s\S]*?\n\);/)![0]
|
|
await pool.query(serverSql)
|
|
await pool.query(await readFile(new URL("../../drizzle/0014_router_logs.sql", import.meta.url), "utf8"))
|
|
// Shared configuration writes require the real protections and write-ahead audit
|
|
// tables even when this test owns only a disposable logs schema.
|
|
await pool.query(await readFile(new URL("../../drizzle/0015_mcp.sql", import.meta.url), "utf8"))
|
|
await pool.query(await readFile(new URL("../../drizzle/0016_mcp_routing_observations.sql", import.meta.url), "utf8"))
|
|
const sourceMigration = await readFile(new URL("../../drizzle/0029_mcp_routing_sources.sql", import.meta.url), "utf8")
|
|
await pool.query(sourceMigration)
|
|
await pool.query(sourceMigration)
|
|
const { rows: [server] } = await pool.query<{ id: number }>(`INSERT INTO servers(name,host,port,use_ssl) VALUES('logs-fixture','127.0.0.1',$1,false) RETURNING id`, [address.port])
|
|
const serverId = Number(server.id)
|
|
await updateLogSettings({ enabled: true, intervalSec: 5, retentionDays: 30 })
|
|
const filter = logFiltersSchema.parse({ serverId })
|
|
const { rows: [expired] } = await pool.query<{ id: string }>(`INSERT INTO router_log_entries
|
|
(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES($1,'logs-fixture','127.0.0.1','*expired','12:00:00',now()-interval '400 days',now()-interval '400 days',ARRAY['system'],'info','expired',0) RETURNING id::text`, [serverId])
|
|
const maintenance = await pool.connect()
|
|
try {
|
|
await maintenance.query("BEGIN")
|
|
await maintenance.query("SELECT pg_advisory_xact_lock($1::bigint)", [PARTITION_WRITER_ADVISORY_KEY])
|
|
const before = requests
|
|
const deferred = await collectRouterLogsOnce()
|
|
assert.equal(requests, before)
|
|
assert.equal(deferred.servers[0].added, 0)
|
|
assert.deepEqual(deferred.errors, [])
|
|
assert.deepEqual(await getLogStates(serverId), [])
|
|
assert.equal((await readLogEntry(expired.id, serverId))?.message, "expired")
|
|
} finally { await maintenance.query("ROLLBACK"); maintenance.release() }
|
|
const { rows: [keep] } = await pool.query<{ id: string }>(`INSERT INTO router_log_entries
|
|
(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES($1,'logs-fixture','127.0.0.1','*keep','12:00:00',now()-interval '45 days',now()-interval '45 days',ARRAY['system'],'info','keep after retention increase',0) RETURNING id::text`, [serverId])
|
|
beforeLogResponse = () => updateLogSettings({ retentionDays: 90 })
|
|
const collected = await collectRouterLogsOnce()
|
|
assert.deepEqual(collected.errors, [])
|
|
assert.equal(await readLogEntry(expired.id, serverId), null)
|
|
assert.equal((await readLogEntry(keep.id, serverId))?.message, "keep after retention increase")
|
|
await pool.query("DELETE FROM router_log_entries WHERE id=$1", [keep.id])
|
|
await updateLogSettings({ retentionDays: 14 })
|
|
console.log("PASS retention: cleanup uses the setting committed during RouterOS collection")
|
|
let page = logPageSchema.parse(JSON.parse(JSON.stringify(await readLogPage(filter, { limit: 200 }))))
|
|
assert.equal(page.entries.length, 200); assert.ok(page.nextCursor)
|
|
const older = await readLogPage(filter, { limit: 200, cursor: page.nextCursor!, watermark: page.watermark })
|
|
assert.equal(older.entries.length, 101)
|
|
assert.equal(new Set([...older.entries, ...page.entries].map(e => e.id)).size, 301)
|
|
const allServers = logFiltersSchema.parse({})
|
|
const scoped = { ...allServers, allowedServerIds: [serverId] }
|
|
const scopedFirst = await readLogPage(scoped, { limit: 200 })
|
|
const hiddenServerId = serverId + 1
|
|
const { rows: [hidden] } = await pool.query<{ id: string }>(`INSERT INTO router_log_entries
|
|
(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
VALUES($1,'hidden','192.0.2.2','*1','12:00:00',now(),now(),ARRAY['system'],'info','hidden event',0) RETURNING id::text`, [hiddenServerId])
|
|
assert.equal((await readLogPage(allServers, { limit: 200 })).watermark, hidden.id, "Unscoped history keeps its global watermark")
|
|
assert.deepEqual(await readLogPage(scoped, { limit: 200 }), scopedFirst, "Hidden writes cannot advance a scoped watermark")
|
|
assert.deepEqual(await readLogPage({ ...scoped, serverId }, { limit: 200 }), scopedFirst)
|
|
const emptyPage = { entries: [], nextCursor: null, watermark: "0" }
|
|
assert.deepEqual(await readLogPage({ ...scoped, serverId: hiddenServerId }, { limit: 200 }), emptyPage)
|
|
assert.deepEqual(await readLogPage({ ...scoped, allowedServerIds: [] }, { limit: 200 }), emptyPage)
|
|
assert.deepEqual(await readLogPage(scoped, { limit: 200, cursor: scopedFirst.nextCursor!, watermark: scopedFirst.watermark }), older)
|
|
assert.equal((await readLogEntry(hidden.id))?.serverId, hiddenServerId)
|
|
assert.equal(await readLogEntry(hidden.id, serverId), null)
|
|
assert.equal(await readLogContext(hidden.id, serverId), null)
|
|
assert.equal((await readLogContext(hidden.id, hiddenServerId))?.entry.serverId, hiddenServerId)
|
|
await pool.query("DELETE FROM router_log_entries WHERE id=$1", [hidden.id])
|
|
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
|
await collectRouterLogsOnce()
|
|
assert.equal((await readLogSummary(filter)).total, 301, "Repeated snapshot is not reimported")
|
|
entries = [...entries.slice(5), { ".id": "*302", time: "12:00:01", topics: "system,error,critical", message: "login failure" }]
|
|
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
|
// Concurrent clients and background collection must issue one read, not duplicate writes.
|
|
const beforeRequests = requests
|
|
const a = subscribeLogDevice(serverId), b = subscribeLogDevice(serverId)
|
|
await Promise.all([collectRouterLogsOnce(), collectRouterLogsOnce(), collectRouterLogsOnce()])
|
|
a(); b()
|
|
assert.equal(requests - beforeRequests, 1)
|
|
const stream = await readLogStream(filter, page.watermark)
|
|
assert.equal(stream.entries.length, 1)
|
|
const unmatched = await readLogStream(logFiltersSchema.parse({ serverId, q: "not-present-in-buffer" }), page.watermark)
|
|
assert.equal(unmatched.entries.length, 0)
|
|
assert.equal(unmatched.cursor, stream.cursor, "Filtered stream advances across non-matching entries")
|
|
const critical = logEntrySchema.parse(JSON.parse(JSON.stringify(stream.entries[0])))
|
|
assert.equal(critical.level, "critical"); assert.equal(critical.timeKind, "receipt")
|
|
const context = await readLogContext(critical.id)
|
|
assert.equal(context?.before.length, 10)
|
|
const filtered = logFiltersSchema.parse({ serverId, levels: "critical,error", q: "LOGIN" })
|
|
assert.equal((await readLogPage(filtered, { limit: 200 })).entries.length, 1)
|
|
const summary = logSummarySchema.parse(JSON.parse(JSON.stringify(await readLogSummary(filtered))))
|
|
assert.equal(summary.total, 1); assert.equal(summary.levels.critical, 1)
|
|
entries = [{ ".id": "*1", time: "01:00:00", topics: "system,warning", message: "reboot" }]
|
|
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
|
await collectRouterLogsOnce()
|
|
assert.equal((await getLogStates(serverId))[0].generation, 1)
|
|
assert.ok((await getLogStates(serverId))[0].gapAt)
|
|
denied = true
|
|
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
|
await collectRouterLogsOnce()
|
|
assert.ok((await getLogStates(serverId))[0].lastError)
|
|
const deniedRequests = requests
|
|
await collectRouterLogsOnce()
|
|
assert.equal(requests, deniedRequests, "Error backoff prevents repeated logins")
|
|
denied = false
|
|
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute',next_poll_at=NULL")
|
|
await collectRouterLogsOnce()
|
|
assert.equal((await getLogStates(serverId))[0].lastError, null)
|
|
// Receipt-based retention, including events whose device timestamp is wrong.
|
|
await pool.query(`UPDATE router_log_entries SET received_at=now()-interval '15 days' WHERE id=$1`, [critical.id])
|
|
await collectRouterLogsOnce()
|
|
assert.equal((await pool.query("SELECT id FROM router_log_entries WHERE id=$1", [critical.id])).rowCount, 0)
|
|
await pool.query("DELETE FROM servers WHERE id=$1", [serverId])
|
|
assert.ok((await readLogSummary(filter)).total > 0, "Deleting device preserves collected history")
|
|
await verifyMultiServerHistory()
|
|
const count = process.env.LOGS_PERFORMANCE === "1" ? 1_000_000 : 5000
|
|
const started = performance.now()
|
|
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
|
SELECT 999,'perf','192.0.2.1',i::text,'12:00:00',now(),now()-(i * interval '0.1 seconds'),ARRAY['system','info'],'info','performance event '||i,0 FROM generate_series(1,$1::int) AS i`, [count])
|
|
const performanceFilters = logFiltersSchema.parse({ serverId: 999, from: new Date(Date.now() - 14 * 86400000).toISOString() })
|
|
const readStarted = performance.now()
|
|
page = logPageSchema.parse(JSON.parse(JSON.stringify(await readLogPage(performanceFilters, { limit: 200 }))))
|
|
assert.equal(page.entries.length, 200)
|
|
const performanceSummary = await readLogSummary(performanceFilters)
|
|
assert.equal(performanceSummary.total, count)
|
|
console.log(`PASS SQL: snapshot, rotation, real repeats, concurrency, restart-safe state, cursor/SSE, filters, context, retention, deletion; ${count} events inserted in ${Math.round(readStarted - started)}ms; history+summary ${Math.round(performance.now() - readStarted)}ms`)
|
|
} finally {
|
|
await new Promise<void>(resolve => router.close(() => resolve()))
|
|
await pool.end()
|
|
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
|
|
await admin.end()
|
|
}
|