feat(logs): добавить раздел логов с историей и живым потоком
Docker images / prepare-release (push) Successful in 9s
Docker images / backend-test (push) Successful in 4m35s
Docker images / frontend-image (push) Successful in 3m29s
Docker images / updater-image (push) Successful in 41s
Docker images / netflow-agent-image (push) Successful in 34s
Docker images / backend-image (push) Successful in 1m44s
Docker images / publish-release (push) Successful in 10s
Docker images / notify-webhook (push) Successful in 2s
Docker images / prepare-release (push) Successful in 9s
Docker images / backend-test (push) Successful in 4m35s
Docker images / frontend-image (push) Successful in 3m29s
Docker images / updater-image (push) Successful in 41s
Docker images / netflow-agent-image (push) Successful in 34s
Docker images / backend-image (push) Successful in 1m44s
Docker images / publish-release (push) Successful in 10s
Docker images / notify-webhook (push) Successful in 2s
This commit is contained in:
1 parent
e461dc21bd
commit
5efd0339fb
37 files changed
+1399
-7
No files matched your search
@@ -0,0 +1,108 @@
|
||||
/** Integration/performance tests use a fresh, disposable schema, never application tables. */
|
||||
import assert from "node:assert/strict"
|
||||
import { readFile } from "node:fs/promises"
|
||||
import { createServer } from "node:http"
|
||||
import { randomBytes } from "node:crypto"
|
||||
import pg from "pg"
|
||||
import { logEntrySchema, logFiltersSchema, logPageSchema, logSummarySchema } from "@mmapp/contracts"
|
||||
import { pool } from "../db/index.js"
|
||||
import { env } from "../config.js"
|
||||
import { collectRouterLogsOnce, getLogStates, subscribeLogDevice, updateLogSettings } from "./router-logs-collector.js"
|
||||
import { readLogContext, readLogPage, readLogStream, readLogSummary } from "./router-logs-query.js"
|
||||
|
||||
const schema = "test_router_logs_" + randomBytes(6).toString("hex")
|
||||
const admin = new pg.Pool({ connectionString: env.DATABASE_URL, connectionTimeoutMillis: 3000 })
|
||||
try { await admin.query("SELECT 1") } catch { await admin.end(); console.error("PostgreSQL недоступен; SQL/live/performance проверки не выполнены"); process.exit(1) }
|
||||
if (!/^[a-z_0-9]+$/.test(schema)) throw new Error("Unsafe test schema")
|
||||
await admin.query(`CREATE SCHEMA ${schema}`)
|
||||
pool.options.options = `-c search_path=${schema}`
|
||||
let entries = Array.from({ length: 301 }, (_, i) => ({ ".id": `*${i}`, time: "12:00:00", topics: "system,info", message: "entry " + i }))
|
||||
let requests = 0, denied = false
|
||||
const router = createServer((req, res) => {
|
||||
requests++
|
||||
const url = new URL(req.url!, "http://127.0.0.1")
|
||||
assert.equal(url.pathname, "/rest/log")
|
||||
assert.equal(url.searchParams.get("buffer"), "memory")
|
||||
res.writeHead(denied ? 401 : 200, { "content-type": "application/json" }); res.end(JSON.stringify(denied ? { error: "denied" } : entries))
|
||||
})
|
||||
await new Promise<void>(resolve => router.listen(0, "127.0.0.1", resolve))
|
||||
const address = router.address()
|
||||
assert.ok(address && typeof address !== "string")
|
||||
try {
|
||||
const initial = await readFile(new URL("../../drizzle/0000_postgresql.sql", import.meta.url), "utf8")
|
||||
const serverSql = initial.match(/CREATE TABLE IF NOT EXISTS servers \([\s\S]*?\n\);/)![0]
|
||||
await pool.query(serverSql)
|
||||
await pool.query(await readFile(new URL("../../drizzle/0014_router_logs.sql", import.meta.url), "utf8"))
|
||||
const { rows: [server] } = await pool.query<{ id: number }>(`INSERT INTO servers(name,host,port,use_ssl) VALUES('logs-fixture','127.0.0.1',$1,false) RETURNING id`, [address.port])
|
||||
const serverId = Number(server.id)
|
||||
await updateLogSettings({ enabled: true, intervalSec: 5 })
|
||||
const filter = logFiltersSchema.parse({ serverId })
|
||||
await collectRouterLogsOnce()
|
||||
let page = logPageSchema.parse(JSON.parse(JSON.stringify(await readLogPage(filter, { limit: 200 }))))
|
||||
assert.equal(page.entries.length, 200); assert.ok(page.nextCursor)
|
||||
const older = await readLogPage(filter, { limit: 200, cursor: page.nextCursor!, watermark: page.watermark })
|
||||
assert.equal(older.entries.length, 101)
|
||||
assert.equal(new Set([...older.entries, ...page.entries].map(e => e.id)).size, 301)
|
||||
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
||||
await collectRouterLogsOnce()
|
||||
assert.equal((await readLogSummary(filter)).total, 301, "Repeated snapshot is not reimported")
|
||||
entries = [...entries.slice(5), { ".id": "*302", time: "12:00:01", topics: "system,error,critical", message: "login failure" }]
|
||||
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
||||
// Concurrent clients and background collection must issue one read, not duplicate writes.
|
||||
const beforeRequests = requests
|
||||
const a = subscribeLogDevice(serverId), b = subscribeLogDevice(serverId)
|
||||
await Promise.all([collectRouterLogsOnce(), collectRouterLogsOnce(), collectRouterLogsOnce()])
|
||||
a(); b()
|
||||
assert.equal(requests - beforeRequests, 1)
|
||||
const stream = await readLogStream(filter, page.watermark)
|
||||
assert.equal(stream.entries.length, 1)
|
||||
const unmatched = await readLogStream(logFiltersSchema.parse({ serverId, q: "not-present-in-buffer" }), page.watermark)
|
||||
assert.equal(unmatched.entries.length, 0)
|
||||
assert.equal(unmatched.cursor, stream.cursor, "Filtered stream advances across non-matching entries")
|
||||
const critical = logEntrySchema.parse(JSON.parse(JSON.stringify(stream.entries[0])))
|
||||
assert.equal(critical.level, "critical"); assert.equal(critical.timeKind, "receipt")
|
||||
const context = await readLogContext(critical.id)
|
||||
assert.equal(context?.before.length, 10)
|
||||
const filtered = logFiltersSchema.parse({ serverId, levels: "critical,error", q: "LOGIN" })
|
||||
assert.equal((await readLogPage(filtered, { limit: 200 })).entries.length, 1)
|
||||
const summary = logSummarySchema.parse(JSON.parse(JSON.stringify(await readLogSummary(filtered))))
|
||||
assert.equal(summary.total, 1); assert.equal(summary.levels.critical, 1)
|
||||
entries = [{ ".id": "*1", time: "01:00:00", topics: "system,warning", message: "reboot" }]
|
||||
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
||||
await collectRouterLogsOnce()
|
||||
assert.equal((await getLogStates(serverId))[0].generation, 1)
|
||||
assert.ok((await getLogStates(serverId))[0].gapAt)
|
||||
denied = true
|
||||
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute'")
|
||||
await collectRouterLogsOnce()
|
||||
assert.ok((await getLogStates(serverId))[0].lastError)
|
||||
const deniedRequests = requests
|
||||
await collectRouterLogsOnce()
|
||||
assert.equal(requests, deniedRequests, "Error backoff prevents repeated logins")
|
||||
denied = false
|
||||
await pool.query("UPDATE router_log_collection_state SET last_collected_at=now()-interval '1 minute',next_poll_at=NULL")
|
||||
await collectRouterLogsOnce()
|
||||
assert.equal((await getLogStates(serverId))[0].lastError, null)
|
||||
// Receipt-based retention, including events whose device timestamp is wrong.
|
||||
await pool.query(`UPDATE router_log_entries SET received_at=now()-interval '15 days' WHERE id=$1`, [critical.id])
|
||||
await collectRouterLogsOnce()
|
||||
assert.equal((await pool.query("SELECT id FROM router_log_entries WHERE id=$1", [critical.id])).rowCount, 0)
|
||||
await pool.query("DELETE FROM servers WHERE id=$1", [serverId])
|
||||
assert.ok((await readLogSummary(filter)).total > 0, "Deleting device preserves collected history")
|
||||
const count = process.env.LOGS_PERFORMANCE === "1" ? 1_000_000 : 5000
|
||||
const started = performance.now()
|
||||
await pool.query(`INSERT INTO router_log_entries(server_id,server_name,host,router_id,raw_time,received_at,effective_at,topics,level,message,generation)
|
||||
SELECT 999,'perf','192.0.2.1',i::text,'12:00:00',now(),now()-(i * interval '0.1 seconds'),ARRAY['system','info'],'info','performance event '||i,0 FROM generate_series(1,$1::int) AS i`, [count])
|
||||
const performanceFilters = logFiltersSchema.parse({ serverId: 999, from: new Date(Date.now() - 14 * 86400000).toISOString() })
|
||||
const readStarted = performance.now()
|
||||
page = logPageSchema.parse(JSON.parse(JSON.stringify(await readLogPage(performanceFilters, { limit: 200 }))))
|
||||
assert.equal(page.entries.length, 200)
|
||||
const performanceSummary = await readLogSummary(performanceFilters)
|
||||
assert.equal(performanceSummary.total, count)
|
||||
console.log(`PASS SQL: snapshot, rotation, real repeats, concurrency, restart-safe state, cursor/SSE, filters, context, retention, deletion; ${count} events inserted in ${Math.round(readStarted - started)}ms; history+summary ${Math.round(performance.now() - readStarted)}ms`)
|
||||
} finally {
|
||||
await new Promise<void>(resolve => router.close(() => resolve()))
|
||||
await pool.end()
|
||||
await admin.query(`DROP SCHEMA ${schema} CASCADE`)
|
||||
await admin.end()
|
||||
}
|
||||
Reference in new issue
Block a user