denozord/EvoFirewall
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

perf(api): ретенция статистики, устранение N+1 и пагинация списков

Browse Source 
- retention: ежедневный cron (03:17) + запуск на старте удаляет сырые
  agent_stats_samples старше STATS_RETENTION_DAYS (по умолчанию 30);
  lifetime-итоги на агенте и агрегаты сохраняются
- N+1 на поллинг-пути агента: evaluateAgentPolicy теперь делает по одному
  батч-запросу на записи списков / резолвы hostname / имена списков вместо
  запроса на каждое правило; mapPolicyRules и port-rules GET — аналогично
- пагинация: GET /agents, /lists, /rules, /install-links принимают
  limit/offset и возвращают total; без параметров — прежнее поведение
- openapi.yaml: approve-bulk, PUT /policy-sets/{id}/agents, параметры
  пагинации (redocly lint OK); тесты (47 passed)
This commit is contained in:
Denozordec
2026-09-20 19:20:30 +07:00
parent 454c5009d1
commit 7a3f1fad25
23 changed files with 462 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/api/src/app.ts
+28
View File
@@ -140,6 +140,34 @@ export async function buildApp(opts: BuildAppOptions = {}) {
preventOverrun: true,
}),
)
// Raw stats samples grow ~1 row per agent per sync interval; drop old
// ones daily. Lifetime totals live on agents; aggregates are kept.
const runRetention = async () => {
const cutoff = new Date(
Date.now() - config.statsRetentionDays * 86_400_000,
).toISOString()
const deleted = repos.deleteStatsSamplesBefore(app.db, cutoff)
if (deleted > 0) {
app.log.info(
{ deleted, cutoff },
`stats retention: raw samples older than ${config.statsRetentionDays}d removed`,
)
}
}
app.scheduler.addCronJob(
new CronJob(
{ cronExpression: '17 3 * * *' },
new AsyncTask('stats-retention', runRetention, (err) => {
app.log.warn({ err }, 'stats retention failed')
}),
{ preventOverrun: true },
),
)
// First cleanup right at startup, not only after the next 03:17.
runRetention().catch((err) =>
app.log.warn({ err }, 'stats retention failed'),
)
}
return app
apps/api/src/config.ts
+5
View File
@@ -15,6 +15,7 @@ export interface AppConfig {
enrollSeed: string
corsOrigins: string[]
secretKey: string | null
statsRetentionDays: number
}
function boolEnv(v: string | undefined, fallback: boolean): boolean {
@@ -65,6 +66,10 @@ export function loadConfig(): AppConfig {
.map((s) => s.trim().replace(/\/$/, ''))
.filter(Boolean),
secretKey: process.env.EVOFW_SECRET_KEY?.trim() || null,
statsRetentionDays: Math.max(
1,
Math.min(3650, Number(process.env.STATS_RETENTION_DAYS ?? '30') || 30),
),
}
// Fail-safe: a production process must not start wide open or with
apps/api/src/routes/agents.ts
+12 -8
View File
@@ -13,6 +13,7 @@ import { buildInstallUrls } from '../services/install-links.js'
import type { AppConfig } from '../config.js'
import { auditMutation } from '../services/audit.js'
import { mapAgent } from '../services/row-mappers.js'
import { applyPagination } from '../services/pagination.js'
export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
app,
@@ -20,11 +21,12 @@ export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
) => {
const { config } = opts
app.get('/agents', async () => {
const all = repos.listAgents(app.db)
const linksByAgent = repos.mapActiveInstallLinksByAgentId(app.db)
return {
items: all.map((a) => {
app.get<{ Querystring: { limit?: string; offset?: string } }>(
'/agents',
async (req) => {
const all = repos.listAgents(app.db)
const linksByAgent = repos.mapActiveInstallLinksByAgentId(app.db)
const items = all.map((a) => {
const link = linksByAgent.get(a.id)
if (!link) {
return mapAgent(a)
@@ -39,9 +41,11 @@ export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
installCurl: urls.curl.by_slug,
installLinkId: link.id,
})
}),
}
})
})
const paged = applyPagination(items, req.query)
return { items: paged.items, total: paged.total }
},
)
app.get<{ Params: { id: string } }>('/agents/:id', async (req) => {
const a = repos.getAgent(app.db, req.params.id)
apps/api/src/routes/install-links.ts
+11 -5
View File
@@ -9,6 +9,7 @@ import {
import { hashToken } from '../plugins/auth.js'
import type { AppConfig } from '../config.js'
import { auditMutation } from '../services/audit.js'
import { applyPagination } from '../services/pagination.js'
export const installLinksRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
app,
@@ -16,11 +17,16 @@ export const installLinksRoutes: FastifyPluginAsync<{ config: AppConfig }> = asy
) => {
const { config } = opts
app.get('/install-links', async () => ({
items: repos
.listInstallLinks(app.db)
.map((row) => mapInstallLink(row, config.publicBaseUrl)),
}))
app.get<{ Querystring: { limit?: string; offset?: string } }>(
'/install-links',
async (req) => {
const items = repos
.listInstallLinks(app.db)
.map((row) => mapInstallLink(row, config.publicBaseUrl))
const paged = applyPagination(items, req.query)
return { items: paged.items, total: paged.total }
},
)
app.post('/install-links', async (req, reply) => {
const body = createInstallLinkBodySchema.parse(req.body)
apps/api/src/routes/lists.ts
+25 -20
View File
@@ -16,6 +16,7 @@ import {
import type { AppConfig } from '../config.js'
import { auditMutation } from '../services/audit.js'
import { maskListConfig, sealListConfig } from '../services/secret-cipher.js'
import { applyPagination } from '../services/pagination.js'
export const listsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
app,
@@ -23,26 +24,30 @@ export const listsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
) => {
const { config } = opts
app.get('/lists', async () => {
const lists = repos.listIpLists(app.db)
const counts = repos.countEntriesByListIds(
app.db,
lists.map((l) => l.id),
)
const items = lists.map((l) => ({
id: l.id,
name: l.name,
type: l.type,
config_json: maskListConfig(l.configJson),
content_hash: l.contentHash,
refreshed_at: l.refreshedAt,
last_error: l.lastError,
entry_count: counts.get(l.id) ?? 0,
created_at: l.createdAt,
updated_at: l.updatedAt,
}))
return { items }
})
app.get<{ Querystring: { limit?: string; offset?: string } }>(
'/lists',
async (req) => {
const lists = repos.listIpLists(app.db)
const counts = repos.countEntriesByListIds(
app.db,
lists.map((l) => l.id),
)
const items = lists.map((l) => ({
id: l.id,
name: l.name,
type: l.type,
config_json: maskListConfig(l.configJson),
content_hash: l.contentHash,
refreshed_at: l.refreshedAt,
last_error: l.lastError,
entry_count: counts.get(l.id) ?? 0,
created_at: l.createdAt,
updated_at: l.updatedAt,
}))
const paged = applyPagination(items, req.query)
return { items: paged.items, total: paged.total }
},
)
app.post('/lists', async (req) => {
const body = createIpListBodySchema.parse(req.body)
apps/api/src/routes/port-acl.ts
+11 -6
View File
@@ -63,12 +63,17 @@ export const portAclRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
async (req) => {
const agent = repos.getAgent(app.db, req.params.id)
if (!agent) throw new AppError('NOT_FOUND', 'Agent not found', 404)
const items = repos.listAgentPortRules(app.db, agent.id).map((row) => {
const listName = row.listId
? repos.getIpList(app.db, row.listId)?.name
: null
return mapPortRule(row, listName)
})
const rows = repos.listAgentPortRules(app.db, agent.id)
const listNames = repos.mapIpListNames(
app.db,
rows.map((r) => r.listId).filter((id): id is string => Boolean(id)),
)
const items = rows.map((row) =>
mapPortRule(
row,
row.listId ? (listNames.get(row.listId) ?? null) : null,
),
)
return { items }
},
)
apps/api/src/routes/rules.ts
+23 -22
View File
@@ -12,7 +12,8 @@ import {
} from '../services/policy/resolve-hostname.js'
import type { AppConfig } from '../config.js'
import { auditMutation } from '../services/audit.js'
import { mapPolicyRule } from '../services/row-mappers.js'
import { mapPolicyRule, mapPolicyRules } from '../services/row-mappers.js'
import { applyPagination } from '../services/pagination.js'
export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
app,
@@ -26,29 +27,28 @@ export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
const s = repos.getPolicySet(app.db, req.params.id)
if (!s) throw new AppError('NOT_FOUND', 'Policy set not found', 404)
return {
items: repos
.listPolicyRules(app.db, s.id)
.map((r) => mapPolicyRule(r, app.db)),
items: mapPolicyRules(repos.listPolicyRules(app.db, s.id), app.db),
}
},
)
app.get<{ Querystring: { set_id?: string; agent_id?: string } }>(
'/rules',
async (req) => {
if (req.query.agent_id) {
return {
items: repos
.listPolicyRulesForAgent(app.db, req.query.agent_id)
.map((r) => mapPolicyRule(r, app.db)),
}
app.get<{
Querystring: { set_id?: string; agent_id?: string; limit?: string; offset?: string }
}>('/rules', async (req) => {
if (req.query.agent_id) {
return {
items: mapPolicyRules(
repos.listPolicyRulesForAgent(app.db, req.query.agent_id),
app.db,
),
}
const items = repos
.listPolicyRules(app.db, req.query.set_id)
.map((r) => mapPolicyRule(r, app.db))
return { items }
},
)
}
const paged = applyPagination(
mapPolicyRules(repos.listPolicyRules(app.db, req.query.set_id), app.db),
req.query,
)
return { items: paged.items, total: paged.total }
})
app.post('/rules', async (req) => {
const body = createPolicyRuleBodySchema.parse(req.body)
@@ -174,9 +174,10 @@ export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
details: { set_id: s.id, ordered_ids: body.ordered_ids },
})
return {
items: repos
.listPolicyRules(app.db, s.id)
.map((r) => mapPolicyRule(r, app.db)),
items: mapPolicyRules(
repos.listPolicyRules(app.db, s.id),
app.db,
),
}
},
)
apps/api/src/services/agents-policy.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
describe('agents CRUD critical paths', () => {
apps/api/src/services/bulk-ops.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
async function createInvitedAgent(
apps/api/src/services/install-links.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
describe('install-links', () => {
apps/api/src/services/ip-block-stats.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
async function enrollApprovedLinux(
apps/api/src/services/pagination.ts
+22
View File
@@ -0,0 +1,22 @@
/**
* Backwards-compatible list pagination: without limit/offset the response is
* the full list (total === items.length); with them, a page plus the total.
*/
export function applyPagination<T>(
items: T[],
query: { limit?: string; offset?: string },
): { items: T[]; total: number } {
const total = items.length
const limitRaw = query.limit !== undefined ? Number(query.limit) : NaN
const offsetRaw = query.offset !== undefined ? Number(query.offset) : NaN
if (!Number.isFinite(limitRaw) && !Number.isFinite(offsetRaw)) {
return { items, total }
}
const limit = Number.isFinite(limitRaw)
? Math.max(1, Math.min(1000, Math.floor(limitRaw)))
: items.length
const offset = Number.isFinite(offsetRaw)
? Math.max(0, Math.floor(offsetRaw))
: 0
return { items: items.slice(offset, offset + limit), total }
}
apps/api/src/services/policy/evaluate.ts
+51 -16
View File
@@ -54,13 +54,41 @@ export type EvaluatedPolicy = {
}
}
function expandList(db: Db, listId: string | null | undefined): string[] {
if (!listId) return []
return repos.listIpListEntries(db, listId).map((e) => e.cidr)
/**
* Prefetched expansion data: one batched query per kind instead of a query
* per rule (this code runs on every agent policy poll, ~60s per agent).
*/
type ExpansionContext = {
entriesByList: Map<string, string[]>
resolvedByRule: Map<string, string[]>
listNames: Map<string, string>
}
function buildExpansionContext(
db: Db,
rules: { id: string; listId: string | null; hostname: string | null }[],
portRuleRows: { listId: string | null }[],
): ExpansionContext {
const listIds = [
...new Set(
[
...rules.map((r) => r.listId?.trim() || ''),
...portRuleRows.map((r) => r.listId?.trim() || ''),
].filter(Boolean),
),
]
const hostnameRuleIds = [
...new Set(rules.filter((r) => r.hostname?.trim()).map((r) => r.id)),
]
return {
entriesByList: repos.mapIpListEntriesByListIds(db, listIds),
resolvedByRule: repos.mapResolvedCidrsByRuleIds(db, hostnameRuleIds),
listNames: repos.mapIpListNames(db, listIds),
}
}
function expandRule(
db: Db,
ctx: ExpansionContext,
rule: {
cidr: string | null
listId: string | null
@@ -70,9 +98,10 @@ function expandRule(
): string[] {
if (rule.cidr?.trim()) return [rule.cidr.trim()]
if (rule.hostname?.trim()) {
return repos.listResolvedForRule(db, rule.id).map((r) => r.cidr)
return ctx.resolvedByRule.get(rule.id) ?? []
}
return expandList(db, rule.listId)
const listId = rule.listId?.trim() || ''
return listId ? (ctx.entriesByList.get(listId) ?? []) : []
}
function resolveDefaultAction(agentDefaultAction: string | null | undefined): DefaultAction {
@@ -83,7 +112,7 @@ function resolveDefaultAction(agentDefaultAction: string | null | undefined): De
}
function sourceMeta(
db: Db,
ctx: ExpansionContext,
rule: {
cidr: string | null
listId: string | null
@@ -97,12 +126,12 @@ function sourceMeta(
return { kind: 'hostname', label: rule.hostname.trim() }
}
const listId = rule.listId?.trim() || ''
const name = listId ? repos.getIpList(db, listId)?.name : null
const name = listId ? ctx.listNames.get(listId) : null
return { kind: 'list', label: name || listId || 'list' }
}
function expandPortSrcCidrs(
db: Db,
ctx: ExpansionContext,
row: {
srcKind: string
srcCidr: string | null
@@ -114,18 +143,22 @@ function expandPortSrcCidrs(
return [row.srcCidr.trim()]
}
if (row.srcKind === 'list') {
const cidrs = expandList(db, row.listId)
const cidrs = row.listId
? (ctx.entriesByList.get(row.listId.trim()) ?? [])
: []
return cidrs.length ? uniqCidrs(cidrs) : []
}
return []
}
function expandPortRules(db: Db, agentId: string): EvaluatedPortRule[] {
const rows = repos.listEnabledAgentPortRules(db, agentId)
function expandPortRules(
ctx: ExpansionContext,
rows: ReturnType<typeof repos.listEnabledAgentPortRules>,
): EvaluatedPortRule[] {
const out: EvaluatedPortRule[] = []
for (const row of rows) {
const action = row.action === 'close' ? 'close' : 'open'
const srcCidrs = expandPortSrcCidrs(db, row)
const srcCidrs = expandPortSrcCidrs(ctx, row)
if (!srcCidrs.length) continue
const portStart = Math.max(1, Math.min(65535, row.portStart))
const portEnd = Math.max(portStart, Math.min(65535, row.portEnd))
@@ -161,6 +194,8 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
.filter((s) => s.enabled === 1)
const ordered = repos.listPolicyRulesForAgent(db, agentId)
const overrides = repos.listOverrides(db, agentId)
const portRuleRows = repos.listEnabledAgentPortRules(db, agentId)
const ctx = buildExpansionContext(db, ordered, portRuleRows)
const deny: string[] = []
const allow: string[] = []
@@ -169,7 +204,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
let rulesAllow = 0
for (const rule of ordered) {
const cidrs = expandRule(db, rule)
const cidrs = expandRule(ctx, rule)
const action = rule.action === 'deny' ? 'deny' : 'allow'
if (action === 'deny') {
deny.push(...cidrs)
@@ -178,7 +213,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
allow.push(...cidrs)
rulesAllow += 1
}
const src = sourceMeta(db, rule)
const src = sourceMeta(ctx, rule)
const setName =
assignedSets.find((s) => s.setId === rule.setId)?.name ?? null
chain.push({
@@ -214,7 +249,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
const conflictsDropped = allowRaw.length - allowCidrs.length
const defaultAction = resolveDefaultAction(agent.defaultAction)
const policyMode = legacyModeFromDefaultAction(defaultAction)
const portRules = expandPortRules(db, agentId)
const portRules = expandPortRules(ctx, portRuleRows)
const payload = JSON.stringify({
apply_version: POLICY_APPLY_VERSION,
apps/api/src/services/policy/policy-mode.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
async function createAgent(
apps/api/src/services/port-acl.test.ts
+1
View File
@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
async function enrollApprovedLinux(
apps/api/src/services/retention-pagination.test.ts
+101
View File
@@ -0,0 +1,101 @@
import { describe, it, expect, afterAll } from 'vitest'
import { buildApp } from '../app.js'
import { repos } from '@evofw/db'
import type { AppConfig } from '../config.js'
const testConfig: AppConfig = {
databaseUrl: 'sqlite::memory:',
jwtSecret: 'test',
jwtTtlHours: 24,
serverPort: 8080,
staticDir: null,
logLevel: 'error',
authRequired: false,
authIssuer: 'https://auth.test',
authPortalUrl: 'http://localhost:5175',
publicBaseUrl: 'https://fw.example.com',
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
describe('stats retention + list pagination', () => {
const appPromise = buildApp({ memory: true, config: testConfig })
afterAll(async () => {
const app = await appPromise
await app.close()
})
it('deleteStatsSamplesBefore drops only old samples', async () => {
const app = await appPromise
await app.ready()
const created = await app.inject({
method: 'POST',
url: '/api/v1/install-links',
payload: { name: 'retention-agent', platform: 'linux' },
})
const agentId = (created.json() as { agent_id: string }).agent_id
const iso = (daysAgo: number) =>
new Date(Date.now() - daysAgo * 86_400_000).toISOString()
for (const daysAgo of [60, 45, 10, 0]) {
repos.insertStatsSample(app.db, {
id: crypto.randomUUID(),
agentId,
packetsDropped: daysAgo,
packetsAccepted: 0,
prefixCount: 0,
kernelMethod: null,
recordedAt: iso(daysAgo),
})
}
const cutoff = new Date(Date.now() - 30 * 86_400_000).toISOString()
const deleted = repos.deleteStatsSamplesBefore(app.db, cutoff)
expect(deleted).toBe(2)
const remaining = repos.listStatsSamples(app.db, agentId, 100)
expect(remaining.map((s) => s.packetsDropped).sort()).toEqual([0, 10])
})
it('list endpoints return total and honor limit/offset', async () => {
const app = await appPromise
await app.ready()
for (const name of ['page-a', 'page-b', 'page-c']) {
await app.inject({
method: 'POST',
url: '/api/v1/install-links',
payload: { name, platform: 'linux' },
})
}
await app.inject({
method: 'POST',
url: '/api/v1/lists',
payload: { name: 'page-list', type: 'static' },
})
const all = await app.inject({ method: 'GET', url: '/api/v1/agents' })
expect(all.statusCode).toBe(200)
const allBody = all.json() as { items: unknown[]; total: number }
expect(allBody.items.length).toBe(allBody.total)
const page = await app.inject({
method: 'GET',
url: '/api/v1/agents?limit=1&offset=1',
})
const pageBody = page.json() as { items: unknown[]; total: number }
expect(pageBody.items).toHaveLength(1)
expect(pageBody.total).toBe(allBody.total)
const lists = await app.inject({
method: 'GET',
url: '/api/v1/lists?limit=1',
})
const listsBody = lists.json() as { items: unknown[]; total: number }
expect(listsBody.items).toHaveLength(1)
expect(listsBody.total).toBeGreaterThanOrEqual(1)
})
})
apps/api/src/services/row-mappers.ts
+30 -3
View File
@@ -67,6 +67,21 @@ export function mapPolicySets(
export function mapPolicyRule(
r: NonNullable<ReturnType<typeof repos.getPolicyRule>>,
db: Parameters<typeof repos.listResolvedForRule>[0],
) {
return mapPolicyRuleWithCounts(r, r.hostname ? countResolved(db, [r.id]).get(r.id) ?? 0 : undefined)
}
function countResolved(
db: Parameters<typeof repos.listResolvedForRule>[0],
ruleIds: string[],
): Map<string, number> {
const cidrs = repos.mapResolvedCidrsByRuleIds(db, ruleIds)
return new Map([...cidrs].map(([id, list]) => [id, list.length]))
}
function mapPolicyRuleWithCounts(
r: NonNullable<ReturnType<typeof repos.getPolicyRule>>,
resolvedCount: number | undefined,
) {
return {
id: r.id,
@@ -77,11 +92,23 @@ export function mapPolicyRule(
list_id: r.listId,
cidr: r.cidr,
hostname: r.hostname,
resolved_count: r.hostname
? repos.listResolvedForRule(db, r.id).length
: undefined,
resolved_count: resolvedCount,
comment: r.comment,
created_at: r.createdAt,
updated_at: r.updatedAt,
}
}
/** Batched variant for list endpoints: one resolved-counts query for all rules. */
export function mapPolicyRules(
rules: NonNullable<ReturnType<typeof repos.getPolicyRule>>[],
db: Parameters<typeof repos.listResolvedForRule>[0],
) {
const counts = countResolved(
db,
rules.filter((r) => r.hostname).map((r) => r.id),
)
return rules.map((r) =>
mapPolicyRuleWithCounts(r, r.hostname ? counts.get(r.id) ?? 0 : undefined),
)
}
apps/api/src/services/settings-bump.test.ts
+1
View File
@@ -17,6 +17,7 @@ const testConfig: AppConfig = {
enrollSeed: 'test-seed',
corsOrigins: [],
secretKey: null,
statsRetentionDays: 30,
}
describe('settings + bumpAgentsForList', () => {