perf(api): ретенция статистики, устранение N+1 и пагинация списков

- retention: ежедневный cron (03:17) + запуск на старте удаляет сырые
  agent_stats_samples старше STATS_RETENTION_DAYS (по умолчанию 30);
  lifetime-итоги на агенте и агрегаты сохраняются
- N+1 на поллинг-пути агента: evaluateAgentPolicy теперь делает по одному
  батч-запросу на записи списков / резолвы hostname / имена списков вместо
  запроса на каждое правило; mapPolicyRules и port-rules GET — аналогично
- пагинация: GET /agents, /lists, /rules, /install-links принимают
  limit/offset и возвращают total; без параметров — прежнее поведение
- openapi.yaml: approve-bulk, PUT /policy-sets/{id}/agents, параметры
  пагинации (redocly lint OK); тесты (47 passed)

apps/api/src/services/agents-policy.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 describe('agents CRUD critical paths', () => {

apps/api/src/services/bulk-ops.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 async function createInvitedAgent(

apps/api/src/services/install-links.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 describe('install-links', () => {

apps/api/src/services/ip-block-stats.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 async function enrollApprovedLinux(

apps/api/src/services/pagination.ts

@@ -0,0 +1,22 @@
+/**
+ * Backwards-compatible list pagination: without limit/offset the response is
+ * the full list (total === items.length); with them, a page plus the total.
+ */
+export function applyPagination<T>(
+  items: T[],
+  query: { limit?: string; offset?: string },
+): { items: T[]; total: number } {
+  const total = items.length
+  const limitRaw = query.limit !== undefined ? Number(query.limit) : NaN
+  const offsetRaw = query.offset !== undefined ? Number(query.offset) : NaN
+  if (!Number.isFinite(limitRaw) && !Number.isFinite(offsetRaw)) {
+    return { items, total }
+  }
+  const limit = Number.isFinite(limitRaw)
+    ? Math.max(1, Math.min(1000, Math.floor(limitRaw)))
+    : items.length
+  const offset = Number.isFinite(offsetRaw)
+    ? Math.max(0, Math.floor(offsetRaw))
+    : 0
+  return { items: items.slice(offset, offset + limit), total }
+}

apps/api/src/services/policy/evaluate.ts

@@ -54,13 +54,41 @@ export type EvaluatedPolicy = {
   }
 }
 
-function expandList(db: Db, listId: string | null | undefined): string[] {
-  if (!listId) return []
-  return repos.listIpListEntries(db, listId).map((e) => e.cidr)
+/**
+ * Prefetched expansion data: one batched query per kind instead of a query
+ * per rule (this code runs on every agent policy poll, ~60s per agent).
+ */
+type ExpansionContext = {
+  entriesByList: Map<string, string[]>
+  resolvedByRule: Map<string, string[]>
+  listNames: Map<string, string>
+}
+
+function buildExpansionContext(
+  db: Db,
+  rules: { id: string; listId: string | null; hostname: string | null }[],
+  portRuleRows: { listId: string | null }[],
+): ExpansionContext {
+  const listIds = [
+    ...new Set(
+      [
+        ...rules.map((r) => r.listId?.trim() || ''),
+        ...portRuleRows.map((r) => r.listId?.trim() || ''),
+      ].filter(Boolean),
+    ),
+  ]
+  const hostnameRuleIds = [
+    ...new Set(rules.filter((r) => r.hostname?.trim()).map((r) => r.id)),
+  ]
+  return {
+    entriesByList: repos.mapIpListEntriesByListIds(db, listIds),
+    resolvedByRule: repos.mapResolvedCidrsByRuleIds(db, hostnameRuleIds),
+    listNames: repos.mapIpListNames(db, listIds),
+  }
 }
 
 function expandRule(
-  db: Db,
+  ctx: ExpansionContext,
   rule: {
     cidr: string | null
     listId: string | null
@@ -70,9 +98,10 @@ function expandRule(
 ): string[] {
   if (rule.cidr?.trim()) return [rule.cidr.trim()]
   if (rule.hostname?.trim()) {
-    return repos.listResolvedForRule(db, rule.id).map((r) => r.cidr)
+    return ctx.resolvedByRule.get(rule.id) ?? []
   }
-  return expandList(db, rule.listId)
+  const listId = rule.listId?.trim() || ''
+  return listId ? (ctx.entriesByList.get(listId) ?? []) : []
 }
 
 function resolveDefaultAction(agentDefaultAction: string | null | undefined): DefaultAction {
@@ -83,7 +112,7 @@ function resolveDefaultAction(agentDefaultAction: string | null | undefined): De
 }
 
 function sourceMeta(
-  db: Db,
+  ctx: ExpansionContext,
   rule: {
     cidr: string | null
     listId: string | null
@@ -97,12 +126,12 @@ function sourceMeta(
     return { kind: 'hostname', label: rule.hostname.trim() }
   }
   const listId = rule.listId?.trim() || ''
-  const name = listId ? repos.getIpList(db, listId)?.name : null
+  const name = listId ? ctx.listNames.get(listId) : null
   return { kind: 'list', label: name || listId || 'list' }
 }
 
 function expandPortSrcCidrs(
-  db: Db,
+  ctx: ExpansionContext,
   row: {
     srcKind: string
     srcCidr: string | null
@@ -114,18 +143,22 @@ function expandPortSrcCidrs(
     return [row.srcCidr.trim()]
   }
   if (row.srcKind === 'list') {
-    const cidrs = expandList(db, row.listId)
+    const cidrs = row.listId
+      ? (ctx.entriesByList.get(row.listId.trim()) ?? [])
+      : []
     return cidrs.length ? uniqCidrs(cidrs) : []
   }
   return []
 }
 
-function expandPortRules(db: Db, agentId: string): EvaluatedPortRule[] {
-  const rows = repos.listEnabledAgentPortRules(db, agentId)
+function expandPortRules(
+  ctx: ExpansionContext,
+  rows: ReturnType<typeof repos.listEnabledAgentPortRules>,
+): EvaluatedPortRule[] {
   const out: EvaluatedPortRule[] = []
   for (const row of rows) {
     const action = row.action === 'close' ? 'close' : 'open'
-    const srcCidrs = expandPortSrcCidrs(db, row)
+    const srcCidrs = expandPortSrcCidrs(ctx, row)
     if (!srcCidrs.length) continue
     const portStart = Math.max(1, Math.min(65535, row.portStart))
     const portEnd = Math.max(portStart, Math.min(65535, row.portEnd))
@@ -161,6 +194,8 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
     .filter((s) => s.enabled === 1)
   const ordered = repos.listPolicyRulesForAgent(db, agentId)
   const overrides = repos.listOverrides(db, agentId)
+  const portRuleRows = repos.listEnabledAgentPortRules(db, agentId)
+  const ctx = buildExpansionContext(db, ordered, portRuleRows)
 
   const deny: string[] = []
   const allow: string[] = []
@@ -169,7 +204,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
   let rulesAllow = 0
 
   for (const rule of ordered) {
-    const cidrs = expandRule(db, rule)
+    const cidrs = expandRule(ctx, rule)
     const action = rule.action === 'deny' ? 'deny' : 'allow'
     if (action === 'deny') {
       deny.push(...cidrs)
@@ -178,7 +213,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
       allow.push(...cidrs)
       rulesAllow += 1
     }
-    const src = sourceMeta(db, rule)
+    const src = sourceMeta(ctx, rule)
     const setName =
       assignedSets.find((s) => s.setId === rule.setId)?.name ?? null
     chain.push({
@@ -214,7 +249,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy {
   const conflictsDropped = allowRaw.length - allowCidrs.length
   const defaultAction = resolveDefaultAction(agent.defaultAction)
   const policyMode = legacyModeFromDefaultAction(defaultAction)
-  const portRules = expandPortRules(db, agentId)
+  const portRules = expandPortRules(ctx, portRuleRows)
 
   const payload = JSON.stringify({
     apply_version: POLICY_APPLY_VERSION,

apps/api/src/services/policy/policy-mode.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 async function createAgent(

apps/api/src/services/port-acl.test.ts

@@ -16,6 +16,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 async function enrollApprovedLinux(

apps/api/src/services/retention-pagination.test.ts

@@ -0,0 +1,101 @@
+import { describe, it, expect, afterAll } from 'vitest'
+import { buildApp } from '../app.js'
+import { repos } from '@evofw/db'
+import type { AppConfig } from '../config.js'
+
+const testConfig: AppConfig = {
+  databaseUrl: 'sqlite::memory:',
+  jwtSecret: 'test',
+  jwtTtlHours: 24,
+  serverPort: 8080,
+  staticDir: null,
+  logLevel: 'error',
+  authRequired: false,
+  authIssuer: 'https://auth.test',
+  authPortalUrl: 'http://localhost:5175',
+  publicBaseUrl: 'https://fw.example.com',
+  enrollSeed: 'test-seed',
+  corsOrigins: [],
+  secretKey: null,
+  statsRetentionDays: 30,
+}
+
+describe('stats retention + list pagination', () => {
+  const appPromise = buildApp({ memory: true, config: testConfig })
+
+  afterAll(async () => {
+    const app = await appPromise
+    await app.close()
+  })
+
+  it('deleteStatsSamplesBefore drops only old samples', async () => {
+    const app = await appPromise
+    await app.ready()
+
+    const created = await app.inject({
+      method: 'POST',
+      url: '/api/v1/install-links',
+      payload: { name: 'retention-agent', platform: 'linux' },
+    })
+    const agentId = (created.json() as { agent_id: string }).agent_id
+    const iso = (daysAgo: number) =>
+      new Date(Date.now() - daysAgo * 86_400_000).toISOString()
+    for (const daysAgo of [60, 45, 10, 0]) {
+      repos.insertStatsSample(app.db, {
+        id: crypto.randomUUID(),
+        agentId,
+        packetsDropped: daysAgo,
+        packetsAccepted: 0,
+        prefixCount: 0,
+        kernelMethod: null,
+        recordedAt: iso(daysAgo),
+      })
+    }
+
+    const cutoff = new Date(Date.now() - 30 * 86_400_000).toISOString()
+    const deleted = repos.deleteStatsSamplesBefore(app.db, cutoff)
+    expect(deleted).toBe(2)
+
+    const remaining = repos.listStatsSamples(app.db, agentId, 100)
+    expect(remaining.map((s) => s.packetsDropped).sort()).toEqual([0, 10])
+  })
+
+  it('list endpoints return total and honor limit/offset', async () => {
+    const app = await appPromise
+    await app.ready()
+
+    for (const name of ['page-a', 'page-b', 'page-c']) {
+      await app.inject({
+        method: 'POST',
+        url: '/api/v1/install-links',
+        payload: { name, platform: 'linux' },
+      })
+    }
+    await app.inject({
+      method: 'POST',
+      url: '/api/v1/lists',
+      payload: { name: 'page-list', type: 'static' },
+    })
+
+    const all = await app.inject({ method: 'GET', url: '/api/v1/agents' })
+    expect(all.statusCode).toBe(200)
+    const allBody = all.json() as { items: unknown[]; total: number }
+    expect(allBody.items.length).toBe(allBody.total)
+
+    const page = await app.inject({
+      method: 'GET',
+      url: '/api/v1/agents?limit=1&offset=1',
+    })
+    const pageBody = page.json() as { items: unknown[]; total: number }
+    expect(pageBody.items).toHaveLength(1)
+    expect(pageBody.total).toBe(allBody.total)
+
+    const lists = await app.inject({
+      method: 'GET',
+      url: '/api/v1/lists?limit=1',
+    })
+    const listsBody = lists.json() as { items: unknown[]; total: number }
+    expect(listsBody.items).toHaveLength(1)
+    expect(listsBody.total).toBeGreaterThanOrEqual(1)
+  })
+})

apps/api/src/services/row-mappers.ts

@@ -67,6 +67,21 @@ export function mapPolicySets(
 export function mapPolicyRule(
   r: NonNullable<ReturnType<typeof repos.getPolicyRule>>,
   db: Parameters<typeof repos.listResolvedForRule>[0],
+) {
+  return mapPolicyRuleWithCounts(r, r.hostname ? countResolved(db, [r.id]).get(r.id) ?? 0 : undefined)
+}
+
+function countResolved(
+  db: Parameters<typeof repos.listResolvedForRule>[0],
+  ruleIds: string[],
+): Map<string, number> {
+  const cidrs = repos.mapResolvedCidrsByRuleIds(db, ruleIds)
+  return new Map([...cidrs].map(([id, list]) => [id, list.length]))
+}
+
+function mapPolicyRuleWithCounts(
+  r: NonNullable<ReturnType<typeof repos.getPolicyRule>>,
+  resolvedCount: number | undefined,
 ) {
   return {
     id: r.id,
@@ -77,11 +92,23 @@ export function mapPolicyRule(
     list_id: r.listId,
     cidr: r.cidr,
     hostname: r.hostname,
-    resolved_count: r.hostname
-      ? repos.listResolvedForRule(db, r.id).length
-      : undefined,
+    resolved_count: resolvedCount,
     comment: r.comment,
     created_at: r.createdAt,
     updated_at: r.updatedAt,
   }
 }
+
+/** Batched variant for list endpoints: one resolved-counts query for all rules. */
+export function mapPolicyRules(
+  rules: NonNullable<ReturnType<typeof repos.getPolicyRule>>[],
+  db: Parameters<typeof repos.listResolvedForRule>[0],
+) {
+  const counts = countResolved(
+    db,
+    rules.filter((r) => r.hostname).map((r) => r.id),
+  )
+  return rules.map((r) =>
+    mapPolicyRuleWithCounts(r, r.hostname ? counts.get(r.id) ?? 0 : undefined),
+  )
+}

apps/api/src/services/settings-bump.test.ts

@@ -17,6 +17,7 @@ const testConfig: AppConfig = {
   enrollSeed: 'test-seed',
   corsOrigins: [],
   secretKey: null,
+  statsRetentionDays: 30,
 }
 
 describe('settings + bumpAgentsForList', () => {