diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 44060aa..6306681 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -140,6 +140,34 @@ export async function buildApp(opts: BuildAppOptions = {}) { preventOverrun: true, }), ) + + // Raw stats samples grow ~1 row per agent per sync interval; drop old + // ones daily. Lifetime totals live on agents; aggregates are kept. + const runRetention = async () => { + const cutoff = new Date( + Date.now() - config.statsRetentionDays * 86_400_000, + ).toISOString() + const deleted = repos.deleteStatsSamplesBefore(app.db, cutoff) + if (deleted > 0) { + app.log.info( + { deleted, cutoff }, + `stats retention: raw samples older than ${config.statsRetentionDays}d removed`, + ) + } + } + app.scheduler.addCronJob( + new CronJob( + { cronExpression: '17 3 * * *' }, + new AsyncTask('stats-retention', runRetention, (err) => { + app.log.warn({ err }, 'stats retention failed') + }), + { preventOverrun: true }, + ), + ) + // First cleanup right at startup, not only after the next 03:17. + runRetention().catch((err) => + app.log.warn({ err }, 'stats retention failed'), + ) } return app diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index ff692b3..dcd5eed 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -15,6 +15,7 @@ export interface AppConfig { enrollSeed: string corsOrigins: string[] secretKey: string | null + statsRetentionDays: number } function boolEnv(v: string | undefined, fallback: boolean): boolean { @@ -65,6 +66,10 @@ export function loadConfig(): AppConfig { .map((s) => s.trim().replace(/\/$/, '')) .filter(Boolean), secretKey: process.env.EVOFW_SECRET_KEY?.trim() || null, + statsRetentionDays: Math.max( + 1, + Math.min(3650, Number(process.env.STATS_RETENTION_DAYS ?? '30') || 30), + ), } // Fail-safe: a production process must not start wide open or with diff --git a/apps/api/src/routes/agents.ts b/apps/api/src/routes/agents.ts index d7448ef..b2bf52c 100644 --- a/apps/api/src/routes/agents.ts +++ b/apps/api/src/routes/agents.ts @@ -13,6 +13,7 @@ import { buildInstallUrls } from '../services/install-links.js' import type { AppConfig } from '../config.js' import { auditMutation } from '../services/audit.js' import { mapAgent } from '../services/row-mappers.js' +import { applyPagination } from '../services/pagination.js' export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( app, @@ -20,11 +21,12 @@ export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( ) => { const { config } = opts - app.get('/agents', async () => { - const all = repos.listAgents(app.db) - const linksByAgent = repos.mapActiveInstallLinksByAgentId(app.db) - return { - items: all.map((a) => { + app.get<{ Querystring: { limit?: string; offset?: string } }>( + '/agents', + async (req) => { + const all = repos.listAgents(app.db) + const linksByAgent = repos.mapActiveInstallLinksByAgentId(app.db) + const items = all.map((a) => { const link = linksByAgent.get(a.id) if (!link) { return mapAgent(a) @@ -39,9 +41,11 @@ export const agentsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( installCurl: urls.curl.by_slug, installLinkId: link.id, }) - }), - } - }) + }) + const paged = applyPagination(items, req.query) + return { items: paged.items, total: paged.total } + }, + ) app.get<{ Params: { id: string } }>('/agents/:id', async (req) => { const a = repos.getAgent(app.db, req.params.id) diff --git a/apps/api/src/routes/install-links.ts b/apps/api/src/routes/install-links.ts index e5bbf63..ffde1bb 100644 --- a/apps/api/src/routes/install-links.ts +++ b/apps/api/src/routes/install-links.ts @@ -9,6 +9,7 @@ import { import { hashToken } from '../plugins/auth.js' import type { AppConfig } from '../config.js' import { auditMutation } from '../services/audit.js' +import { applyPagination } from '../services/pagination.js' export const installLinksRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( app, @@ -16,11 +17,16 @@ export const installLinksRoutes: FastifyPluginAsync<{ config: AppConfig }> = asy ) => { const { config } = opts - app.get('/install-links', async () => ({ - items: repos - .listInstallLinks(app.db) - .map((row) => mapInstallLink(row, config.publicBaseUrl)), - })) + app.get<{ Querystring: { limit?: string; offset?: string } }>( + '/install-links', + async (req) => { + const items = repos + .listInstallLinks(app.db) + .map((row) => mapInstallLink(row, config.publicBaseUrl)) + const paged = applyPagination(items, req.query) + return { items: paged.items, total: paged.total } + }, + ) app.post('/install-links', async (req, reply) => { const body = createInstallLinkBodySchema.parse(req.body) diff --git a/apps/api/src/routes/lists.ts b/apps/api/src/routes/lists.ts index 39fcc0e..106209e 100644 --- a/apps/api/src/routes/lists.ts +++ b/apps/api/src/routes/lists.ts @@ -16,6 +16,7 @@ import { import type { AppConfig } from '../config.js' import { auditMutation } from '../services/audit.js' import { maskListConfig, sealListConfig } from '../services/secret-cipher.js' +import { applyPagination } from '../services/pagination.js' export const listsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( app, @@ -23,26 +24,30 @@ export const listsRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( ) => { const { config } = opts - app.get('/lists', async () => { - const lists = repos.listIpLists(app.db) - const counts = repos.countEntriesByListIds( - app.db, - lists.map((l) => l.id), - ) - const items = lists.map((l) => ({ - id: l.id, - name: l.name, - type: l.type, - config_json: maskListConfig(l.configJson), - content_hash: l.contentHash, - refreshed_at: l.refreshedAt, - last_error: l.lastError, - entry_count: counts.get(l.id) ?? 0, - created_at: l.createdAt, - updated_at: l.updatedAt, - })) - return { items } - }) + app.get<{ Querystring: { limit?: string; offset?: string } }>( + '/lists', + async (req) => { + const lists = repos.listIpLists(app.db) + const counts = repos.countEntriesByListIds( + app.db, + lists.map((l) => l.id), + ) + const items = lists.map((l) => ({ + id: l.id, + name: l.name, + type: l.type, + config_json: maskListConfig(l.configJson), + content_hash: l.contentHash, + refreshed_at: l.refreshedAt, + last_error: l.lastError, + entry_count: counts.get(l.id) ?? 0, + created_at: l.createdAt, + updated_at: l.updatedAt, + })) + const paged = applyPagination(items, req.query) + return { items: paged.items, total: paged.total } + }, + ) app.post('/lists', async (req) => { const body = createIpListBodySchema.parse(req.body) diff --git a/apps/api/src/routes/port-acl.ts b/apps/api/src/routes/port-acl.ts index a6208f1..4ffae6b 100644 --- a/apps/api/src/routes/port-acl.ts +++ b/apps/api/src/routes/port-acl.ts @@ -63,12 +63,17 @@ export const portAclRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( async (req) => { const agent = repos.getAgent(app.db, req.params.id) if (!agent) throw new AppError('NOT_FOUND', 'Agent not found', 404) - const items = repos.listAgentPortRules(app.db, agent.id).map((row) => { - const listName = row.listId - ? repos.getIpList(app.db, row.listId)?.name - : null - return mapPortRule(row, listName) - }) + const rows = repos.listAgentPortRules(app.db, agent.id) + const listNames = repos.mapIpListNames( + app.db, + rows.map((r) => r.listId).filter((id): id is string => Boolean(id)), + ) + const items = rows.map((row) => + mapPortRule( + row, + row.listId ? (listNames.get(row.listId) ?? null) : null, + ), + ) return { items } }, ) diff --git a/apps/api/src/routes/rules.ts b/apps/api/src/routes/rules.ts index 57e8b50..5e6f034 100644 --- a/apps/api/src/routes/rules.ts +++ b/apps/api/src/routes/rules.ts @@ -12,7 +12,8 @@ import { } from '../services/policy/resolve-hostname.js' import type { AppConfig } from '../config.js' import { auditMutation } from '../services/audit.js' -import { mapPolicyRule } from '../services/row-mappers.js' +import { mapPolicyRule, mapPolicyRules } from '../services/row-mappers.js' +import { applyPagination } from '../services/pagination.js' export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( app, @@ -26,29 +27,28 @@ export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( const s = repos.getPolicySet(app.db, req.params.id) if (!s) throw new AppError('NOT_FOUND', 'Policy set not found', 404) return { - items: repos - .listPolicyRules(app.db, s.id) - .map((r) => mapPolicyRule(r, app.db)), + items: mapPolicyRules(repos.listPolicyRules(app.db, s.id), app.db), } }, ) - app.get<{ Querystring: { set_id?: string; agent_id?: string } }>( - '/rules', - async (req) => { - if (req.query.agent_id) { - return { - items: repos - .listPolicyRulesForAgent(app.db, req.query.agent_id) - .map((r) => mapPolicyRule(r, app.db)), - } + app.get<{ + Querystring: { set_id?: string; agent_id?: string; limit?: string; offset?: string } + }>('/rules', async (req) => { + if (req.query.agent_id) { + return { + items: mapPolicyRules( + repos.listPolicyRulesForAgent(app.db, req.query.agent_id), + app.db, + ), } - const items = repos - .listPolicyRules(app.db, req.query.set_id) - .map((r) => mapPolicyRule(r, app.db)) - return { items } - }, - ) + } + const paged = applyPagination( + mapPolicyRules(repos.listPolicyRules(app.db, req.query.set_id), app.db), + req.query, + ) + return { items: paged.items, total: paged.total } + }) app.post('/rules', async (req) => { const body = createPolicyRuleBodySchema.parse(req.body) @@ -174,9 +174,10 @@ export const rulesRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( details: { set_id: s.id, ordered_ids: body.ordered_ids }, }) return { - items: repos - .listPolicyRules(app.db, s.id) - .map((r) => mapPolicyRule(r, app.db)), + items: mapPolicyRules( + repos.listPolicyRules(app.db, s.id), + app.db, + ), } }, ) diff --git a/apps/api/src/services/agents-policy.test.ts b/apps/api/src/services/agents-policy.test.ts index 32d2b75..8ec9208 100644 --- a/apps/api/src/services/agents-policy.test.ts +++ b/apps/api/src/services/agents-policy.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } describe('agents CRUD critical paths', () => { diff --git a/apps/api/src/services/bulk-ops.test.ts b/apps/api/src/services/bulk-ops.test.ts index fcf15d0..85d9954 100644 --- a/apps/api/src/services/bulk-ops.test.ts +++ b/apps/api/src/services/bulk-ops.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } async function createInvitedAgent( diff --git a/apps/api/src/services/install-links.test.ts b/apps/api/src/services/install-links.test.ts index 1addb95..6e47148 100644 --- a/apps/api/src/services/install-links.test.ts +++ b/apps/api/src/services/install-links.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } describe('install-links', () => { diff --git a/apps/api/src/services/ip-block-stats.test.ts b/apps/api/src/services/ip-block-stats.test.ts index dc47af2..dfa0de3 100644 --- a/apps/api/src/services/ip-block-stats.test.ts +++ b/apps/api/src/services/ip-block-stats.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } async function enrollApprovedLinux( diff --git a/apps/api/src/services/pagination.ts b/apps/api/src/services/pagination.ts new file mode 100644 index 0000000..2f2b930 --- /dev/null +++ b/apps/api/src/services/pagination.ts @@ -0,0 +1,22 @@ +/** + * Backwards-compatible list pagination: without limit/offset the response is + * the full list (total === items.length); with them, a page plus the total. + */ +export function applyPagination( + items: T[], + query: { limit?: string; offset?: string }, +): { items: T[]; total: number } { + const total = items.length + const limitRaw = query.limit !== undefined ? Number(query.limit) : NaN + const offsetRaw = query.offset !== undefined ? Number(query.offset) : NaN + if (!Number.isFinite(limitRaw) && !Number.isFinite(offsetRaw)) { + return { items, total } + } + const limit = Number.isFinite(limitRaw) + ? Math.max(1, Math.min(1000, Math.floor(limitRaw))) + : items.length + const offset = Number.isFinite(offsetRaw) + ? Math.max(0, Math.floor(offsetRaw)) + : 0 + return { items: items.slice(offset, offset + limit), total } +} diff --git a/apps/api/src/services/policy/evaluate.ts b/apps/api/src/services/policy/evaluate.ts index de7d8c0..ef44530 100644 --- a/apps/api/src/services/policy/evaluate.ts +++ b/apps/api/src/services/policy/evaluate.ts @@ -54,13 +54,41 @@ export type EvaluatedPolicy = { } } -function expandList(db: Db, listId: string | null | undefined): string[] { - if (!listId) return [] - return repos.listIpListEntries(db, listId).map((e) => e.cidr) +/** + * Prefetched expansion data: one batched query per kind instead of a query + * per rule (this code runs on every agent policy poll, ~60s per agent). + */ +type ExpansionContext = { + entriesByList: Map + resolvedByRule: Map + listNames: Map +} + +function buildExpansionContext( + db: Db, + rules: { id: string; listId: string | null; hostname: string | null }[], + portRuleRows: { listId: string | null }[], +): ExpansionContext { + const listIds = [ + ...new Set( + [ + ...rules.map((r) => r.listId?.trim() || ''), + ...portRuleRows.map((r) => r.listId?.trim() || ''), + ].filter(Boolean), + ), + ] + const hostnameRuleIds = [ + ...new Set(rules.filter((r) => r.hostname?.trim()).map((r) => r.id)), + ] + return { + entriesByList: repos.mapIpListEntriesByListIds(db, listIds), + resolvedByRule: repos.mapResolvedCidrsByRuleIds(db, hostnameRuleIds), + listNames: repos.mapIpListNames(db, listIds), + } } function expandRule( - db: Db, + ctx: ExpansionContext, rule: { cidr: string | null listId: string | null @@ -70,9 +98,10 @@ function expandRule( ): string[] { if (rule.cidr?.trim()) return [rule.cidr.trim()] if (rule.hostname?.trim()) { - return repos.listResolvedForRule(db, rule.id).map((r) => r.cidr) + return ctx.resolvedByRule.get(rule.id) ?? [] } - return expandList(db, rule.listId) + const listId = rule.listId?.trim() || '' + return listId ? (ctx.entriesByList.get(listId) ?? []) : [] } function resolveDefaultAction(agentDefaultAction: string | null | undefined): DefaultAction { @@ -83,7 +112,7 @@ function resolveDefaultAction(agentDefaultAction: string | null | undefined): De } function sourceMeta( - db: Db, + ctx: ExpansionContext, rule: { cidr: string | null listId: string | null @@ -97,12 +126,12 @@ function sourceMeta( return { kind: 'hostname', label: rule.hostname.trim() } } const listId = rule.listId?.trim() || '' - const name = listId ? repos.getIpList(db, listId)?.name : null + const name = listId ? ctx.listNames.get(listId) : null return { kind: 'list', label: name || listId || 'list' } } function expandPortSrcCidrs( - db: Db, + ctx: ExpansionContext, row: { srcKind: string srcCidr: string | null @@ -114,18 +143,22 @@ function expandPortSrcCidrs( return [row.srcCidr.trim()] } if (row.srcKind === 'list') { - const cidrs = expandList(db, row.listId) + const cidrs = row.listId + ? (ctx.entriesByList.get(row.listId.trim()) ?? []) + : [] return cidrs.length ? uniqCidrs(cidrs) : [] } return [] } -function expandPortRules(db: Db, agentId: string): EvaluatedPortRule[] { - const rows = repos.listEnabledAgentPortRules(db, agentId) +function expandPortRules( + ctx: ExpansionContext, + rows: ReturnType, +): EvaluatedPortRule[] { const out: EvaluatedPortRule[] = [] for (const row of rows) { const action = row.action === 'close' ? 'close' : 'open' - const srcCidrs = expandPortSrcCidrs(db, row) + const srcCidrs = expandPortSrcCidrs(ctx, row) if (!srcCidrs.length) continue const portStart = Math.max(1, Math.min(65535, row.portStart)) const portEnd = Math.max(portStart, Math.min(65535, row.portEnd)) @@ -161,6 +194,8 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy { .filter((s) => s.enabled === 1) const ordered = repos.listPolicyRulesForAgent(db, agentId) const overrides = repos.listOverrides(db, agentId) + const portRuleRows = repos.listEnabledAgentPortRules(db, agentId) + const ctx = buildExpansionContext(db, ordered, portRuleRows) const deny: string[] = [] const allow: string[] = [] @@ -169,7 +204,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy { let rulesAllow = 0 for (const rule of ordered) { - const cidrs = expandRule(db, rule) + const cidrs = expandRule(ctx, rule) const action = rule.action === 'deny' ? 'deny' : 'allow' if (action === 'deny') { deny.push(...cidrs) @@ -178,7 +213,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy { allow.push(...cidrs) rulesAllow += 1 } - const src = sourceMeta(db, rule) + const src = sourceMeta(ctx, rule) const setName = assignedSets.find((s) => s.setId === rule.setId)?.name ?? null chain.push({ @@ -214,7 +249,7 @@ export function evaluateAgentPolicy(db: Db, agentId: string): EvaluatedPolicy { const conflictsDropped = allowRaw.length - allowCidrs.length const defaultAction = resolveDefaultAction(agent.defaultAction) const policyMode = legacyModeFromDefaultAction(defaultAction) - const portRules = expandPortRules(db, agentId) + const portRules = expandPortRules(ctx, portRuleRows) const payload = JSON.stringify({ apply_version: POLICY_APPLY_VERSION, diff --git a/apps/api/src/services/policy/policy-mode.test.ts b/apps/api/src/services/policy/policy-mode.test.ts index 8a888c0..abd42b9 100644 --- a/apps/api/src/services/policy/policy-mode.test.ts +++ b/apps/api/src/services/policy/policy-mode.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } async function createAgent( diff --git a/apps/api/src/services/port-acl.test.ts b/apps/api/src/services/port-acl.test.ts index f13b9d8..ee7949b 100644 --- a/apps/api/src/services/port-acl.test.ts +++ b/apps/api/src/services/port-acl.test.ts @@ -16,6 +16,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } async function enrollApprovedLinux( diff --git a/apps/api/src/services/retention-pagination.test.ts b/apps/api/src/services/retention-pagination.test.ts new file mode 100644 index 0000000..66c716a --- /dev/null +++ b/apps/api/src/services/retention-pagination.test.ts @@ -0,0 +1,101 @@ +import { describe, it, expect, afterAll } from 'vitest' +import { buildApp } from '../app.js' +import { repos } from '@evofw/db' +import type { AppConfig } from '../config.js' + +const testConfig: AppConfig = { + databaseUrl: 'sqlite::memory:', + jwtSecret: 'test', + jwtTtlHours: 24, + serverPort: 8080, + staticDir: null, + logLevel: 'error', + authRequired: false, + authIssuer: 'https://auth.test', + authPortalUrl: 'http://localhost:5175', + publicBaseUrl: 'https://fw.example.com', + enrollSeed: 'test-seed', + corsOrigins: [], + secretKey: null, + statsRetentionDays: 30, +} + +describe('stats retention + list pagination', () => { + const appPromise = buildApp({ memory: true, config: testConfig }) + + afterAll(async () => { + const app = await appPromise + await app.close() + }) + + it('deleteStatsSamplesBefore drops only old samples', async () => { + const app = await appPromise + await app.ready() + + const created = await app.inject({ + method: 'POST', + url: '/api/v1/install-links', + payload: { name: 'retention-agent', platform: 'linux' }, + }) + const agentId = (created.json() as { agent_id: string }).agent_id + const iso = (daysAgo: number) => + new Date(Date.now() - daysAgo * 86_400_000).toISOString() + for (const daysAgo of [60, 45, 10, 0]) { + repos.insertStatsSample(app.db, { + id: crypto.randomUUID(), + agentId, + packetsDropped: daysAgo, + packetsAccepted: 0, + prefixCount: 0, + kernelMethod: null, + recordedAt: iso(daysAgo), + }) + } + + const cutoff = new Date(Date.now() - 30 * 86_400_000).toISOString() + const deleted = repos.deleteStatsSamplesBefore(app.db, cutoff) + expect(deleted).toBe(2) + + const remaining = repos.listStatsSamples(app.db, agentId, 100) + expect(remaining.map((s) => s.packetsDropped).sort()).toEqual([0, 10]) + }) + + it('list endpoints return total and honor limit/offset', async () => { + const app = await appPromise + await app.ready() + + for (const name of ['page-a', 'page-b', 'page-c']) { + await app.inject({ + method: 'POST', + url: '/api/v1/install-links', + payload: { name, platform: 'linux' }, + }) + } + await app.inject({ + method: 'POST', + url: '/api/v1/lists', + payload: { name: 'page-list', type: 'static' }, + }) + + const all = await app.inject({ method: 'GET', url: '/api/v1/agents' }) + expect(all.statusCode).toBe(200) + const allBody = all.json() as { items: unknown[]; total: number } + expect(allBody.items.length).toBe(allBody.total) + + const page = await app.inject({ + method: 'GET', + url: '/api/v1/agents?limit=1&offset=1', + }) + const pageBody = page.json() as { items: unknown[]; total: number } + expect(pageBody.items).toHaveLength(1) + expect(pageBody.total).toBe(allBody.total) + + const lists = await app.inject({ + method: 'GET', + url: '/api/v1/lists?limit=1', + }) + const listsBody = lists.json() as { items: unknown[]; total: number } + expect(listsBody.items).toHaveLength(1) + expect(listsBody.total).toBeGreaterThanOrEqual(1) + }) +}) diff --git a/apps/api/src/services/row-mappers.ts b/apps/api/src/services/row-mappers.ts index 035ac08..2906243 100644 --- a/apps/api/src/services/row-mappers.ts +++ b/apps/api/src/services/row-mappers.ts @@ -67,6 +67,21 @@ export function mapPolicySets( export function mapPolicyRule( r: NonNullable>, db: Parameters[0], +) { + return mapPolicyRuleWithCounts(r, r.hostname ? countResolved(db, [r.id]).get(r.id) ?? 0 : undefined) +} + +function countResolved( + db: Parameters[0], + ruleIds: string[], +): Map { + const cidrs = repos.mapResolvedCidrsByRuleIds(db, ruleIds) + return new Map([...cidrs].map(([id, list]) => [id, list.length])) +} + +function mapPolicyRuleWithCounts( + r: NonNullable>, + resolvedCount: number | undefined, ) { return { id: r.id, @@ -77,11 +92,23 @@ export function mapPolicyRule( list_id: r.listId, cidr: r.cidr, hostname: r.hostname, - resolved_count: r.hostname - ? repos.listResolvedForRule(db, r.id).length - : undefined, + resolved_count: resolvedCount, comment: r.comment, created_at: r.createdAt, updated_at: r.updatedAt, } } + +/** Batched variant for list endpoints: one resolved-counts query for all rules. */ +export function mapPolicyRules( + rules: NonNullable>[], + db: Parameters[0], +) { + const counts = countResolved( + db, + rules.filter((r) => r.hostname).map((r) => r.id), + ) + return rules.map((r) => + mapPolicyRuleWithCounts(r, r.hostname ? counts.get(r.id) ?? 0 : undefined), + ) +} diff --git a/apps/api/src/services/settings-bump.test.ts b/apps/api/src/services/settings-bump.test.ts index 0ed4cc2..ddfaaee 100644 --- a/apps/api/src/services/settings-bump.test.ts +++ b/apps/api/src/services/settings-bump.test.ts @@ -17,6 +17,7 @@ const testConfig: AppConfig = { enrollSeed: 'test-seed', corsOrigins: [], secretKey: null, + statsRetentionDays: 30, } describe('settings + bumpAgentsForList', () => { diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 9dcd712..08f8224 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -43,9 +43,12 @@ paths: summary: List install links tags: [agents] security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/Offset' responses: '200': - description: Links + description: Links (items + total) post: summary: Create install link + invited agent tags: [agents] @@ -70,9 +73,33 @@ paths: summary: List agents tags: [agents] security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/Offset' responses: '200': - description: Agents + description: Agents (items + total) + + /api/v1/agents/approve-bulk: + post: + summary: Approve many pending/invited agents in one request + tags: [agents] + security: [{ bearerAuth: [] }] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [agent_ids] + properties: + agent_ids: + type: array + maxItems: 1000 + items: { type: string } + responses: + '200': + description: Approved agents (skips non-pending/invited) /api/v1/agents/{id}: get: @@ -199,9 +226,12 @@ paths: summary: List IP lists tags: [lists] security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/Offset' responses: '200': - description: Lists + description: Lists (items + total) post: summary: Create IP list tags: [lists] @@ -328,14 +358,33 @@ paths: '200': description: Reordered + /api/v1/policy-sets/{id}/agents: + put: + summary: Replace which agents have this set assigned + description: >- + Listed agents gain the set (other assignments preserved), unlisted + agents lose it. Body — { agent_ids: string[] } (empty array clears). + tags: [policies] + security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/Id' + responses: + '200': + description: '{ agent_ids, added, removed }' + '404': + description: Set or agent not found + /api/v1/rules: get: summary: List policy rules (optional set filter) tags: [policies] security: [{ bearerAuth: [] }] + parameters: + - $ref: '#/components/parameters/Limit' + - $ref: '#/components/parameters/Offset' responses: '200': - description: Rules + description: Rules (items + total) post: summary: Create policy rule tags: [policies] @@ -787,6 +836,16 @@ components: in: path required: true schema: { type: string } + Limit: + name: limit + in: query + schema: { type: integer, minimum: 1, maximum: 1000 } + description: Page size (without limit/offset the full list is returned) + Offset: + name: offset + in: query + schema: { type: integer, minimum: 0 } + description: Page offset (response includes total) schemas: AgentPortRule: type: object diff --git a/packages/db/src/repositories/index.ts b/packages/db/src/repositories/index.ts index 3be3668..534b16a 100644 --- a/packages/db/src/repositories/index.ts +++ b/packages/db/src/repositories/index.ts @@ -14,6 +14,8 @@ export { updateIpList, deleteIpList, listIpListEntries, + mapIpListEntriesByListIds, + mapIpListNames, countEntriesByListIds, replaceIpListEntries, } from './lists.js' @@ -45,6 +47,7 @@ export { deletePolicyRule, listHostnameRules, listResolvedForRule, + mapResolvedCidrsByRuleIds, replaceResolvedForRule, listOverrides, insertOverride, @@ -57,6 +60,7 @@ export { listStatsSamples, listRecentStats, deleteStatsSamplesForAgent, + deleteStatsSamplesBefore, upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, @@ -121,6 +125,8 @@ import { updateIpList, deleteIpList, listIpListEntries, + mapIpListEntriesByListIds, + mapIpListNames, countEntriesByListIds, replaceIpListEntries, } from './lists.js' @@ -151,6 +157,7 @@ import { deletePolicyRule, listHostnameRules, listResolvedForRule, + mapResolvedCidrsByRuleIds, replaceResolvedForRule, listOverrides, insertOverride, @@ -162,6 +169,7 @@ import { listStatsSamples, listRecentStats, deleteStatsSamplesForAgent, + deleteStatsSamplesBefore, upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, @@ -216,6 +224,8 @@ export const repos = { updateIpList, deleteIpList, listIpListEntries, + mapIpListEntriesByListIds, + mapIpListNames, countEntriesByListIds, replaceIpListEntries, listPolicySets, @@ -240,6 +250,7 @@ export const repos = { deletePolicyRule, listHostnameRules, listResolvedForRule, + mapResolvedCidrsByRuleIds, replaceResolvedForRule, listOverrides, insertOverride, @@ -248,6 +259,7 @@ export const repos = { listStatsSamples, listRecentStats, deleteStatsSamplesForAgent, + deleteStatsSamplesBefore, upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, diff --git a/packages/db/src/repositories/lists.ts b/packages/db/src/repositories/lists.ts index 01537bf..6568040 100644 --- a/packages/db/src/repositories/lists.ts +++ b/packages/db/src/repositories/lists.ts @@ -39,6 +39,34 @@ export function listIpListEntries(db: Db, listId: string) { .all() } +/** Entries for many lists in one query (agent policy hot path). */ +export function mapIpListEntriesByListIds( + db: Db, + listIds: string[], +): Map { + const map = new Map() + for (const id of listIds) map.set(id, []) + if (listIds.length === 0) return map + const rows = db + .select({ listId: ipListEntries.listId, cidr: ipListEntries.cidr }) + .from(ipListEntries) + .where(inArray(ipListEntries.listId, listIds)) + .all() + for (const r of rows) map.get(r.listId)?.push(r.cidr) + return map +} + +/** List names for many ids in one query. */ +export function mapIpListNames(db: Db, ids: string[]): Map { + if (ids.length === 0) return new Map() + const rows = db + .select({ id: ipLists.id, name: ipLists.name }) + .from(ipLists) + .where(inArray(ipLists.id, ids)) + .all() + return new Map(rows.map((r) => [r.id, r.name])) +} + /** Entry counts for many lists in one query. */ export function countEntriesByListIds( db: Db, diff --git a/packages/db/src/repositories/policy.ts b/packages/db/src/repositories/policy.ts index 984651c..2f11846 100644 --- a/packages/db/src/repositories/policy.ts +++ b/packages/db/src/repositories/policy.ts @@ -319,6 +319,26 @@ export function listResolvedForRule(db: Db, ruleId: string) { .all() } +/** Resolved CIDRs for many hostname rules in one query. */ +export function mapResolvedCidrsByRuleIds( + db: Db, + ruleIds: string[], +): Map { + const map = new Map() + for (const id of ruleIds) map.set(id, []) + if (ruleIds.length === 0) return map + const rows = db + .select({ + ruleId: policyRuleResolved.ruleId, + cidr: policyRuleResolved.cidr, + }) + .from(policyRuleResolved) + .where(inArray(policyRuleResolved.ruleId, ruleIds)) + .all() + for (const r of rows) map.get(r.ruleId)?.push(r.cidr) + return map +} + export function replaceResolvedForRule(db: Db, ruleId: string, cidrs: string[]) { db.transaction((tx) => { tx.delete(policyRuleResolved) diff --git a/packages/db/src/repositories/stats.ts b/packages/db/src/repositories/stats.ts index 7a00e76..18c07df 100644 --- a/packages/db/src/repositories/stats.ts +++ b/packages/db/src/repositories/stats.ts @@ -1,4 +1,4 @@ -import { and, eq, desc, sql, inArray } from 'drizzle-orm' +import { and, eq, desc, sql, inArray, lt } from 'drizzle-orm' import type { Db } from '../client.js' import { agentIpBlockStats, @@ -38,6 +38,18 @@ export function deleteStatsSamplesForAgent(db: Db, agentId: string) { .run() } +/** + * Retention: drop raw samples recorded before the cutoff ISO timestamp. + * Lifetime totals live on the agent row; per-IP/per-port aggregates are kept. + */ +export function deleteStatsSamplesBefore(db: Db, cutoffIso: string): number { + const result = db + .delete(agentStatsSamples) + .where(lt(agentStatsSamples.recordedAt, cutoffIso)) + .run() + return result.changes +} + export type IpHitInput = { ip: string; packets: number } /** Gap after which a new presence report counts as a re-hit (left EVOFW_HITS). */