From 402182195fc401de848410312082a06d108a0735 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Fri, 7 Aug 2026 15:44:27 +0700 Subject: [PATCH] feat(api, test, web): enhance IP hit tracking and reset logic for agents - Implemented a reset mechanism for per-IP baselines in the agent routes, ensuring accurate tracking after policy application. - Updated tests to simulate traffic flush scenarios, verifying that IP hit statistics reset correctly and accumulate as expected. - Modified the UI to reflect changes in terminology from "Sync windows" to "Hits" for better clarity in agent details. - Enhanced documentation to explain the new behavior of IP hit tracking and baseline resets, improving user understanding. These changes improve the accuracy and usability of IP hit tracking for agents, particularly in scenarios involving policy changes. --- apps/api/src/routes/agent.ts | 6 + apps/api/src/services/ip-block-stats.test.ts | 153 ++++++++++++++---- .../components/agents/agent-blocked-ips.tsx | 4 +- docs/agents.md | 9 +- packages/db/src/repositories/index.ts | 5 + packages/db/src/repositories/stats.ts | 38 ++++- 6 files changed, 171 insertions(+), 44 deletions(-) diff --git a/apps/api/src/routes/agent.ts b/apps/api/src/routes/agent.ts index a1bdb6f..0d8624e 100644 --- a/apps/api/src/routes/agent.ts +++ b/apps/api/src/routes/agent.ts @@ -199,6 +199,12 @@ export const agentRoutes: FastifyPluginAsync<{ config: AppConfig }> = async ( const totalDropped = (prev?.totalPacketsDropped ?? 0) + deltaDropped const totalAccepted = (prev?.totalPacketsAccepted ?? 0) + deltaAccepted + // Chain/set counters were flushed (policy apply). Zero per-IP baselines so + // the next epoch of element counters accumulates (zeros are omitted from ip_hits). + if (reportedDropped < prevDropped) { + repos.resetIpBlockStatsBaselines(app.db, agentId) + } + repos.updateAgent(app.db, agentId, { lastApplyAt: now, lastApplyStatus: body.status, diff --git a/apps/api/src/services/ip-block-stats.test.ts b/apps/api/src/services/ip-block-stats.test.ts index 10d90a3..df1caf9 100644 --- a/apps/api/src/services/ip-block-stats.test.ts +++ b/apps/api/src/services/ip-block-stats.test.ts @@ -140,6 +140,73 @@ describe('apply-report ip_hits / blocked-ips', () => { expect(body2.items.find((i) => i.ip === '203.0.113.10')?.packets).toBe(18) expect(body2.items.find((i) => i.ip === '198.51.100.0/24')?.packets).toBe(5) + // Simulate nft flush: Traffic absolute drops; ip_hits omit zeros → baselines must reset + const reportReset = await app.inject({ + method: 'POST', + url: '/v1/agent/apply-report', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + payload: { + status: 'ok', + prefix_count: 2, + packets_dropped: 0, + packets_accepted: 0, + kernel_method: 'nft', + source: 'agent', + ip_hits: [], + }, + }) + expect(reportReset.statusCode).toBe(200) + + const report3 = await app.inject({ + method: 'POST', + url: '/v1/agent/apply-report', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + payload: { + status: 'ok', + prefix_count: 2, + packets_dropped: 7, + packets_accepted: 0, + kernel_method: 'nft', + source: 'agent', + ip_hits: [ + { ip: '203.0.113.10', packets: 4 }, + { ip: '198.51.100.0/24', packets: 3 }, + ], + }, + }) + expect(report3.statusCode).toBe(200) + + const listAfterFlush = await app.inject({ + method: 'GET', + url: `/api/v1/agents/${agentId}/blocked-ips`, + }) + const bodyFlush = listAfterFlush.json() as { + items: { ip: string; packets: number }[] + } + // First epoch 18+5 plus second epoch 4+3 + expect(bodyFlush.items.find((i) => i.ip === '203.0.113.10')?.packets).toBe( + 22, + ) + expect( + bodyFlush.items.find((i) => i.ip === '198.51.100.0/24')?.packets, + ).toBe(8) + + const agentAfter = await app.inject({ + method: 'GET', + url: `/api/v1/agents/${agentId}`, + }) + const agentBody = agentAfter.json() as { + total_packets_dropped?: number + } + // Traffic: 25 + 0 + 7 = 32 + expect(agentBody.total_packets_dropped).toBe(32) + const reset = await app.inject({ method: 'POST', url: `/api/v1/agents/${agentId}/stats/reset`, @@ -186,7 +253,7 @@ describe('apply-report ip_hits / blocked-ips', () => { expect(report.statusCode).toBeGreaterThanOrEqual(400) }) - it('mikrotik presence mode increments packets and refreshes last_seen', async () => { + it('mikrotik presence: continuous sync refreshes last_seen without +1; rehit after stale gap', async () => { const app = await appPromise await app.ready() @@ -217,22 +284,27 @@ describe('apply-report ip_hits / blocked-ips', () => { url: `/api/v1/agents/${link.agent_id}/approve`, }) - const report1 = await app.inject({ - method: 'POST', - url: '/v1/agent/apply-report', - headers: { - authorization: `Bearer ${token}`, - 'content-type': 'application/json', - }, - payload: { - status: 'ok', - packets_dropped: 3, - kernel_method: 'address-list', - source: 'mikrotik', - ip_hits: [{ ip: '203.0.113.50', packets: 1 }], - }, + const reportPayload = (dropped: number) => ({ + status: 'ok', + packets_dropped: dropped, + kernel_method: 'address-list', + source: 'mikrotik', + ip_hits: [{ ip: '203.0.113.50', packets: 1 }], }) - expect(report1.statusCode).toBe(200) + + expect( + ( + await app.inject({ + method: 'POST', + url: '/v1/agent/apply-report', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + payload: reportPayload(3), + }) + ).statusCode, + ).toBe(200) const list1 = await app.inject({ method: 'GET', @@ -247,22 +319,19 @@ describe('apply-report ip_hits / blocked-ips', () => { await new Promise((r) => setTimeout(r, 5)) - const report2 = await app.inject({ - method: 'POST', - url: '/v1/agent/apply-report', - headers: { - authorization: `Bearer ${token}`, - 'content-type': 'application/json', - }, - payload: { - status: 'ok', - packets_dropped: 5, - kernel_method: 'address-list', - source: 'mikrotik', - ip_hits: [{ ip: '203.0.113.50', packets: 1 }], - }, - }) - expect(report2.statusCode).toBe(200) + expect( + ( + await app.inject({ + method: 'POST', + url: '/v1/agent/apply-report', + headers: { + authorization: `Bearer ${token}`, + 'content-type': 'application/json', + }, + payload: reportPayload(5), + }) + ).statusCode, + ).toBe(200) const list2 = await app.inject({ method: 'GET', @@ -271,7 +340,25 @@ describe('apply-report ip_hits / blocked-ips', () => { const body2 = list2.json() as { items: { ip: string; packets: number; last_seen_at: string }[] } - expect(body2.items[0]?.packets).toBe(2) + expect(body2.items[0]?.packets).toBe(1) expect(body2.items[0]!.last_seen_at >= firstSeen).toBe(true) + + // Re-entry after stale gap (> PRESENCE_REHIT_STALE_MS): simulate via future `now` + const { repos } = await import('@evofw/db') + repos.upsertIpBlockStats( + app.db, + link.agent_id, + [{ ip: '203.0.113.50', packets: 1 }], + new Date(Date.now() + 200_000).toISOString(), + { mode: 'presence' }, + ) + + const list3 = await app.inject({ + method: 'GET', + url: `/api/v1/agents/${link.agent_id}/blocked-ips`, + }) + expect( + (list3.json() as { items: { packets: number }[] }).items[0]?.packets, + ).toBe(2) }) }) diff --git a/apps/web/src/components/agents/agent-blocked-ips.tsx b/apps/web/src/components/agents/agent-blocked-ips.tsx index 3999e24..f62e1c7 100644 --- a/apps/web/src/components/agents/agent-blocked-ips.tsx +++ b/apps/web/src/components/agents/agent-blocked-ips.tsx @@ -58,9 +58,9 @@ export function AgentBlockedIps({ agentId, platform }: AgentBlockedIpsProps) { const isMikrotik = platform === 'mikrotik' const q = useQuery(agentBlockedIpsQueryOptions(agentId)) - const packetsTitle = isMikrotik ? 'Sync windows' : 'Packets' + const packetsTitle = isMikrotik ? 'Hits' : 'Packets' const description = isMikrotik - ? 'Src /32 из EVOFW_HITS (add-src при deny, timeout 1h). Sync windows — сколько минут IP был в hits.' + ? 'Src /32 из EVOFW_HITS (add-src при deny, timeout 1h). Hits — входы в список (не каждый sync); Last seen обновляется, пока IP в hits.' : 'Drop-пакеты по записям deny (nft/ipset). Top по накопленным packets.' const columns = useMemo[]>( diff --git a/docs/agents.md b/docs/agents.md index c8376d1..627ab97 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -61,8 +61,9 @@ Linux agent reports optional `ip_hits` in `POST /v1/agent/apply-report`: - Upgrade path: on install-link re-run, `last_hash` is cleared once so sets can be recreated (chain deleted before set replace). - **ipset:** prefers `hash:net … counters` on create; existing sets without counters are left as-is. - Payload: only entries with `packets > 0`, **top 200** by packets. -- Control plane: `agent_ip_block_stats`, `GET /api/v1/agents/:id/blocked-ips`, reset via `POST …/stats/reset`. -- UI: agent detail → **Blocked IPs**. +- Control plane: `agent_ip_block_stats`, accumulates **deltas** of absolute kernel counters (как Traffic ↓). После flush set/chain (policy apply) CP сбрасывает per-IP baseline (`last_reported`), иначе вторая эпоха счётчиков теряется (Traffic растёт, Blocked IPs — нет). +- `GET /api/v1/agents/:id/blocked-ips`, reset via `POST …/stats/reset`. +- UI: agent detail → **Blocked IPs**. Sum of Blocked IPs ≈ Traffic ↓ для deny (при default accept); default-drop / allow в Traffic считаются отдельно. IPv6 skipped. @@ -92,9 +93,9 @@ Traffic ↓/↑ в UI — сумма `packets` с `evofw-deny-drop-*` / `evofw-a 1. `evofw-deny-hit-input/forward` — `add-src-to-address-list` → `EVOFW_HITS`, `address-list-timeout=1h` (passthrough). 2. `evofw-deny-drop-input/forward` — `drop` по `EVOFW_DENY`. -В `EVOFW_HITS` попадают реальные src **/32**. Policy rebuild **не** чистит HITS (только DENY/ALLOW). Sync шлёт top-200 в `ip_hits`; CP mode **presence**: `last_seen` каждый report, `packets` = число sync-окон (~минут), пока IP в HITS. +В `EVOFW_HITS` попадают реальные src **/32**. Policy rebuild **не** чистит HITS (только DENY/ALLOW). Sync шлёт top-200 в `ip_hits`; CP mode **presence**: `last_seen` обновляется, пока IP в HITS; `packets` (Hits) увеличивается только при первом появлении или **повторном входе** после исчезновения из списка (~>2.5 мин без report), а не на каждый sync. -UI: agent detail → **Blocked IPs** (колонка Sync windows). +UI: agent detail → **Blocked IPs** (колонка Hits). **Default action** задаётся на **агенте** (`default_action: accept | drop`): diff --git a/packages/db/src/repositories/index.ts b/packages/db/src/repositories/index.ts index 9e25dda..1144d1c 100644 --- a/packages/db/src/repositories/index.ts +++ b/packages/db/src/repositories/index.ts @@ -60,7 +60,10 @@ export { upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, + resetIpBlockStatsBaselines, + PRESENCE_REHIT_STALE_MS, } from './stats.js' +export type { UpsertIpBlockStatsOptions, IpHitInput } from './stats.js' export { getSetting, @@ -138,6 +141,7 @@ import { upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, + resetIpBlockStatsBaselines, } from './stats.js' import { getSetting, @@ -207,6 +211,7 @@ export const repos = { upsertIpBlockStats, listIpBlockStats, deleteIpBlockStatsForAgent, + resetIpBlockStatsBaselines, getSetting, setSetting, listSettings, diff --git a/packages/db/src/repositories/stats.ts b/packages/db/src/repositories/stats.ts index 9c89701..83e40f7 100644 --- a/packages/db/src/repositories/stats.ts +++ b/packages/db/src/repositories/stats.ts @@ -36,19 +36,24 @@ export function deleteStatsSamplesForAgent(db: Db, agentId: string) { export type IpHitInput = { ip: string; packets: number } +/** Gap after which a new presence report counts as a re-hit (left EVOFW_HITS). */ +export const PRESENCE_REHIT_STALE_MS = 150_000 + export type UpsertIpBlockStatsOptions = { /** - * MikroTik EVOFW_HITS presence: always refresh last_seen; - * packets += 1 per report (sync-window sightings). + * MikroTik EVOFW_HITS presence: refresh last_seen every report while IP is listed; + * increment packets only on first see or re-hit after stale gap (left the list). * Linux keeps absolute counter deltas (default). */ mode?: 'absolute' | 'presence' + /** Override re-hit gap (tests). Default PRESENCE_REHIT_STALE_MS. */ + presenceStaleMs?: number } /** * Upsert per-IP drop counters. * - absolute (Linux): agent reports kernel counters; CP accumulates deltas. - * - presence (MikroTik): each report sighting → last_seen=now, packets+=1. + * - presence (MikroTik): last_seen while in HITS; packets = entries into HITS (not per-sync). */ export function upsertIpBlockStats( db: Db, @@ -58,6 +63,9 @@ export function upsertIpBlockStats( opts: UpsertIpBlockStatsOptions = {}, ) { const mode = opts.mode ?? 'absolute' + const staleMs = opts.presenceStaleMs ?? PRESENCE_REHIT_STALE_MS + const nowMs = Date.parse(now) + for (const hit of hits) { const ip = hit.ip.trim() if (!ip) continue @@ -88,11 +96,18 @@ export function upsertIpBlockStats( .run() continue } + const prevSeen = Date.parse(existing.lastSeenAt) + const gapMs = Number.isFinite(prevSeen) ? nowMs - prevSeen : staleMs + 1 + const isRehit = gapMs > staleMs db.update(agentIpBlockStats) .set({ - packets: (existing.packets ?? 0) + 1, - lastReportedPackets: (existing.lastReportedPackets ?? 0) + 1, lastSeenAt: now, + ...(isRehit + ? { + packets: (existing.packets ?? 0) + 1, + lastReportedPackets: (existing.lastReportedPackets ?? 0) + 1, + } + : {}), }) .where(eq(agentIpBlockStats.id, existing.id)) .run() @@ -144,3 +159,16 @@ export function deleteIpBlockStatsForAgent(db: Db, agentId: string) { .where(eq(agentIpBlockStats.agentId, agentId)) .run() } + +/** + * After nft/ipset counter flush (policy apply), kernel absolutes restart at 0. + * Traffic detects this via packets_dropped drop; per-IP baselines must reset too, + * otherwise ip_hits with packets=0 are omitted from the report and lastReported + * stays high → second epoch deltas are lost (Traffic 14 vs Blocked IPs 7). + */ +export function resetIpBlockStatsBaselines(db: Db, agentId: string) { + db.update(agentIpBlockStats) + .set({ lastReportedPackets: 0 }) + .where(eq(agentIpBlockStats.agentId, agentId)) + .run() +}