feat(api, test, web): enhance IP hit tracking and reset logic for agents
- Implemented a reset mechanism for per-IP baselines in the agent routes, ensuring accurate tracking after policy application. - Updated tests to simulate traffic flush scenarios, verifying that IP hit statistics reset correctly and accumulate as expected. - Modified the UI to reflect changes in terminology from "Sync windows" to "Hits" for better clarity in agent details. - Enhanced documentation to explain the new behavior of IP hit tracking and baseline resets, improving user understanding. These changes improve the accuracy and usability of IP hit tracking for agents, particularly in scenarios involving policy changes.
This commit is contained in:
@@ -199,6 +199,12 @@ export const agentRoutes: FastifyPluginAsync<{ config: AppConfig }> = async (
|
||||
const totalDropped = (prev?.totalPacketsDropped ?? 0) + deltaDropped
|
||||
const totalAccepted = (prev?.totalPacketsAccepted ?? 0) + deltaAccepted
|
||||
|
||||
// Chain/set counters were flushed (policy apply). Zero per-IP baselines so
|
||||
// the next epoch of element counters accumulates (zeros are omitted from ip_hits).
|
||||
if (reportedDropped < prevDropped) {
|
||||
repos.resetIpBlockStatsBaselines(app.db, agentId)
|
||||
}
|
||||
|
||||
repos.updateAgent(app.db, agentId, {
|
||||
lastApplyAt: now,
|
||||
lastApplyStatus: body.status,
|
||||
|
||||
@@ -140,6 +140,73 @@ describe('apply-report ip_hits / blocked-ips', () => {
|
||||
expect(body2.items.find((i) => i.ip === '203.0.113.10')?.packets).toBe(18)
|
||||
expect(body2.items.find((i) => i.ip === '198.51.100.0/24')?.packets).toBe(5)
|
||||
|
||||
// Simulate nft flush: Traffic absolute drops; ip_hits omit zeros → baselines must reset
|
||||
const reportReset = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: {
|
||||
status: 'ok',
|
||||
prefix_count: 2,
|
||||
packets_dropped: 0,
|
||||
packets_accepted: 0,
|
||||
kernel_method: 'nft',
|
||||
source: 'agent',
|
||||
ip_hits: [],
|
||||
},
|
||||
})
|
||||
expect(reportReset.statusCode).toBe(200)
|
||||
|
||||
const report3 = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: {
|
||||
status: 'ok',
|
||||
prefix_count: 2,
|
||||
packets_dropped: 7,
|
||||
packets_accepted: 0,
|
||||
kernel_method: 'nft',
|
||||
source: 'agent',
|
||||
ip_hits: [
|
||||
{ ip: '203.0.113.10', packets: 4 },
|
||||
{ ip: '198.51.100.0/24', packets: 3 },
|
||||
],
|
||||
},
|
||||
})
|
||||
expect(report3.statusCode).toBe(200)
|
||||
|
||||
const listAfterFlush = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/v1/agents/${agentId}/blocked-ips`,
|
||||
})
|
||||
const bodyFlush = listAfterFlush.json() as {
|
||||
items: { ip: string; packets: number }[]
|
||||
}
|
||||
// First epoch 18+5 plus second epoch 4+3
|
||||
expect(bodyFlush.items.find((i) => i.ip === '203.0.113.10')?.packets).toBe(
|
||||
22,
|
||||
)
|
||||
expect(
|
||||
bodyFlush.items.find((i) => i.ip === '198.51.100.0/24')?.packets,
|
||||
).toBe(8)
|
||||
|
||||
const agentAfter = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/v1/agents/${agentId}`,
|
||||
})
|
||||
const agentBody = agentAfter.json() as {
|
||||
total_packets_dropped?: number
|
||||
}
|
||||
// Traffic: 25 + 0 + 7 = 32
|
||||
expect(agentBody.total_packets_dropped).toBe(32)
|
||||
|
||||
const reset = await app.inject({
|
||||
method: 'POST',
|
||||
url: `/api/v1/agents/${agentId}/stats/reset`,
|
||||
@@ -186,7 +253,7 @@ describe('apply-report ip_hits / blocked-ips', () => {
|
||||
expect(report.statusCode).toBeGreaterThanOrEqual(400)
|
||||
})
|
||||
|
||||
it('mikrotik presence mode increments packets and refreshes last_seen', async () => {
|
||||
it('mikrotik presence: continuous sync refreshes last_seen without +1; rehit after stale gap', async () => {
|
||||
const app = await appPromise
|
||||
await app.ready()
|
||||
|
||||
@@ -217,22 +284,27 @@ describe('apply-report ip_hits / blocked-ips', () => {
|
||||
url: `/api/v1/agents/${link.agent_id}/approve`,
|
||||
})
|
||||
|
||||
const report1 = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: {
|
||||
status: 'ok',
|
||||
packets_dropped: 3,
|
||||
kernel_method: 'address-list',
|
||||
source: 'mikrotik',
|
||||
ip_hits: [{ ip: '203.0.113.50', packets: 1 }],
|
||||
},
|
||||
const reportPayload = (dropped: number) => ({
|
||||
status: 'ok',
|
||||
packets_dropped: dropped,
|
||||
kernel_method: 'address-list',
|
||||
source: 'mikrotik',
|
||||
ip_hits: [{ ip: '203.0.113.50', packets: 1 }],
|
||||
})
|
||||
expect(report1.statusCode).toBe(200)
|
||||
|
||||
expect(
|
||||
(
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: reportPayload(3),
|
||||
})
|
||||
).statusCode,
|
||||
).toBe(200)
|
||||
|
||||
const list1 = await app.inject({
|
||||
method: 'GET',
|
||||
@@ -247,22 +319,19 @@ describe('apply-report ip_hits / blocked-ips', () => {
|
||||
|
||||
await new Promise((r) => setTimeout(r, 5))
|
||||
|
||||
const report2 = await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: {
|
||||
status: 'ok',
|
||||
packets_dropped: 5,
|
||||
kernel_method: 'address-list',
|
||||
source: 'mikrotik',
|
||||
ip_hits: [{ ip: '203.0.113.50', packets: 1 }],
|
||||
},
|
||||
})
|
||||
expect(report2.statusCode).toBe(200)
|
||||
expect(
|
||||
(
|
||||
await app.inject({
|
||||
method: 'POST',
|
||||
url: '/v1/agent/apply-report',
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
'content-type': 'application/json',
|
||||
},
|
||||
payload: reportPayload(5),
|
||||
})
|
||||
).statusCode,
|
||||
).toBe(200)
|
||||
|
||||
const list2 = await app.inject({
|
||||
method: 'GET',
|
||||
@@ -271,7 +340,25 @@ describe('apply-report ip_hits / blocked-ips', () => {
|
||||
const body2 = list2.json() as {
|
||||
items: { ip: string; packets: number; last_seen_at: string }[]
|
||||
}
|
||||
expect(body2.items[0]?.packets).toBe(2)
|
||||
expect(body2.items[0]?.packets).toBe(1)
|
||||
expect(body2.items[0]!.last_seen_at >= firstSeen).toBe(true)
|
||||
|
||||
// Re-entry after stale gap (> PRESENCE_REHIT_STALE_MS): simulate via future `now`
|
||||
const { repos } = await import('@evofw/db')
|
||||
repos.upsertIpBlockStats(
|
||||
app.db,
|
||||
link.agent_id,
|
||||
[{ ip: '203.0.113.50', packets: 1 }],
|
||||
new Date(Date.now() + 200_000).toISOString(),
|
||||
{ mode: 'presence' },
|
||||
)
|
||||
|
||||
const list3 = await app.inject({
|
||||
method: 'GET',
|
||||
url: `/api/v1/agents/${link.agent_id}/blocked-ips`,
|
||||
})
|
||||
expect(
|
||||
(list3.json() as { items: { packets: number }[] }).items[0]?.packets,
|
||||
).toBe(2)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -58,9 +58,9 @@ export function AgentBlockedIps({ agentId, platform }: AgentBlockedIpsProps) {
|
||||
const isMikrotik = platform === 'mikrotik'
|
||||
const q = useQuery(agentBlockedIpsQueryOptions(agentId))
|
||||
|
||||
const packetsTitle = isMikrotik ? 'Sync windows' : 'Packets'
|
||||
const packetsTitle = isMikrotik ? 'Hits' : 'Packets'
|
||||
const description = isMikrotik
|
||||
? 'Src /32 из EVOFW_HITS (add-src при deny, timeout 1h). Sync windows — сколько минут IP был в hits.'
|
||||
? 'Src /32 из EVOFW_HITS (add-src при deny, timeout 1h). Hits — входы в список (не каждый sync); Last seen обновляется, пока IP в hits.'
|
||||
: 'Drop-пакеты по записям deny (nft/ipset). Top по накопленным packets.'
|
||||
|
||||
const columns = useMemo<ColumnDef<BlockedIpRow>[]>(
|
||||
|
||||
+5
-4
@@ -61,8 +61,9 @@ Linux agent reports optional `ip_hits` in `POST /v1/agent/apply-report`:
|
||||
- Upgrade path: on install-link re-run, `last_hash` is cleared once so sets can be recreated (chain deleted before set replace).
|
||||
- **ipset:** prefers `hash:net … counters` on create; existing sets without counters are left as-is.
|
||||
- Payload: only entries with `packets > 0`, **top 200** by packets.
|
||||
- Control plane: `agent_ip_block_stats`, `GET /api/v1/agents/:id/blocked-ips`, reset via `POST …/stats/reset`.
|
||||
- UI: agent detail → **Blocked IPs**.
|
||||
- Control plane: `agent_ip_block_stats`, accumulates **deltas** of absolute kernel counters (как Traffic ↓). После flush set/chain (policy apply) CP сбрасывает per-IP baseline (`last_reported`), иначе вторая эпоха счётчиков теряется (Traffic растёт, Blocked IPs — нет).
|
||||
- `GET /api/v1/agents/:id/blocked-ips`, reset via `POST …/stats/reset`.
|
||||
- UI: agent detail → **Blocked IPs**. Sum of Blocked IPs ≈ Traffic ↓ для deny (при default accept); default-drop / allow в Traffic считаются отдельно.
|
||||
|
||||
IPv6 skipped.
|
||||
|
||||
@@ -92,9 +93,9 @@ Traffic ↓/↑ в UI — сумма `packets` с `evofw-deny-drop-*` / `evofw-a
|
||||
1. `evofw-deny-hit-input/forward` — `add-src-to-address-list` → `EVOFW_HITS`, `address-list-timeout=1h` (passthrough).
|
||||
2. `evofw-deny-drop-input/forward` — `drop` по `EVOFW_DENY`.
|
||||
|
||||
В `EVOFW_HITS` попадают реальные src **/32**. Policy rebuild **не** чистит HITS (только DENY/ALLOW). Sync шлёт top-200 в `ip_hits`; CP mode **presence**: `last_seen` каждый report, `packets` = число sync-окон (~минут), пока IP в HITS.
|
||||
В `EVOFW_HITS` попадают реальные src **/32**. Policy rebuild **не** чистит HITS (только DENY/ALLOW). Sync шлёт top-200 в `ip_hits`; CP mode **presence**: `last_seen` обновляется, пока IP в HITS; `packets` (Hits) увеличивается только при первом появлении или **повторном входе** после исчезновения из списка (~>2.5 мин без report), а не на каждый sync.
|
||||
|
||||
UI: agent detail → **Blocked IPs** (колонка Sync windows).
|
||||
UI: agent detail → **Blocked IPs** (колонка Hits).
|
||||
|
||||
**Default action** задаётся на **агенте** (`default_action: accept | drop`):
|
||||
|
||||
|
||||
@@ -60,7 +60,10 @@ export {
|
||||
upsertIpBlockStats,
|
||||
listIpBlockStats,
|
||||
deleteIpBlockStatsForAgent,
|
||||
resetIpBlockStatsBaselines,
|
||||
PRESENCE_REHIT_STALE_MS,
|
||||
} from './stats.js'
|
||||
export type { UpsertIpBlockStatsOptions, IpHitInput } from './stats.js'
|
||||
|
||||
export {
|
||||
getSetting,
|
||||
@@ -138,6 +141,7 @@ import {
|
||||
upsertIpBlockStats,
|
||||
listIpBlockStats,
|
||||
deleteIpBlockStatsForAgent,
|
||||
resetIpBlockStatsBaselines,
|
||||
} from './stats.js'
|
||||
import {
|
||||
getSetting,
|
||||
@@ -207,6 +211,7 @@ export const repos = {
|
||||
upsertIpBlockStats,
|
||||
listIpBlockStats,
|
||||
deleteIpBlockStatsForAgent,
|
||||
resetIpBlockStatsBaselines,
|
||||
getSetting,
|
||||
setSetting,
|
||||
listSettings,
|
||||
|
||||
@@ -36,19 +36,24 @@ export function deleteStatsSamplesForAgent(db: Db, agentId: string) {
|
||||
|
||||
export type IpHitInput = { ip: string; packets: number }
|
||||
|
||||
/** Gap after which a new presence report counts as a re-hit (left EVOFW_HITS). */
|
||||
export const PRESENCE_REHIT_STALE_MS = 150_000
|
||||
|
||||
export type UpsertIpBlockStatsOptions = {
|
||||
/**
|
||||
* MikroTik EVOFW_HITS presence: always refresh last_seen;
|
||||
* packets += 1 per report (sync-window sightings).
|
||||
* MikroTik EVOFW_HITS presence: refresh last_seen every report while IP is listed;
|
||||
* increment packets only on first see or re-hit after stale gap (left the list).
|
||||
* Linux keeps absolute counter deltas (default).
|
||||
*/
|
||||
mode?: 'absolute' | 'presence'
|
||||
/** Override re-hit gap (tests). Default PRESENCE_REHIT_STALE_MS. */
|
||||
presenceStaleMs?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Upsert per-IP drop counters.
|
||||
* - absolute (Linux): agent reports kernel counters; CP accumulates deltas.
|
||||
* - presence (MikroTik): each report sighting → last_seen=now, packets+=1.
|
||||
* - presence (MikroTik): last_seen while in HITS; packets = entries into HITS (not per-sync).
|
||||
*/
|
||||
export function upsertIpBlockStats(
|
||||
db: Db,
|
||||
@@ -58,6 +63,9 @@ export function upsertIpBlockStats(
|
||||
opts: UpsertIpBlockStatsOptions = {},
|
||||
) {
|
||||
const mode = opts.mode ?? 'absolute'
|
||||
const staleMs = opts.presenceStaleMs ?? PRESENCE_REHIT_STALE_MS
|
||||
const nowMs = Date.parse(now)
|
||||
|
||||
for (const hit of hits) {
|
||||
const ip = hit.ip.trim()
|
||||
if (!ip) continue
|
||||
@@ -88,11 +96,18 @@ export function upsertIpBlockStats(
|
||||
.run()
|
||||
continue
|
||||
}
|
||||
const prevSeen = Date.parse(existing.lastSeenAt)
|
||||
const gapMs = Number.isFinite(prevSeen) ? nowMs - prevSeen : staleMs + 1
|
||||
const isRehit = gapMs > staleMs
|
||||
db.update(agentIpBlockStats)
|
||||
.set({
|
||||
packets: (existing.packets ?? 0) + 1,
|
||||
lastReportedPackets: (existing.lastReportedPackets ?? 0) + 1,
|
||||
lastSeenAt: now,
|
||||
...(isRehit
|
||||
? {
|
||||
packets: (existing.packets ?? 0) + 1,
|
||||
lastReportedPackets: (existing.lastReportedPackets ?? 0) + 1,
|
||||
}
|
||||
: {}),
|
||||
})
|
||||
.where(eq(agentIpBlockStats.id, existing.id))
|
||||
.run()
|
||||
@@ -144,3 +159,16 @@ export function deleteIpBlockStatsForAgent(db: Db, agentId: string) {
|
||||
.where(eq(agentIpBlockStats.agentId, agentId))
|
||||
.run()
|
||||
}
|
||||
|
||||
/**
|
||||
* After nft/ipset counter flush (policy apply), kernel absolutes restart at 0.
|
||||
* Traffic detects this via packets_dropped drop; per-IP baselines must reset too,
|
||||
* otherwise ip_hits with packets=0 are omitted from the report and lastReported
|
||||
* stays high → second epoch deltas are lost (Traffic 14 vs Blocked IPs 7).
|
||||
*/
|
||||
export function resetIpBlockStatsBaselines(db: Db, agentId: string) {
|
||||
db.update(agentIpBlockStats)
|
||||
.set({ lastReportedPackets: 0 })
|
||||
.where(eq(agentIpBlockStats.agentId, agentId))
|
||||
.run()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user