fix(network-map): убрать тяжёлую классификацию из poll карты

Не сканировать RIPE и каталог по каждой строке окна; сервисы по уникальным dst и кэшу ASN. Порог доли на карте настраиваемый и отключаемый.

Co-authored-by: Cursor <[email protected]>

backend/src/services/traffic-flow-map-hops.test.ts

@@ -4,7 +4,7 @@ import {
   ingestParsedFlowsForServerForTests,
   resetFlowRingsForTests,
 } from "./traffic-flow-ingest.js"
-import { buildFlowMapHops } from "./traffic-flow-map-hops.js"
+import { buildFlowMapHops, resetFlowMapHopsCacheForTests } from "./traffic-flow-map-hops.js"
 import { seedFlowTopologyForTests, type FlowTopology } from "./traffic-flow-topology.js"
 import { disableCatalogFetchForTests, resetFlowCatalogForTests } from "./traffic-flow-classify.js"
 import {
@@ -114,6 +114,7 @@ ingestParsedFlowsForServerForTests(3, [
 ])
 
 try {
+  resetFlowMapHopsCacheForTests()
   const def = buildFlowMapHops({ minutes: 5 })
   assert.equal(def.excludeOverlayApplied, true)
   assert.equal(def.excludeMeshApplied, true)
@@ -142,6 +143,7 @@ try {
   assert.ok(wan, "WAN hop from home-router")
   assert.equal(wan.bytes, 3000)
 
+  resetFlowMapHopsCacheForTests()
   const withAll = buildFlowMapHops({ minutes: 5, excludeOverlay: false, excludeMesh: false })
   const overlayIface = withAll.hops.find((h) => h.iface === "gre-jh-en" && h.fromId === "7")
   assert.ok(overlayIface && overlayIface.bytes >= 5_000_000)
@@ -200,7 +202,8 @@ ingestParsedFlowsForServerForTests(7, [
   payloadFlow("203.0.113.50", 9400),
 ])
 try {
-  const six = buildFlowMapHops({ minutes: 5 })
+  resetFlowMapHopsCacheForTests()
+  const six = buildFlowMapHops({ minutes: 5, minSharePct: 5 })
   assert.equal(six.totalBytes, 10_000)
   const google = six.services?.find((s) => s.id === "svc:google")
   assert.ok(google, "Google ≥ 5%")
@@ -227,9 +230,13 @@ ingestParsedFlowsForServerForTests(7, [
   payloadFlow("203.0.113.50", 9600),
 ])
 try {
-  const four = buildFlowMapHops({ minutes: 5 })
+  resetFlowMapHopsCacheForTests()
+  const four = buildFlowMapHops({ minutes: 5, minSharePct: 5 })
   assert.equal(four.totalBytes, 10_000)
   assert.ok(!(four.services ?? []).some((s) => s.id === "svc:google"), "Google < 5% hidden")
+  resetFlowMapHopsCacheForTests()
+  const off = buildFlowMapHops({ minutes: 5, minSharePct: 0 })
+  assert.ok(off.services?.some((s) => s.id === "svc:google"), "порог 0 показывает Google 4%")
 } finally {
   resetFlowRingsForTests()
   resetIfaceCacheForTests()
@@ -260,7 +267,8 @@ ingestParsedFlowsForServerForTests(7, [
   payloadFlow("203.0.113.50", 1000),
 ])
 try {
-  const greOnly = buildFlowMapHops({ minutes: 5, excludeOverlay: false })
+  resetFlowMapHopsCacheForTests()
+  const greOnly = buildFlowMapHops({ minutes: 5, excludeOverlay: false, minSharePct: 0 })
   assert.ok(!(greOnly.services ?? []).some((s) => s.label === "GRE"), "GRE is not a destination service")
 } finally {
   seedFlowTopologyForTests(null)

backend/src/services/traffic-flow-map-hops.ts

@@ -2,20 +2,23 @@ import { eq } from "drizzle-orm"
 import type { FlowMapHop, FlowMapHopsDto, FlowMapService, FlowMapServiceEdge } from "@mmapp/contracts/traffic-flow"
 import { db } from "../db/index.js"
 import { servers, userInterfaceBindings } from "../db/schema.js"
-import { flowRowMatchesFilter } from "./traffic-flow-apps.js"
+import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js"
 import {
   isNamedInternetService,
+  lookupBrand,
   mapServiceNodeId,
 } from "./traffic-flow-brands.js"
-import { classifyFlowDst, refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
 import { dedupFlowRowsMaxBytes } from "./traffic-flow-dedup.js"
 import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-ingest.js"
 import { resolveIfaceName } from "./traffic-flow-ifaces.js"
 import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js"
-import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js"
+import { lookupRipeCached, type FlowIpMeta } from "./traffic-flow-ripe.js"
+import { getTrafficFlowSettingsRow } from "./traffic-flow-settings.js"
 import { loadFlowTopology, resolveEn } from "./traffic-flow-topology.js"
 
-export const MAP_SERVICE_SHARE_THRESHOLD = 0.05
+export const DEFAULT_MAP_SERVICE_MIN_SHARE_PCT = 5
+export const MAP_SERVICE_NODE_CAP = 20
+const HOPS_CACHE_TTL_MS = 2000
 
 export interface FlowMapHopsQuery {
   minutes: number
@@ -25,6 +28,8 @@ export interface FlowMapHopsQuery {
   dedup?: boolean
   excludeMesh?: boolean
   excludeOverlay?: boolean
+  /** Переопределение порога (тесты). Иначе из настроек NetFlow. */
+  minSharePct?: number
 }
 
 interface HopAcc {
@@ -39,6 +44,39 @@ interface HopAcc {
   bytesRev: number
 }
 
+interface DstAcc {
+  bytes: number
+  proto: number
+  dstPort: number
+  srcPort: number
+  fromBytes: Map<string, number>
+}
+
+let hopsCache: { key: string; at: number; dto: FlowMapHopsDto } | null = null
+
+export function resetFlowMapHopsCacheForTests(): void {
+  hopsCache = null
+}
+
+export function clampMapServiceMinSharePct(n: unknown): number {
+  const v = typeof n === "number" ? n : Number(n)
+  if (!Number.isFinite(v)) return DEFAULT_MAP_SERVICE_MIN_SHARE_PCT
+  return Math.min(100, Math.max(0, v))
+}
+
+function hopsQueryKey(q: FlowMapHopsQuery, minSharePct: number): string {
+  return JSON.stringify({
+    minutes: q.minutes,
+    serverId: q.serverId ?? null,
+    userId: q.userId ?? null,
+    iface: q.iface ?? null,
+    dedup: q.dedup !== false,
+    excludeMesh: q.excludeMesh !== false,
+    excludeOverlay: q.excludeOverlay !== false,
+    minSharePct,
+  })
+}
+
 function userIfaceAllow(userId: string): Map<number, Set<string>> | null {
   if (!userId) return null
   const binds = db.select().from(userInterfaceBindings).where(eq(userInterfaceBindings.userId, userId)).all()
@@ -89,8 +127,36 @@ function toHop(a: HopAcc, windowSec: number): FlowMapHop {
   }
 }
 
-/** Hop-rates для карты сети: те же фильтры, что у общего NetFlow (dedup / mesh / overlay). */
-export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
+/** Имя бренда без каталога EvoBGP — только ASN/CIDR кэш + proto. */
+function classifyMapDstLite(
+  dst: string,
+  proto: number,
+  dstPort: number,
+  srcPort: number,
+  ripe: FlowIpMeta | null,
+): { service: string; category: string } | null {
+  if (proto === 47 || proto === 50) return null
+  const app = applicationName(proto, dstPort, srcPort)
+  if (app === "WireGuard" || app === "DNS" || app === "SSH" || app === "BGP") return null
+  if (/youtube/i.test(ripe?.holder ?? "")) {
+    return { service: "YouTube", category: "Видео / стриминг" }
+  }
+  const brand = lookupBrand(dst, ripe?.asn ?? 0)
+  if (!brand || !isNamedInternetService(brand.service, brand.category)) return null
+  return brand
+}
+
+function resolveMinSharePct(q: FlowMapHopsQuery): number {
+  if (q.minSharePct != null) return clampMapServiceMinSharePct(q.minSharePct)
+  try {
+    const row = getTrafficFlowSettingsRow() as { mapServiceMinSharePct?: number }
+    return clampMapServiceMinSharePct(row.mapServiceMinSharePct ?? DEFAULT_MAP_SERVICE_MIN_SHARE_PCT)
+  } catch {
+    return DEFAULT_MAP_SERVICE_MIN_SHARE_PCT
+  }
+}
+
+function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number): FlowMapHopsDto {
   const windowSec = Math.max(60, q.minutes * 60)
   const raw = listFlowRowsForWindow(q.minutes)
   const allow = q.userId ? userIfaceAllow(q.userId) : null
@@ -122,13 +188,9 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
 
   const working = wantDedup ? dedupFlowRowsMaxBytes(matched) : matched
   const hops = new Map<string, HopAcc>()
-  const svcTotals = new Map<string, { label: string; category: string; bytes: number }>()
-  const svcEdges = new Map<string, { fromId: string; toId: string; bytes: number; bytesFwd: number; bytesRev: number }>()
-  const dsts = new Set<string>()
+  const dstAcc = new Map<string, DstAcc>()
   let totalBytes = 0
 
-  refreshFlowCatalogInBackground()
-
   for (const r of working) {
     const inRes = resolveIfaceName(r.serverId, r.inIface)
     const outRes = resolveIfaceName(r.serverId, r.outIface)
@@ -215,37 +277,53 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
     }
 
     totalBytes += r.bytes
-    dsts.add(r.dst)
-    const ripe = lookupRipeCached(r.dst)
-    const classified = classifyFlowDst(r.dst, r.proto, r.dstPort, r.srcPort, ripe)
-    if (isNamedInternetService(classified.service, classified.category)) {
-      const toId = mapServiceNodeId(classified.service)
-      const prevSvc = svcTotals.get(toId)
-      if (prevSvc) prevSvc.bytes += r.bytes
-      else svcTotals.set(toId, { label: classified.service, category: classified.category, bytes: r.bytes })
+    const svcFromId = String((enOut ?? enIn)?.id ?? r.serverId)
+    const prevDst = dstAcc.get(r.dst)
+    if (prevDst) {
+      prevDst.bytes += r.bytes
+      prevDst.fromBytes.set(svcFromId, (prevDst.fromBytes.get(svcFromId) ?? 0) + r.bytes)
+    } else {
+      dstAcc.set(r.dst, {
+        bytes: r.bytes,
+        proto: r.proto,
+        dstPort: r.dstPort,
+        srcPort: r.srcPort,
+        fromBytes: new Map([[svcFromId, r.bytes]]),
+      })
+    }
+  }
 
-      const svcEn = enOut ?? enIn
-      const svcFromId = svcEn ? String(svcEn.id) : fromId
-      const edgeKey = `${svcFromId}|${toId}`
+  const svcTotals = new Map<string, { label: string; category: string; bytes: number }>()
+  const svcEdges = new Map<string, { fromId: string; toId: string; bytes: number; bytesFwd: number; bytesRev: number }>()
+
+  for (const [dst, acc] of dstAcc) {
+    const ripe = lookupRipeCached(dst)
+    const classified = classifyMapDstLite(dst, acc.proto, acc.dstPort, acc.srcPort, ripe)
+    if (!classified) continue
+    const toId = mapServiceNodeId(classified.service)
+    const prevSvc = svcTotals.get(toId)
+    if (prevSvc) prevSvc.bytes += acc.bytes
+    else svcTotals.set(toId, { label: classified.service, category: classified.category, bytes: acc.bytes })
+    for (const [fromId, bytes] of acc.fromBytes) {
+      const edgeKey = `${fromId}|${toId}`
       const prevEdge = svcEdges.get(edgeKey)
       if (prevEdge) {
-        prevEdge.bytes += r.bytes
-        prevEdge.bytesFwd += r.bytes
+        prevEdge.bytes += bytes
+        prevEdge.bytesFwd += bytes
       } else {
         svcEdges.set(edgeKey, {
-          fromId: svcFromId,
+          fromId,
           toId,
-          bytes: r.bytes,
-          bytesFwd: r.bytes,
+          bytes,
+          bytesFwd: bytes,
           bytesRev: 0,
         })
       }
     }
   }
 
-  enqueueRipeMisses(dsts)
-
-  const services: FlowMapService[] = [...svcTotals.entries()]
+  const minShare = minSharePct / 100
+  let services: FlowMapService[] = [...svcTotals.entries()]
     .map(([id, s]) => ({
       id,
       label: s.label,
@@ -254,8 +332,11 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
       bps: (s.bytes * 8) / windowSec,
       share: totalBytes > 0 ? s.bytes / totalBytes : 0,
     }))
-    .filter((s) => s.share >= MAP_SERVICE_SHARE_THRESHOLD)
     .sort((a, b) => b.bytes - a.bytes)
+  if (minSharePct > 0) {
+    services = services.filter((s) => s.share >= minShare)
+  }
+  services = services.slice(0, MAP_SERVICE_NODE_CAP)
   const keepSvc = new Set(services.map((s) => s.id))
   const serviceEdges: FlowMapServiceEdge[] = [...svcEdges.values()]
     .filter((e) => keepSvc.has(e.toId))
@@ -273,15 +354,29 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
   return {
     hops: [...hops.values()]
       .map((a) => toHop(a, windowSec))
-      .sort((a, b) => b.bytes - a.bytes),
+      .sort((a, b) => a.bytes === b.bytes ? 0 : b.bytes - a.bytes),
     live: listener.bound,
     rangeMinutes: q.minutes,
     windowSec,
     totalBytes,
     services,
     serviceEdges,
+    mapServiceMinSharePct: minSharePct,
     dedupApplied: wantDedup,
     excludeMeshApplied: excludeMesh,
     excludeOverlayApplied: excludeOverlay,
   }
 }
+
+/** Hop-rates для карты сети: те же фильтры, что у общего NetFlow (dedup / mesh / overlay). */
+export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
+  const minSharePct = resolveMinSharePct(q)
+  const key = hopsQueryKey(q, minSharePct)
+  const now = Date.now()
+  if (hopsCache && hopsCache.key === key && now - hopsCache.at < HOPS_CACHE_TTL_MS) {
+    return hopsCache.dto
+  }
+  const dto = buildFlowMapHopsUncached(q, minSharePct)
+  hopsCache = { key, at: now, dto }
+  return dto
+}

backend/src/services/traffic-flow-ripe.test.ts

@@ -7,6 +7,7 @@ import {
   lookupRipeCached,
   resetRipeCacheForTests,
   ripeFetchCountForTests,
+  ripeLastCandidateCountForTests,
   seedRipeCacheForTests,
   setRipeFetchForTests,
 } from "./traffic-flow-ripe.js"
@@ -100,4 +101,36 @@ await flushRipeQueueForTests()
 assert.equal(lookupRipeCached("1.0.0.1")?.country, "US")
 assert.equal(lookupRipeCached("1.0.0.1")?.asn, 13335)
 
+resetRipeCacheForTests()
+disableRipePersistForTests()
+for (let i = 0; i < 3000; i++) {
+  const o2 = Math.floor(i / 256)
+  const o3 = i % 256
+  seedRipeCacheForTests({
+    prefix: `203.${o2}.${o3}.0/24`,
+    asn: 64500,
+    country: "NL",
+    lat: null,
+    lng: null,
+    holder: "NOISE",
+    ok: true,
+    fetchedAt: Date.now(),
+  })
+}
+seedRipeCacheForTests({
+  prefix: "8.8.8.0/24",
+  asn: 15169,
+  country: "US",
+  lat: null,
+  lng: null,
+  holder: "GOOGLE",
+  ok: true,
+  fetchedAt: Date.now(),
+})
+assert.equal(lookupRipeCached("8.8.8.8")?.asn, 15169)
+assert.ok(
+  ripeLastCandidateCountForTests() < 8,
+  `index should not scan all prefixes, got ${ripeLastCandidateCountForTests()}`,
+)
+
 console.log("traffic-flow-ripe.test.ts: ok")

backend/src/services/traffic-flow-ripe.ts

@@ -1,5 +1,5 @@
 import { sqliteDatabase } from "../db/index.js"
-import { ipInCidrV4, ipv4ToInt, isNonPublicIp, parseCidrV4 } from "./traffic-flow-ip.js"
+import { ipv4ToInt, isNonPublicIp, parseCidrV4 } from "./traffic-flow-ip.js"
 import { resolveRipeCountry } from "./traffic-flow-brands.js"
 
 export interface FlowIpMeta {
@@ -28,6 +28,18 @@ const queue: string[] = []
 const queued = new Set<string>()
 const recentFetches: number[] = []
 
+interface RipeIndexed {
+  entry: FlowIpMeta
+  net: number
+  mask: number
+  prefixLen: number
+}
+
+/** /24 → кандидаты с prefixLen ≥ 24. Более широкие префиксы — в `wideIndex`. */
+const v24Index = new Map<number, RipeIndexed[]>()
+const wideIndex: RipeIndexed[] = []
+let lastCandidateCount = 0
+
 let persistEnabled = true
 let enqueueEnabled = true
 let loaded = false
@@ -50,6 +62,9 @@ export function resetRipeCacheForTests(): void {
   queue.length = 0
   queued.clear()
   recentFetches.length = 0
+  v24Index.clear()
+  wideIndex.length = 0
+  lastCandidateCount = 0
   loaded = persistEnabled ? false : true
   workerRunning = false
   fetchCount = 0
@@ -58,10 +73,15 @@ export function resetRipeCacheForTests(): void {
 }
 
 export function seedRipeCacheForTests(entry: FlowIpMeta): void {
-  mem.set(entry.prefix, { ...entry })
+  remember(entry)
   loaded = true
 }
 
+/** Сколько CIDR смотрели в последнем lookup (для теста индекса /24). */
+export function ripeLastCandidateCountForTests(): number {
+  return lastCandidateCount
+}
+
 export function setRipeFetchForTests(fn: typeof fetch): void {
   fetchImpl = fn
   fetchCount = 0
@@ -87,6 +107,48 @@ function isFresh(entry: FlowIpMeta): boolean {
   return Date.now() - entry.fetchedAt < ttlMs(entry.ok)
 }
 
+function unindexPrefix(prefix: string): void {
+  const parsed = parseCidrV4(prefix)
+  if (!parsed) return
+  if (parsed.prefixLen >= 24) {
+    const key = parsed.net >>> 8
+    const list = v24Index.get(key)
+    if (!list) return
+    const next = list.filter((row) => row.entry.prefix !== prefix)
+    if (next.length) v24Index.set(key, next)
+    else v24Index.delete(key)
+    return
+  }
+  const idx = wideIndex.findIndex((row) => row.entry.prefix === prefix)
+  if (idx >= 0) wideIndex.splice(idx, 1)
+}
+
+function indexEntry(entry: FlowIpMeta): void {
+  const parsed = parseCidrV4(entry.prefix)
+  if (!parsed) return
+  const row: RipeIndexed = {
+    entry,
+    net: parsed.net,
+    mask: parsed.mask,
+    prefixLen: parsed.prefixLen,
+  }
+  if (parsed.prefixLen >= 24) {
+    const key = parsed.net >>> 8
+    const list = v24Index.get(key)
+    if (list) list.push(row)
+    else v24Index.set(key, [row])
+    return
+  }
+  wideIndex.push(row)
+}
+
+function remember(entry: FlowIpMeta): void {
+  const prev = mem.get(entry.prefix)
+  if (prev) unindexPrefix(prev.prefix)
+  mem.set(entry.prefix, entry)
+  indexEntry(entry)
+}
+
 function loadSqlite(): void {
   if (loaded || !persistEnabled) {
     loaded = true
@@ -111,7 +173,7 @@ function loadSqlite(): void {
       const fetchedAt = Date.parse(r.fetched_at)
       const asn = Number(r.asn ?? 0) || 0
       const holder = r.holder || ""
-      mem.set(r.prefix, {
+      remember({
         prefix: r.prefix,
         asn,
         country: resolveRipeCountry(r.country || "", asn, holder) || "—",
@@ -193,20 +255,24 @@ function negative(prefix: string): FlowIpMeta {
 export function lookupRipeCached(ip: string): FlowIpMeta | null {
   loadSqlite()
   const trimmed = String(ip ?? "").trim()
+  lastCandidateCount = 0
   if (!trimmed) return null
   if (isNonPublicIp(trimmed)) {
     return negative(`${trimmed.includes(":") ? trimmed : trimmed}/32`)
   }
+  const addr = ipv4ToInt(trimmed)
+  if (addr == null) return null
+  const bucket = v24Index.get(addr >>> 8)
+  const candidates = bucket ? bucket.concat(wideIndex) : wideIndex
+  lastCandidateCount = candidates.length
   let best: FlowIpMeta | null = null
   let bestLen = -1
-  for (const entry of mem.values()) {
-    if (!isFresh(entry)) continue
-    const parsed = parseCidrV4(entry.prefix)
-    if (!parsed) continue
-    if (!ipInCidrV4(trimmed, entry.prefix)) continue
-    if (parsed.prefixLen > bestLen) {
-      best = entry
-      bestLen = parsed.prefixLen
+  for (const row of candidates) {
+    if (!isFresh(row.entry)) continue
+    if (((addr & row.mask) >>> 0) !== row.net) continue
+    if (row.prefixLen > bestLen) {
+      best = row.entry
+      bestLen = row.prefixLen
     }
   }
   return best
@@ -316,13 +382,13 @@ async function resolveIp(ip: string): Promise<FlowIpMeta | null> {
         ok: Boolean(asn || country),
         fetchedAt: Date.now(),
       }
-      mem.set(prefix, entry)
+      remember(entry)
       persist(entry)
       return entry
     } catch {
       const prefix = `${ip}/32`
       const entry = negative(prefix)
-      mem.set(prefix, entry)
+      remember(entry)
       persist(entry)
       return entry
     } finally {

backend/src/services/traffic-flow-settings.ts

@@ -53,6 +53,7 @@ export function toTrafficFlowSettingsDto(
     hubServerId: row.hubServerId ?? null,
     retentionHours: row.retentionHours,
     topN: row.topN,
+    mapServiceMinSharePct: Number(row.mapServiceMinSharePct ?? 5),
     lastDatagramAt: row.lastDatagramAt ?? null,
     lastExporterIp: row.lastExporterIp ?? null,
     lastError: row.lastError || null,
@@ -75,6 +76,9 @@ export function updateTrafficFlowSettings(patch: TrafficFlowSettingsPatch) {
     hubServerId: patch.hubServerId === undefined ? row.hubServerId : patch.hubServerId,
     retentionHours: patch.retentionHours ?? row.retentionHours,
     topN: patch.topN ?? row.topN,
+    mapServiceMinSharePct: patch.mapServiceMinSharePct == null
+      ? row.mapServiceMinSharePct
+      : Math.min(100, Math.max(0, patch.mapServiceMinSharePct)),
     updatedAt: nowIso(),
   }).where(eq(trafficFlowSettings.id, 1)).run()
   return getTrafficFlowSettingsRow()