feat(network-map): показать конечные сервисы на карте сети

NetFlow ≥ 5% окна, узлы с логотипом бренда справа от EN, скорость потока на рёбрах к сервисам.

Co-authored-by: Cursor <[email protected]>

backend/src/services/traffic-flow-brands.test.ts

@@ -4,6 +4,8 @@ import {
   countryFromHolder,
   lookupBrand,
   OTHER_SERVICE,
+  isNamedInternetService,
+  mapServiceNodeId,
   resolveRipeCountry,
 } from "./traffic-flow-brands.js"
 
@@ -21,9 +23,17 @@ assert.equal(brandByAsn(15169)?.category, "Веб")
 assert.equal(lookupBrand("208.65.153.1", 0)?.service, "YouTube")
 assert.equal(brandByAsn(32590)?.service, "Steam")
 assert.equal(brandByAsn(32590)?.category, "Игры")
+assert.equal(brandByAsn(16509)?.service, "AWS")
+assert.equal(brandByAsn(57976)?.service, "Blizzard")
 assert.equal(brandByAsn(401115)?.service, "ChatGPT")
 assert.equal(lookupBrand("1.1.1.1", 13335)?.service, "Cloudflare")
 assert.equal(lookupBrand("203.0.113.9", 64500), null)
 assert.equal(OTHER_SERVICE, "Прочее")
+assert.equal(isNamedInternetService("Google", "Веб"), true)
+assert.equal(isNamedInternetService("Прочее", "Прочее"), false)
+assert.equal(isNamedInternetService("GRE", "Туннель"), false)
+assert.equal(isNamedInternetService("DNS", "DNS"), false)
+assert.equal(mapServiceNodeId("AWS"), "svc:aws")
+assert.equal(mapServiceNodeId("Cloudflare"), "svc:cloudflare")
 
 console.log("traffic-flow-brands.test.ts: ok")

backend/src/services/traffic-flow-brands.ts

@@ -12,11 +12,12 @@ const ASN_BRANDS = new Map<number, BrandHit>([
   [209242, { service: "Cloudflare", category: "CDN" }],
   [54113, { service: "Fastly", category: "CDN" }],
   [20940, { service: "Akamai", category: "CDN" }],
-  [16509, { service: "Amazon", category: "CDN" }],
-  [14618, { service: "Amazon", category: "CDN" }],
+  [16509, { service: "AWS", category: "CDN" }],
+  [14618, { service: "AWS", category: "CDN" }],
   [8075, { service: "Microsoft", category: "CDN" }],
   [13238, { service: "Yandex", category: "CDN" }],
   [32590, { service: "Steam", category: "Игры" }],
+  [57976, { service: "Blizzard", category: "Игры" }],
   [2906, { service: "Netflix", category: "Видео / стриминг" }],
   [40027, { service: "Netflix", category: "Видео / стриминг" }],
   [15169, { service: "Google", category: "Веб" }],
@@ -41,6 +42,7 @@ const ASN_HQ_COUNTRY = new Map<number, string>([
   [8075, "US"],
   [15169, "US"],
   [32590, "US"],
+  [57976, "US"],
   [2906, "US"],
   [40027, "US"],
   [36040, "US"],
@@ -105,3 +107,32 @@ export function brandByCidr(ip: string): BrandHit | null {
 export function lookupBrand(ip: string, asn: number): BrandHit | null {
   return brandByCidr(ip) || brandByAsn(asn)
 }
+
+const SKIP_MAP_SERVICES = new Set([
+  OTHER_SERVICE,
+  "GRE",
+  "ESP",
+  "WireGuard",
+  "DNS",
+  "SSH",
+  "BGP",
+])
+
+const SKIP_MAP_CATEGORIES = new Set(["Туннель", "DNS", "SSH", "BGP"])
+
+/** Именованный интернет-сервис для карты (не туннель и не «Прочее»). */
+export function isNamedInternetService(service: string, category: string): boolean {
+  const s = service.trim()
+  const c = category.trim()
+  if (!s || SKIP_MAP_SERVICES.has(s) || SKIP_MAP_CATEGORIES.has(c)) return false
+  return true
+}
+
+export function mapServiceNodeId(label: string): string {
+  const slug = label
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+  return `svc:${slug || "unknown"}`
+}

backend/src/services/traffic-flow-map-hops.test.ts

@@ -11,6 +11,7 @@ import {
   disableRipeEnqueueForTests,
   disableRipePersistForTests,
   resetRipeCacheForTests,
+  seedRipeCacheForTests,
 } from "./traffic-flow-ripe.js"
 
 disableCatalogFetchForTests()
@@ -154,4 +155,119 @@ try {
   resetFlowCatalogForTests()
 }
 
+console.log("traffic-flow-map-hops.test.ts: hops ok")
+
+function googleRipe() {
+  seedRipeCacheForTests({
+    prefix: "8.8.8.0/24",
+    asn: 15169,
+    country: "US",
+    lat: 37.4,
+    lng: -122.1,
+    holder: "GOOGLE",
+    ok: true,
+    fetchedAt: Date.now(),
+  })
+}
+
+function payloadFlow(dst: string, bytes: number) {
+  return {
+    src: "10.100.1.17",
+    dst,
+    proto: 6,
+    srcPort: 51234,
+    dstPort: 443,
+    bytes,
+    packets: Math.max(1, Math.round(bytes / 1200)),
+    inIface: "2",
+    outIface: "3",
+    nextHop: "198.51.100.1",
+  }
+}
+
+resetFlowRingsForTests()
+resetIfaceCacheForTests()
+resetRipeCacheForTests()
+disableRipeEnqueueForTests()
+seedFlowTopologyForTests(topo)
+rememberServerIfaces(7, [
+  { ".id": "*2", name: "gre-client" },
+  { ".id": "*3", name: "gre-jh-en" },
+])
+googleRipe()
+ingestParsedFlowsForServerForTests(7, [
+  payloadFlow("8.8.8.8", 600),
+  payloadFlow("203.0.113.50", 9400),
+])
+try {
+  const six = buildFlowMapHops({ minutes: 5 })
+  assert.equal(six.totalBytes, 10_000)
+  const google = six.services?.find((s) => s.id === "svc:google")
+  assert.ok(google, "Google ≥ 5%")
+  assert.ok(google.share >= 0.05)
+  assert.ok(six.serviceEdges?.some((e) => e.toId === "svc:google" && e.fromId === "9"))
+} finally {
+  resetFlowRingsForTests()
+  resetIfaceCacheForTests()
+  resetRipeCacheForTests()
+}
+
+resetFlowRingsForTests()
+resetIfaceCacheForTests()
+resetRipeCacheForTests()
+disableRipeEnqueueForTests()
+seedFlowTopologyForTests(topo)
+rememberServerIfaces(7, [
+  { ".id": "*2", name: "gre-client" },
+  { ".id": "*3", name: "gre-jh-en" },
+])
+googleRipe()
+ingestParsedFlowsForServerForTests(7, [
+  payloadFlow("8.8.8.8", 400),
+  payloadFlow("203.0.113.50", 9600),
+])
+try {
+  const four = buildFlowMapHops({ minutes: 5 })
+  assert.equal(four.totalBytes, 10_000)
+  assert.ok(!(four.services ?? []).some((s) => s.id === "svc:google"), "Google < 5% hidden")
+} finally {
+  resetFlowRingsForTests()
+  resetIfaceCacheForTests()
+  resetRipeCacheForTests()
+}
+
+resetFlowRingsForTests()
+resetIfaceCacheForTests()
+resetRipeCacheForTests()
+disableRipeEnqueueForTests()
+seedFlowTopologyForTests(topo)
+rememberServerIfaces(7, [
+  { ".id": "*2", name: "gre-client" },
+  { ".id": "*3", name: "gre-jh-en" },
+])
+ingestParsedFlowsForServerForTests(7, [
+  {
+    src: "203.0.113.10",
+    dst: "198.51.100.1",
+    proto: 47,
+    srcPort: 0,
+    dstPort: 0,
+    bytes: 9_000,
+    packets: 90,
+    inIface: "3",
+    outIface: "3",
+  },
+  payloadFlow("203.0.113.50", 1000),
+])
+try {
+  const greOnly = buildFlowMapHops({ minutes: 5, excludeOverlay: false })
+  assert.ok(!(greOnly.services ?? []).some((s) => s.label === "GRE"), "GRE is not a destination service")
+} finally {
+  seedFlowTopologyForTests(null)
+  resetFlowRingsForTests()
+  resetIfaceCacheForTests()
+  resetRipeCacheForTests()
+  resetFlowCatalogForTests()
+}
+
 console.log("traffic-flow-map-hops.test.ts: ok")

backend/src/services/traffic-flow-map-hops.ts

@@ -1,14 +1,22 @@
 import { eq } from "drizzle-orm"
-import type { FlowMapHop, FlowMapHopsDto } from "@mmapp/contracts/traffic-flow"
+import type { FlowMapHop, FlowMapHopsDto, FlowMapService, FlowMapServiceEdge } from "@mmapp/contracts/traffic-flow"
 import { db } from "../db/index.js"
 import { servers, userInterfaceBindings } from "../db/schema.js"
 import { flowRowMatchesFilter } from "./traffic-flow-apps.js"
+import {
+  isNamedInternetService,
+  mapServiceNodeId,
+} from "./traffic-flow-brands.js"
+import { classifyFlowDst, refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
 import { dedupFlowRowsMaxBytes } from "./traffic-flow-dedup.js"
 import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-ingest.js"
 import { resolveIfaceName } from "./traffic-flow-ifaces.js"
 import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js"
+import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js"
 import { loadFlowTopology, resolveEn } from "./traffic-flow-topology.js"
 
+export const MAP_SERVICE_SHARE_THRESHOLD = 0.05
+
 export interface FlowMapHopsQuery {
   minutes: number
   serverId?: number
@@ -114,6 +122,12 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
 
   const working = wantDedup ? dedupFlowRowsMaxBytes(matched) : matched
   const hops = new Map<string, HopAcc>()
+  const svcTotals = new Map<string, { label: string; category: string; bytes: number }>()
+  const svcEdges = new Map<string, { fromId: string; toId: string; bytes: number; bytesFwd: number; bytesRev: number }>()
+  const dsts = new Set<string>()
+  let totalBytes = 0
+
+  refreshFlowCatalogInBackground()
 
   for (const r of working) {
     const inRes = resolveIfaceName(r.serverId, r.inIface)
@@ -199,8 +213,62 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
         }, r.bytes, "fwd")
       }
     }
+
+    totalBytes += r.bytes
+    dsts.add(r.dst)
+    const ripe = lookupRipeCached(r.dst)
+    const classified = classifyFlowDst(r.dst, r.proto, r.dstPort, r.srcPort, ripe)
+    if (isNamedInternetService(classified.service, classified.category)) {
+      const toId = mapServiceNodeId(classified.service)
+      const prevSvc = svcTotals.get(toId)
+      if (prevSvc) prevSvc.bytes += r.bytes
+      else svcTotals.set(toId, { label: classified.service, category: classified.category, bytes: r.bytes })
+
+      const svcEn = enOut ?? enIn
+      const svcFromId = svcEn ? String(svcEn.id) : fromId
+      const edgeKey = `${svcFromId}|${toId}`
+      const prevEdge = svcEdges.get(edgeKey)
+      if (prevEdge) {
+        prevEdge.bytes += r.bytes
+        prevEdge.bytesFwd += r.bytes
+      } else {
+        svcEdges.set(edgeKey, {
+          fromId: svcFromId,
+          toId,
+          bytes: r.bytes,
+          bytesFwd: r.bytes,
+          bytesRev: 0,
+        })
+      }
+    }
   }
 
+  enqueueRipeMisses(dsts)
+
+  const services: FlowMapService[] = [...svcTotals.entries()]
+    .map(([id, s]) => ({
+      id,
+      label: s.label,
+      category: s.category,
+      bytes: s.bytes,
+      bps: (s.bytes * 8) / windowSec,
+      share: totalBytes > 0 ? s.bytes / totalBytes : 0,
+    }))
+    .filter((s) => s.share >= MAP_SERVICE_SHARE_THRESHOLD)
+    .sort((a, b) => b.bytes - a.bytes)
+  const keepSvc = new Set(services.map((s) => s.id))
+  const serviceEdges: FlowMapServiceEdge[] = [...svcEdges.values()]
+    .filter((e) => keepSvc.has(e.toId))
+    .map((e) => ({
+      fromId: e.fromId,
+      toId: e.toId,
+      bytes: e.bytes,
+      bps: (e.bytes * 8) / windowSec,
+      bpsFwd: (e.bytesFwd * 8) / windowSec,
+      bpsRev: (e.bytesRev * 8) / windowSec,
+    }))
+    .sort((a, b) => b.bytes - a.bytes)
+
   const listener = getFlowListenerState()
   return {
     hops: [...hops.values()]
@@ -209,6 +277,9 @@ export function buildFlowMapHops(q: FlowMapHopsQuery): FlowMapHopsDto {
     live: listener.bound,
     rangeMinutes: q.minutes,
     windowSec,
+    totalBytes,
+    services,
+    serviceEdges,
     dedupApplied: wantDedup,
     excludeMeshApplied: excludeMesh,
     excludeOverlayApplied: excludeOverlay,