From 4ce6169d1407767d67c28d20a4d04e2121288d1b Mon Sep 17 00:00:00 2001 From: Denozordec Date: Sat, 12 Sep 2026 01:07:26 +0700 Subject: [PATCH] feat(traffic-flow): enhance traffic classification and map display - Added a new state variable `mapAsnLoaded` to track ASN loading status in the NetworkMapPage. - Updated the page to conditionally display a message when ASN data is not loaded, improving user feedback. - Enhanced traffic classification tests to include additional scenarios for Google and YouTube services, ensuring comprehensive coverage. - Introduced a new function `pickMapInternetDest` to improve destination selection logic in traffic flow processing. - Updated the traffic flow destination classification to handle new conditions and improve accuracy. Co-authored-by: Cursor --- app/(main)/network-map/page.tsx | 8 +++++ .../services/traffic-flow-classify.test.ts | 23 +++++++++++++ .../src/services/traffic-flow-dest.test.ts | 33 +++++++++++++++++++ backend/src/services/traffic-flow-dest.ts | 29 +++++++++------- .../src/services/traffic-flow-geoip.test.ts | 16 +++++++++ backend/src/services/traffic-flow-ip.test.ts | 26 ++++++++++++++- backend/src/services/traffic-flow-ip.ts | 21 ++++++++++++ backend/src/services/traffic-flow-map-hops.ts | 15 ++++++--- packages/contracts/src/traffic-flow.ts | 2 ++ 9 files changed, 156 insertions(+), 17 deletions(-) diff --git a/app/(main)/network-map/page.tsx b/app/(main)/network-map/page.tsx index 22124e9..c0f857c 100644 --- a/app/(main)/network-map/page.tsx +++ b/app/(main)/network-map/page.tsx @@ -1103,6 +1103,7 @@ export default function NetworkMapPage() { const [mapNamedBytes, setMapNamedBytes] = useState(0) const [mapTotalBytes, setMapTotalBytes] = useState(0) const [mapWindowSec, setMapWindowSec] = useState(300) + const [mapAsnLoaded, setMapAsnLoaded] = useState(true) /** FQDN из GRE outer → IPv4 (ответ POST /api/network/resolve-hosts), для матчинга с WAN. */ const [greResolvedIpv4ByHost, setGreResolvedIpv4ByHost] = useState>({}) const [dataError, setDataError] = useState(null) @@ -1310,6 +1311,7 @@ export default function NetworkMapPage() { if (res.mapServiceMinSharePct != null) setMapSharePct(res.mapServiceMinSharePct) setMapNamedBytes(res.namedBytes ?? 0) setMapTotalBytes(res.totalBytes ?? 0) + if (res.asnLoaded != null) setMapAsnLoaded(res.asnLoaded) if (res.windowSec) setMapWindowSec(res.windowSec) }) .catch((err: unknown) => { @@ -2767,6 +2769,12 @@ export default function NetworkMapPage() { )} + {!mapAsnLoaded && ( +
+ GeoLite2 ASN + не загружена +
+ )}
Скорость diff --git a/backend/src/services/traffic-flow-classify.test.ts b/backend/src/services/traffic-flow-classify.test.ts index 3cf427d..ecf3e6a 100644 --- a/backend/src/services/traffic-flow-classify.test.ts +++ b/backend/src/services/traffic-flow-classify.test.ts @@ -132,6 +132,29 @@ const greIgnore = classifyFlowDst("8.8.8.8", 47, 0, 0, { fetchedAt: Date.now(), }, { ignoreTunnelProto: true }) assert.equal(greIgnore.service, "Google") +const dnsGoogle = classifyFlowDst("8.8.8.8", 17, 53, 53000, { + prefix: "8.8.8.0/24", + asn: 15169, + country: "US", + lat: null, + lng: null, + holder: "GOOGLE", + ok: true, + fetchedAt: Date.now(), +}) +assert.equal(dnsGoogle.service, "Google") +assert.notEqual(dnsGoogle.service, "Прочее") +const ipv6Yt = classifyFlowDst("2001:4860:4860::8888", 17, 443, 50000, { + prefix: "2001:4860:4860::8888/128", + asn: 15169, + country: "US", + lat: null, + lng: null, + holder: "GOOGLE", + ok: true, + fetchedAt: Date.now(), +}) +assert.equal(ipv6Yt.service, "YouTube") const esp = classifyFlowDst("198.51.100.1", 50, 0, 0, null) assert.equal(esp.category, "Туннель") assert.equal(applicationName(17, 443, 50000), "QUIC") diff --git a/backend/src/services/traffic-flow-dest.test.ts b/backend/src/services/traffic-flow-dest.test.ts index 60de39e..2540b68 100644 --- a/backend/src/services/traffic-flow-dest.test.ts +++ b/backend/src/services/traffic-flow-dest.test.ts @@ -149,6 +149,39 @@ assert.equal(mapInternetBrand("8.8.8.8", 6, 443, 51234, { fetchedAt: Date.now(), }).service, "Google") +assert.equal(classifyInternetBrand("8.8.8.8", 17, 53, 53000, { + prefix: "8.8.8.0/24", + asn: 15169, + country: "US", + lat: null, + lng: null, + holder: "GOOGLE", + ok: true, + fetchedAt: Date.now(), +})?.service, "Google") +assert.equal(mapInternetBrand("8.8.8.8", 17, 53, 53000, { + prefix: "8.8.8.0/24", + asn: 15169, + country: "US", + lat: null, + lng: null, + holder: "GOOGLE", + ok: true, + fetchedAt: Date.now(), +}).service, "Google") +assert.equal(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, { + prefix: "2001:4860:4860::8888/128", + asn: 15169, + country: "US", + lat: null, + lng: null, + holder: "GOOGLE", + ok: true, + fetchedAt: Date.now(), +}).service, "YouTube") +assert.equal(mapInternetBrand("64.233.161.1", 17, 443, 50000, null).service, "YouTube") +assert.equal(mapInternetBrand("142.250.1.10", 6, 443, 1, null).service, "YouTube") + resetEngineForTests() seedFlowTopologyForTests(null) resetRipeCacheForTests() diff --git a/backend/src/services/traffic-flow-dest.ts b/backend/src/services/traffic-flow-dest.ts index ac23883..907dbbe 100644 --- a/backend/src/services/traffic-flow-dest.ts +++ b/backend/src/services/traffic-flow-dest.ts @@ -41,7 +41,14 @@ export function destCtxForIface( } } -/** Бренд интернет-dest как на карте: GRE/ESP/WG — транспорт, не сервис. */ +const OTHER_BRAND: FlowClassification = { service: OTHER_SERVICE, category: OTHER_SERVICE } + +function isTunnelProto(proto: number, dstPort: number, srcPort: number): boolean { + if (proto === 47 || proto === 50) return true + return applicationName(proto, dstPort, srcPort) === "WireGuard" +} + +/** Бренд интернет-dest: ASN/CIDR до skip DNS. GRE/ESP/WG — не сервис. */ export function classifyInternetBrand( dst: string, proto: number, @@ -49,17 +56,15 @@ export function classifyInternetBrand( srcPort: number, ripe: FlowIpMeta | null, ): FlowClassification | null { - if (proto === 47 || proto === 50) return null - const app = applicationName(proto, dstPort, srcPort) - if (app === "WireGuard" || app === "DNS" || app === "SSH" || app === "BGP") return null + if (isTunnelProto(proto, dstPort, srcPort)) return null const brand = resolveFlowBrand(dst, ripe?.asn ?? 0, ripe?.holder ?? "", proto, dstPort, srcPort) - if (!brand || !isNamedInternetService(brand.service, brand.category)) return null - return brand + if (brand && isNamedInternetService(brand.service, brand.category)) return brand + const app = applicationName(proto, dstPort, srcPort) + if (app === "DNS" || app === "SSH" || app === "BGP") return null + return null } -const OTHER_BRAND: FlowClassification = { service: OTHER_SERVICE, category: OTHER_SERVICE } - -/** Бренд для карты: именованный сервис или «Прочее» (GRE/ESP не сервис). */ +/** Тот же классификатор, что аналитика (GeoLite2 ASN + catalog). Туннель → Прочее. */ export function mapInternetBrand( dst: string, proto: number, @@ -67,8 +72,10 @@ export function mapInternetBrand( srcPort: number, ripe: FlowIpMeta | null, ): FlowClassification { - if (proto === 47 || proto === 50) return OTHER_BRAND - return classifyInternetBrand(dst, proto, dstPort, srcPort, ripe) ?? OTHER_BRAND + if (isTunnelProto(proto, dstPort, srcPort)) return OTHER_BRAND + const classified = classifyFlowDst(dst, proto, dstPort, srcPort, ripe, { ignoreTunnelProto: true }) + if (isNamedInternetService(classified.service, classified.category)) return classified + return OTHER_BRAND } export function resolveInternetDest(opts: { diff --git a/backend/src/services/traffic-flow-geoip.test.ts b/backend/src/services/traffic-flow-geoip.test.ts index 6ad19c0..b3fee1f 100644 --- a/backend/src/services/traffic-flow-geoip.test.ts +++ b/backend/src/services/traffic-flow-geoip.test.ts @@ -21,6 +21,7 @@ import { minuteDimsSnapshotForTests, } from "./traffic-flow-engine.js" import { classifyFlowDst } from "./traffic-flow-classify.js" +import { mapInternetBrand } from "./traffic-flow-dest.js" import { disableGeoipDbForTests } from "./geoip-settings.js" import { collectGeoipUpdateOnce, @@ -102,6 +103,21 @@ assert.equal(lookupGeoip("6.6.6.6")?.country, "US") const classified = classifyFlowDst("8.8.8.8", 6, 443, 51504, hit) assert.equal(classified.service, "Google") +setGeoipReadersForTests({ + country: fakeCountryReader({ "8.8.8.8": "US", "2001:4860:4860::8888": "US" }), + asn: fakeAsnReader({ + "8.8.8.8": { asn: 15169, org: "GOOGLE" }, + "2001:4860:4860::8888": { asn: 15169, org: "GOOGLE" }, + "64.233.161.1": { asn: 15169, org: "GOOGLE" }, + }), +}) +const v6meta = resolveFlowIp("2001:4860:4860::8888") +assert.equal(v6meta?.asn, 15169) +assert.equal(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, v6meta).service, "YouTube") +assert.notEqual(mapInternetBrand("2001:4860:4860::8888", 17, 443, 50000, v6meta).service, "Прочее") +const cidrYt = mapInternetBrand("64.233.161.1", 17, 443, 50000, resolveFlowIp("64.233.161.1")) +assert.equal(cidrYt.service, "YouTube") + // ── движок: dims country/asn наполняются из geoip-ридеров ──────────────────── resetEngineForTests() ingestParsedFlowsForServerForTests(1, [{ diff --git a/backend/src/services/traffic-flow-ip.test.ts b/backend/src/services/traffic-flow-ip.test.ts index 0526244..34daeef 100644 --- a/backend/src/services/traffic-flow-ip.test.ts +++ b/backend/src/services/traffic-flow-ip.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict" -import { isNonPublicIp, pickInternetDest, pickInternetPeer } from "./traffic-flow-ip.js" +import { isNonPublicIp, pickInternetDest, pickInternetPeer, pickMapInternetDest } from "./traffic-flow-ip.js" assert.equal(isNonPublicIp("10.200.100.53"), true) assert.equal(isNonPublicIp("173.194.151.65"), false) @@ -78,5 +78,29 @@ assert.equal( "", "NAT 0.0.0.0 не dest", ) +assert.equal( + pickMapInternetDest("10.200.100.53", "10.200.100.1", 53880, 443, { + ...client, + natDst: "8.8.8.8", + natDstPort: 443, + }), + "8.8.8.8", + "карта: RFC1918 + NAT Google", +) +assert.equal( + pickMapInternetDest("10.200.100.53", "10.200.100.1", 53880, 443, client), + "", + "карта: RFC1918 без NAT → Прочее", +) +assert.equal( + pickMapInternetDest("173.194.151.65", "10.200.100.53", 12345, 57182, client), + "173.194.151.65", + "карта: реверс googlevideo не :443 — всё равно публичный src", +) +assert.equal( + pickMapInternetDest("203.0.113.10", "198.51.100.1", 0, 0, { ours }), + "", + "карта: JH ours → EN ours всё ещё не dest", +) console.log("traffic-flow-ip.test.ts: ok") diff --git a/backend/src/services/traffic-flow-ip.ts b/backend/src/services/traffic-flow-ip.ts index dd47644..d198f1e 100644 --- a/backend/src/services/traffic-flow-ip.ts +++ b/backend/src/services/traffic-flow-ip.ts @@ -132,6 +132,27 @@ export function pickInternetDest( return dst } +/** + * Dest для карты: сначала pickInternetDest, иначе любой публичный src/dst/NAT + * (без well-known ловушки boundClient — реверс googlevideo не только :80/:443/:53). + */ +export function pickMapInternetDest( + srcRaw: string, + dstRaw: string, + srcPort: number, + dstPort: number, + ctx?: InternetDestCtx, +): string { + const dest = pickInternetDest(srcRaw, dstRaw, srcPort, dstPort, ctx) + if (dest) return dest + const ours = ctx?.ours + const internet = (ip: string) => Boolean(ip) && !isLocalIp(ip, ours) + for (const ip of [usableIp(ctx?.natDst), usableIp(ctx?.natSrc), usableIp(dstRaw), usableIp(srcRaw)]) { + if (internet(ip)) return ip + } + return "" +} + /** * Интернет-сторона потока без топологии: у IPFIX сервис часто в src (Google:443 → RFC1918). * Для куба статистики используйте pickInternetDest. diff --git a/backend/src/services/traffic-flow-map-hops.ts b/backend/src/services/traffic-flow-map-hops.ts index dff293b..54c3a78 100644 --- a/backend/src/services/traffic-flow-map-hops.ts +++ b/backend/src/services/traffic-flow-map-hops.ts @@ -3,14 +3,15 @@ import type { FlowMapHop, FlowMapHopsDto, FlowMapService, FlowMapServiceEdge, Fl import { db } from "../db/index.js" import { userInterfaceBindings } from "../db/schema.js" import { flowRowMatchesFilter } from "./traffic-flow-apps.js" -import { OTHER_SERVICE, mapServiceNodeId } from "./traffic-flow-brands.js" +import { OTHER_SERVICE, isNamedInternetService, mapServiceNodeId } from "./traffic-flow-brands.js" +import { refreshFlowCatalogInBackground } from "./traffic-flow-classify.js" import { dedupFlowRowsAcrossExporters, dedupFlowRowsMaxBytes } from "./traffic-flow-dedup.js" import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-ingest.js" import { resolveIfaceName } from "./traffic-flow-ifaces.js" import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js" import { destCtxForIface, mapInternetBrand } from "./traffic-flow-dest.js" -import { pickInternetDest } from "./traffic-flow-ip.js" -import { resolveFlowIp } from "./traffic-flow-geoip.js" +import { pickMapInternetDest } from "./traffic-flow-ip.js" +import { geoipReadersStatus, resolveFlowIp } from "./traffic-flow-geoip.js" import { getTrafficFlowSettingsRow } from "./traffic-flow-settings.js" import { loadFlowTopology, resolveClient, resolveEn, getServerCatalog, type FlowTopology } from "./traffic-flow-topology.js" import { flowDataEpoch } from "./traffic-flow-engine.js" @@ -192,6 +193,7 @@ async function resolveMinSharePct(q: FlowMapHopsQuery): Promise { } async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number): Promise { + refreshFlowCatalogInBackground() const windowSec = Math.max(60, q.minutes * 60) const raw = await listFlowRowsForWindow(q.minutes) const allow = q.userId ? await userIfaceAllow(q.userId) : null @@ -334,7 +336,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number const inName = resolveIfaceName(r.serverId, r.inIface).name const outName = resolveIfaceName(r.serverId, r.outIface).name totalBytes += r.bytes - const dest = pickInternetDest( + const dest = pickMapInternetDest( r.src, r.dst, r.srcPort, @@ -476,7 +478,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number } const namedBytes = [...svcTotals.values()] - .filter((s) => s.label !== OTHER_SERVICE) + .filter((s) => isNamedInternetService(s.label, s.category)) .reduce((n, s) => n + s.bytes, 0) const unclassifiedBytes = Math.max(0, totalBytes - namedBytes) const shareBase = totalBytes > 0 ? totalBytes : namedBytes @@ -528,6 +530,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number .sort((a, b) => b.bps - a.bps) const listener = getFlowListenerState() + const geo = geoipReadersStatus() return { hops: [...hops.values()] .map((a) => toHop(a, windowSec)) @@ -538,6 +541,8 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number totalBytes, namedBytes, unclassifiedBytes, + asnLoaded: geo.asnLoaded, + countryLoaded: geo.countryLoaded, services, serviceEdges, servicePaths, diff --git a/packages/contracts/src/traffic-flow.ts b/packages/contracts/src/traffic-flow.ts index ce98fda..6155b36 100644 --- a/packages/contracts/src/traffic-flow.ts +++ b/packages/contracts/src/traffic-flow.ts @@ -320,6 +320,8 @@ export const flowMapHopsDtoSchema = z.object({ totalBytes: z.number().nonnegative().optional(), namedBytes: z.number().nonnegative().optional(), unclassifiedBytes: z.number().nonnegative().optional(), + asnLoaded: z.boolean().optional(), + countryLoaded: z.boolean().optional(), services: z.array(flowMapServiceDtoSchema).optional(), serviceEdges: z.array(flowMapServiceEdgeDtoSchema).optional(), servicePaths: z.array(flowMapServicePathDtoSchema).optional(),