diff --git a/app/(main)/network-map/page.tsx b/app/(main)/network-map/page.tsx index 11e3e5c4..2f0d3f4e 100644 --- a/app/(main)/network-map/page.tsx +++ b/app/(main)/network-map/page.tsx @@ -617,7 +617,7 @@ function NetflowRateBadge({ onClick={(e) => { e.stopPropagation(); onOpen?.(e) }} > - Поток NetFlow между узлами (как в «Трафик»: 5 мин, без overlay/mesh). Скорость канала — отдельно. + Поток NetFlow между узлами за интервал карты, без overlay/mesh. {inFlight=false;if(!cancelled&&request===epoch)setFlowLoading(false)}) } queueMicrotask(()=>{if(!cancelled)tick()}) @@ -2224,7 +2224,7 @@ function NetworkMapContent() { {useLiveData ? (readAt ? "Снимок API · ответ "+new Date(readAt).toLocaleTimeString("ru-RU")+" (время браузера)" : "Данные карты неизвестны") : "Демо · mock"} - {useLiveData&&{showNetflow||showServices?(flowReadAt?"NetFlow · 5 мин · ответ "+new Date(flowReadAt).toLocaleTimeString("ru-RU")+" (время браузера) · "+(flowListener?"приёмник включён":"приёмник выключен"):"Данные NetFlow неизвестны"):"NetFlow: отображение и обновление выключены"}} + {useLiveData&&{showNetflow||showServices?(flowReadAt?`NetFlow · ${mapWindowSec / 60} мин${mapWindowSec < 300 ? " из 5 запрошенных" : ""} · ответ `+new Date(flowReadAt).toLocaleTimeString("ru-RU")+" (время браузера) · "+(flowListener?"приёмник включён":"приёмник выключен"):"Данные NetFlow неизвестны"):"NetFlow: отображение и обновление выключены"}} {useLiveData&&flowLoading&&(showNetflow||showServices)&&Обновление NetFlow…} {useLiveData&&flowError&&(showNetflow||showServices)&&NetFlow: {flowError}. {flowReadAt?"Сохранён предыдущий снимок NetFlow; актуальность не подтверждена.":"Первоначальная загрузка NetFlow не подтверждена."}} {useLiveData&&dnsError&&DNS: {dnsError}. Связи по неизвестному адресу не подтверждены.} @@ -2934,7 +2934,7 @@ function NetworkMapContent() { ? "Ping и/или TX/RX — с последнего прогона speed-пробы; проба сопоставляется с этим GRE по WAN и интерфейсам. " : useLiveData?"Измерения GRE неизвестны: нет подходящей пробы. ":"Измерения демонстрационные. "} {spGr?"Проба "+spGr.id+" · время Ping: "+(spGr.lastPingAt?new Date(spGr.lastPingAt).toLocaleString("ru-RU"):"неизвестно")+" · время TX/RX неизвестно. ":""} - Поток — снимок NetFlow за 5 мин (как в «Трафик»: без overlay/mesh), не ёмкость канала. + Поток по данным NetFlow за {mapWindowSec / 60} мин, без overlay/mesh.

diff --git a/backend/src/services/traffic-flow-analytics.test.ts b/backend/src/services/traffic-flow-analytics.test.ts index 3c7f6670..4143db97 100644 --- a/backend/src/services/traffic-flow-analytics.test.ts +++ b/backend/src/services/traffic-flow-analytics.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict" import { rememberServerIfaces, resetIfaceCacheForTests } from "./traffic-flow-ifindex.js" import { ingestParsedFlowsForServerForTests, + getRingMbps, resetFlowRingsForTests, } from "./traffic-flow-ingest.js" import { buildFlowAnalytics, formatLiveSseFromBuilder, getFlowMonthly, listFlowClients, listFlowExporters, resetFlowAnalyticsCacheForTests } from "./traffic-flow-analytics.js" @@ -98,6 +99,9 @@ ingestParsedFlowsForServerForTests(serverId, [ ]) try { + const ring = getRingMbps(serverId) + assert.equal(ring.rxNow,0) + assert.equal(ring.txNow,12800 * 8 / 2 / 1e6) const all = await buildFlowAnalytics({ minutes: 5, serverId }) assert.equal(all.applications[0]?.label, "HTTPS") assert.ok(all.protocols.some((p) => p.label === "TCP")) @@ -119,6 +123,14 @@ try { const other = await buildFlowAnalytics({ minutes: 5, serverId: 99 }) assert.equal(other.conversations, 0) + + ingestParsedFlowsForServerForTests(serverId, [ + { src:"10.1.1.9", dst:"9.9.9.9", proto:6, srcPort:50001, dstPort:443, inIface:"2", bytes:1200, packets:2 }, + { src:"8.8.4.4", dst:"9.9.9.9", proto:6, srcPort:50002, dstPort:443, inIface:"2", bytes:3400, packets:4 }, + ]) + const mixed = await buildFlowAnalytics({ minutes:5, serverId }) + assert.equal(mixed.bytes, 12800 + 1200 + 3400) + assert.equal(mixed.bpsNow, mixed.bytes * 8 / 300, "Unattributed transit remains in the total rate") } finally { resetFlowRingsForTests() resetIfaceCacheForTests() @@ -561,16 +573,21 @@ resetFlowRingsForTests() await dbQuery("DELETE FROM flow_buckets WHERE server_id=$1", [serverId]) for (const minutesAgo of [2, 3, 4, 10]) { await dbQuery(`INSERT INTO flow_buckets (server_id,bucket_at,src,dst,in_iface,bytes,packets) - VALUES ($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'10.1.1.8','8.8.8.8','2',60000,10)`, [serverId, minutesAgo]) + VALUES ($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'10.1.1.8','8.8.8.8','2',60000,10), + ($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'8.8.8.8','10.1.1.8','11',240000,40)`, [serverId, minutesAgo]) } for (const minutes of [5, 15]) for (const dedup of [true, false]) { resetFlowAnalyticsCacheForTests() const history = await buildFlowAnalytics({ minutes, serverId, dedup, excludeMesh:false, excludeOverlay:false }) assert.equal(history.rxSeries.length, minutes) assert.equal(history.rxSeries.filter(v => v > 0).length, minutes === 5 ? 3 : 4) + assert.deepEqual(history.rxSeries,history.txSeries.map(v => v * 4)) const missing = await buildFlowAnalytics({ minutes, serverId, iface:"not-an-interface" }) assert(missing.rxSeries.every(v => v === 0), "Historical chart respects interface filter") } +const historicalCard = (await listFlowExporters(5)).exporters.find(s => s.id === String(serverId))! +assert(historicalCard.txNow > 0) +assert.equal(historicalCard.rxNow,historicalCard.txNow * 4) } finally { for (const table of ["flow_daily_dims", "flow_hour_facts", "flow_daily_facts", "flow_buckets", "traffic_samples", "user_interface_bindings", "server_snapshots"]) { diff --git a/backend/src/services/traffic-flow-analytics.ts b/backend/src/services/traffic-flow-analytics.ts index 08886eea..3f20f334 100644 --- a/backend/src/services/traffic-flow-analytics.ts +++ b/backend/src/services/traffic-flow-analytics.ts @@ -29,7 +29,7 @@ import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js" import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js" import { enqueueRipeMisses } from "./traffic-flow-ripe.js" import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js" -import { resolveInternetDest } from "./traffic-flow-dest.js" +import { flowRateDirection, resolveInternetDest } from "./traffic-flow-dest.js" import { refreshFlowCatalogInBackground } from "./traffic-flow-classify.js" import { enGreIfaceNames, @@ -106,22 +106,29 @@ async function userIfaceAllow(userId: string): Promise> return allow } -function seriesFromRows(rows: PendingFlowRow[], minutes: number): { rx: number[]; tx: number[] } { +function seriesFromRows(rows: PendingFlowRow[], minutes: number, direction: (row: PendingFlowRow) => "rx" | "tx" | "both" | null) { const slots = Math.min(60, Math.max(5, minutes)) const slotMs = (minutes * 60_000) / slots const start = Date.now() - minutes * 60_000 const rx = Array(slots).fill(0) as number[] const tx = Array(slots).fill(0) as number[] + let undirectedBytes = 0 for (const r of rows) { const t = Date.parse(r.bucketAt) if (!Number.isFinite(t)) continue const idx = Math.min(slots - 1, Math.max(0, Math.floor((t - start) / slotMs))) - rx[idx] += r.bytes + const side = direction(r) + if (side === null) undirectedBytes += r.bytes + if (side === "rx" || side === "both") rx[idx] += r.bytes + if (side === "tx" || side === "both") tx[idx] += r.bytes } const slotSec = Math.max(1, slotMs / 1000) return { rx: rx.map((b) => bpsToMbps((b * 8) / slotSec)), - tx, + tx: tx.map((b) => bpsToMbps((b * 8) / slotSec)), + rxNow: bpsToMbps(rx.reduce((a, b) => a + b, 0) * 8 / (minutes * 60)), + txNow: bpsToMbps(tx.reduce((a, b) => a + b, 0) * 8 / (minutes * 60)), + undirectedBytes, } } @@ -233,7 +240,8 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF for (const r of raw) { const resolved = resolveIfaceName(r.serverId, r.inIface) const outResolved = resolveIfaceName(r.serverId, r.outIface) - if (!flowRowMatchesFilter(r, resolved.name, q, allow)) continue + if (!flowRowMatchesFilter(r, resolved.name, q, allow) + && !flowRowMatchesFilter({ serverId: r.serverId, inIface: r.outIface }, outResolved.name, q, allow)) continue ifacesForWire.add(resolved.name) if (outResolved.name && outResolved.name !== "—") ifacesForWire.add(outResolved.name) const plane = classifyFlowPlane({ @@ -441,7 +449,19 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF : { rx: Array(60).fill(0) as number[], tx: Array(60).fill(0) as number[], rxNow: 0, txNow: 0 } // The live ring covers about one minute; historical charts must use the requested window. - const fromBuckets = seriesFromRows(matched, q.minutes) + // Keep the winning interface's individual minute buckets when deduplicating. + const chosenIfaces = wantDedup ? new Map(working.map(row => [flowTupleKey(row), row.inIface])) : null + const chartRows = chosenIfaces ? matched.filter(r => chosenIfaces.get(flowTupleKey(r)) === r.inIface) : matched + const fromBuckets = seriesFromRows(chartRows, q.minutes, row => { + if (ifaceFilter || allow) { + const ingress = flowRowMatchesFilter(row, resolveIfaceName(row.serverId, row.inIface).name, q, allow) + const egress = flowRowMatchesFilter({ serverId: row.serverId, inIface: row.outIface }, resolveIfaceName(row.serverId, row.outIface).name, q, allow) + if (ingress && egress) return "both" + // At a client binding, ingress is the client's upload. + return allow && !ifaceFilter ? (ingress ? "tx" : "rx") : (ingress ? "rx" : "tx") + } + return flowRateDirection(row.serverId, row, topo, catalog.byId.get(row.serverId)?.host) + }) const rxSeries = fromBuckets.rx const txSeries = fromBuckets.tx @@ -491,7 +511,7 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF if (scope && (!isDeepStrictEqual(topo, await loadFlowTopology(scope.serverIds, authorize)) || (await readFlowSourceLimits()).topN !== settings.topN)) throw new Error("Топология или предел источника NetFlow изменились во время чтения") await authorize() return { - bpsNow: (ring.rxNow + ring.txNow) * 1_000_000 || (totalBytes * 8) / windowSec, + bpsNow: (!fromBuckets.undirectedBytes && (ring.rxNow + ring.txNow) * 1_000_000) || (totalBytes * 8) / windowSec, bytes: totalBytes, packets: totalPackets, conversations: conv.size, @@ -536,30 +556,32 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF } } -function summarizeByServer(rows: PendingFlowRow[]) { - const bytes = new Map() - const sessions = new Map() +function groupRowsByServer(rows: PendingFlowRow[]) { + const byServer = new Map() for (const r of rows) { - bytes.set(r.serverId, (bytes.get(r.serverId) ?? 0) + r.bytes) - sessions.set(r.serverId, (sessions.get(r.serverId) ?? 0) + 1) + const group = byServer.get(r.serverId) + if (group) group.push(r) + else byServer.set(r.serverId, [r]) } - return { bytes, sessions } + return byServer } export async function listFlowExporters(minutes: number): Promise { const runtime = await getFlowRuntimeCounters() const inputs = await readFlowExporterInputs(minutes) const rows = inputs.window.rows - const { bytes, sessions } = summarizeByServer(rows) - const ids = new Set([...bytes.keys()]) + const byServer = groupRowsByServer(rows) + const ids = new Set(byServer.keys()) for (const id of inputs.configuredIds) ids.add(id) const catalog = inputs.catalog - const emptySeries = Array(60).fill(0) as number[] + const topo = await loadFlowTopology() const exporters = catalog.list .filter((s) => ids.has(s.id)) .map((s) => { const ring = getRingMbps(s.id, "__all__") - const total = bytes.get(s.id) ?? 0 + const records = byServer.get(s.id) ?? [] + const rates = ring.rx.some(Boolean) || ring.tx.some(Boolean) ? ring + : seriesFromRows(records, minutes, row => flowRateDirection(s.id, row, topo, s.host)) return { id: String(s.id), name: s.name, @@ -567,12 +589,12 @@ export async function listFlowExporters(minutes: number): Promise v > 0) ? ring.rx : emptySeries, - txSeries: ring.tx, - bytes: total, + rxNow: rates.rxNow, + txNow: rates.txNow, + sessions: records.length, + rxSeries: rates.rx, + txSeries: rates.tx, + bytes: records.reduce((sum, row) => sum + row.bytes, 0), } satisfies FlowEntityCard }) .sort((a, b) => b.rxNow - a.rxNow) @@ -599,8 +621,6 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id: byUser.set(b.userId, list) } const rows = inputs.window.rows - const emptySeries = Array(60).fill(0) as number[] - const windowSec = Math.max(60, minutes * 60) const clients: FlowEntityCard[] = [] for (const u of users) { const userBinds = byUser.get(u.id) ?? [] @@ -611,18 +631,15 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id: set.add(b.interfaceName) allow.set(b.serverId, set) } - let total = 0 - let sessions = 0 - for (const r of rows) { - const resolved = resolveIfaceName(r.serverId, r.inIface) - const names = allow.get(r.serverId) - if (!names) continue - if (!names.has(resolved.name) && !names.has(r.inIface)) continue - total += r.bytes - sessions += 1 + const bound = (id: number, iface: string) => { + const names = allow.get(id) + return names?.has(iface) || names?.has(resolveIfaceName(id, iface).name) } - const firstServer = userBinds[0]?.serverId - const ring = firstServer ? getRingMbps(firstServer, "__all__") : { rx: emptySeries, tx: emptySeries, rxNow: 0, txNow: 0 } + const matched = rows.filter(r => bound(r.serverId, r.inIface) || bound(r.serverId, r.outIface)) + const rates = seriesFromRows(matched, minutes, r => { + if (!bound(r.serverId, r.inIface)) return "rx" + return bound(r.serverId, r.outIface) ? "both" : "tx" + }) clients.push({ id: u.id, name: u.login, @@ -630,12 +647,12 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id: site: `${userBinds.length} ifaces`, country: "UN", status: u.active ? "online" : "offline", - rxNow: (total * 8) / windowSec / 1_000_000 || ring.rxNow, - txNow: ring.txNow, - sessions, - rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : emptySeries, - txSeries: ring.tx, - bytes: total, + rxNow: rates.rxNow, + txNow: rates.txNow, + sessions: matched.length, + rxSeries: rates.rx, + txSeries: rates.tx, + bytes: matched.reduce((sum, row) => sum + row.bytes, 0), }) } clients.sort((a, b) => b.rxNow - a.rxNow) diff --git a/backend/src/services/traffic-flow-dest.test.ts b/backend/src/services/traffic-flow-dest.test.ts index 96bfc7b3..48def27c 100644 --- a/backend/src/services/traffic-flow-dest.test.ts +++ b/backend/src/services/traffic-flow-dest.test.ts @@ -4,7 +4,7 @@ import { resetEngineForTests, } from "./traffic-flow-engine.js" import { factsSnapshotForTests } from "./traffic-flow-facts.js" -import { classifyInternetBrand, mapInternetBrand, resolveInternetDest } from "./traffic-flow-dest.js" +import { classifyInternetBrand, flowRateDirection, mapInternetBrand, resolveInternetDest } from "./traffic-flow-dest.js" import { disableCatalogFetchForTests, resetFlowCatalogForTests } from "./traffic-flow-classify.js" import { disableRipeEnqueueForTests, @@ -66,6 +66,11 @@ const topo: FlowTopology = { seedFlowTopologyForTests(topo) rememberServerIfaces(1, [{ name: "gre-client", ifindex: "2" }]) +const publicFlow = { src: "95.167.1.10", dst: "8.8.8.8", inIface: "2", outIface: "ether1" } +assert.equal(flowRateDirection(1, publicFlow, topo), "tx") +assert.equal(flowRateDirection(1, { ...publicFlow, inIface: "ether1", outIface: "2" }, topo), "rx") +assert.equal(flowRateDirection(1, publicFlow, null), null, "Public transit has no inferred download or upload") + ingestParsedFlowsForServerForTests(1, [ { src: "95.167.1.10", diff --git a/backend/src/services/traffic-flow-dest.ts b/backend/src/services/traffic-flow-dest.ts index 2e0476e7..ef7d1de8 100644 --- a/backend/src/services/traffic-flow-dest.ts +++ b/backend/src/services/traffic-flow-dest.ts @@ -4,6 +4,9 @@ import { classifyFlowDst, type FlowClassification } from "./traffic-flow-classif import { resolveFlowIp } from "./traffic-flow-geoip.js" import { canonicalFactIface } from "./traffic-flow-ifindex.js" import { + canonicalIp, + isNonPublicIp, + isUnspecifiedIp, resolveFlowEndpoints, type FlowEndpoints, type InternetDestCtx, @@ -15,6 +18,28 @@ import { type FlowTopology, } from "./traffic-flow-topology.js" +/** Aggregate DL/UL is relative to the exporter's WAN or its bound clients. */ +export function flowRateDirection( + serverId: number, + row: { src: string; dst: string; inIface: string; outIface: string }, + topo: FlowTopology | null, + exporterHost?: string, +): "rx" | "tx" | null { + const ingress = canonicalFactIface(serverId, row.inIface) + const egress = canonicalFactIface(serverId, row.outIface) + const wan = topo?.wanIfaces.get(serverId) + const clients = topo?.clientIfaces.get(serverId) + const download = Boolean(wan?.has(ingress) || clients?.has(egress)) + const upload = Boolean(wan?.has(egress) || clients?.has(ingress)) + if (download !== upload) return download ? "rx" : "tx" + if (isUnspecifiedIp(row.src) || isUnspecifiedIp(row.dst)) return null + const host = canonicalIp(exporterHost) + if (host && canonicalIp(row.src) === host && canonicalIp(row.dst) !== host) return "tx" + if (host && canonicalIp(row.dst) === host && canonicalIp(row.src) !== host) return "rx" + const srcLocal = isNonPublicIp(row.src), dstLocal = isNonPublicIp(row.dst) + return srcLocal === dstLocal ? null : srcLocal ? "tx" : "rx" +} + export interface InternetDestMeta { dest: string ripe: FlowIpMeta | null diff --git a/backend/src/services/traffic-flow-engine-owned.test.ts b/backend/src/services/traffic-flow-engine-owned.test.ts index 9c7f5a68..4a0d6e66 100644 --- a/backend/src/services/traffic-flow-engine-owned.test.ts +++ b/backend/src/services/traffic-flow-engine-owned.test.ts @@ -53,7 +53,7 @@ const dependencies:Record={ "./traffic-flow-ifaces.js":{maybeRefreshIfaces:async()=>{}}, "./traffic-flow-ifindex.js":{canonicalFactIface:(_server:number,iface:string)=>iface}, "./traffic-flow-facts-filter.js":{shouldWriteFlowFact:()=>true}, - "./traffic-flow-dest.js":{resolveInternetDest:()=>({dest:"",ripe:null,classified:{category:"web",service:"https"},country:"XX",asn:0})}, + "./traffic-flow-dest.js":{flowRateDirection:()=>"tx",resolveInternetDest:()=>({dest:"",ripe:null,classified:{category:"web",service:"https"},country:"XX",asn:0})}, "./traffic-flow-topology.js":{peekFlowTopology:()=>({}),peekServerCatalog:()=>({byId:new Map([[1,{type:"router"}]])}),getServerCatalog:()=>{throw Error("unexpected catalog SQL")},loadFlowTopology:()=>{throw Error("unexpected topology SQL")}}, "./traffic-flow-facts.js":{ bumpFlowFact:live.bump,hasPendingFlowFact:live.has,factsPendingSize:()=>live.size, diff --git a/backend/src/services/traffic-flow-engine.ts b/backend/src/services/traffic-flow-engine.ts index f63c5faa..a6774bca 100644 --- a/backend/src/services/traffic-flow-engine.ts +++ b/backend/src/services/traffic-flow-engine.ts @@ -18,7 +18,7 @@ import { maybeRefreshIfaces } from "./traffic-flow-ifaces.js" import { canonicalFactIface } from "./traffic-flow-ifindex.js" import { shouldWriteFlowFact } from "./traffic-flow-facts-filter.js" import { canonicalIp } from "./traffic-flow-ip.js" -import { resolveInternetDest } from "./traffic-flow-dest.js" +import { flowRateDirection, resolveInternetDest } from "./traffic-flow-dest.js" import { getServerCatalog, loadFlowTopology, @@ -315,12 +315,12 @@ function bumpTick(key: string, inBytes: number, outBytes: number): void { tickAccum.set(key, prev) } -function prepareFlowTicks(serverId:number,flow:ParsedFlow):Array<{key:string;inBytes:number;outBytes:number}> { +function prepareFlowTicks(serverId:number,flow:ParsedFlow,direction:"rx"|"tx"|null):Array<{key:string;inBytes:number;outBytes:number}> { const plane=classifyFlowPlaneLite(flow) if(plane==="mgmt")return [] - return [{key:ringKey(serverId,plane==="overlay"?RING_OVERLAY:plane==="client_mesh"?RING_MESH:RING_PAYLOAD),inBytes:flow.bytes,outBytes:0}, + return [{key:ringKey(serverId,plane==="overlay"?RING_OVERLAY:plane==="client_mesh"?RING_MESH:RING_PAYLOAD),inBytes:direction==="rx"?flow.bytes:0,outBytes:direction==="tx"?flow.bytes:0}, ...(flow.inIface?[{key:ringKey(serverId,flow.inIface),inBytes:flow.bytes,outBytes:0}]:[]), - ...(flow.outIface&&flow.outIface!==flow.inIface?[{key:ringKey(serverId,flow.outIface),inBytes:0,outBytes:flow.bytes}]:[])] + ...(flow.outIface?[{key:ringKey(serverId,flow.outIface),inBytes:0,outBytes:flow.bytes}]:[])] } function emptyRing(): { inBps: number[]; outBps: number[] } { @@ -426,7 +426,8 @@ export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): vo const topo = peekFlowTopology(), catalog = peekServerCatalog() if (!topo) refreshColdTopology() if (!catalog) refreshColdCatalog() - const serverType = catalog?.byId.get(serverId)?.type + const server = catalog?.byId.get(serverId) + const serverType = server?.type for (const raw of flows) { const flow = normalizeParsedFlow(raw) const destMeta = resolveInternetDest({src:flow.src,dst:flow.dst,proto:flow.proto,srcPort:flow.srcPort,dstPort:flow.dstPort, @@ -440,7 +441,7 @@ export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): vo natSrc:flow.natSrc,natDst:flow.natDst,natSrcPort:flow.natSrcPort,natDstPort:flow.natDstPort})? {serverId,bucketAt:hourAt,iface:canonicalFactIface(serverId,flow.inIface),country:country||"XX",service:classified.service, asn:destMeta.asn,bytes:flow.bytes,packets:flow.packets}:null - const ticks=prepareFlowTicks(serverId,flow),rollupCacheKey=rollupKey(serverId,bucketAt) + const ticks=prepareFlowTicks(serverId,flow,flowRateDirection(serverId,flow,topo,server?.host)),rollupCacheKey=rollupKey(serverId,bucketAt) if(!admitAggregateRecord(rollupCacheKey,flow,dims,fact,ticks,observedAt))continue for(const tick of ticks)bumpTick(tick.key,tick.inBytes,tick.outBytes) bumpRollup(serverId,bucketAt,flow,flow.bytes,flow.packets,rollupCacheKey) diff --git a/backend/src/services/traffic-flow-read-http.owned.test.ts b/backend/src/services/traffic-flow-read-http.owned.test.ts index 1c05c89e..5fd9569d 100644 --- a/backend/src/services/traffic-flow-read-http.owned.test.ts +++ b/backend/src/services/traffic-flow-read-http.owned.test.ts @@ -36,7 +36,7 @@ try{ await pool.query("INSERT INTO servers(id,name,host) VALUES(1,'visible','192.0.2.1'),(2,'FORBIDDEN_HIDDEN_SERVER','192.0.2.2')") const bucket=new Date(Date.now()-60000);await ensurePartitionFor(pool,'flow_buckets','day',bucket) await pool.query("INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface) VALUES(1,$1,'198.18.1.1','8.8.8.8',6,1,443,100,1,'ether1'),(2,$1,'198.18.2.1','8.8.8.8',6,1,443,999,1,'FORBIDDEN_HIDDEN_IFACE')",[bucket]) - const access=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned HTTP',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','traffic.flow.monthly.read','traffic.flow.live.read'],allowGlobal:false},limits:{callsPerMinute:240}}),actor) + const access=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned HTTP',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','traffic.flow.monthly.read','traffic.flow.live.read','traffic.flow.analytics.read'],allowGlobal:false},limits:{callsPerMinute:240}}),actor) resetFlowRingsForTests();disableRipeEnqueueForTests() const collectorQuality=createFlowIngestQuality();collectorQuality.reject(987654,987,'aggregate-capacity') // Synthetic quality goes through the actual validated heartbeat receiver; @@ -146,15 +146,22 @@ try{ await pool.query(`INSERT INTO traffic_samples(server_id,interface_name,sampled_at,rx_bps,tx_bps,rx_bytes,tx_bytes) SELECT 1,'gre-test',$1::timestamptz + n * interval '1 millisecond',n,2*n,3*n,4*n FROM generate_series(1,50010) n`,[counterStart]) await pool.query(`INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface) - VALUES(1,$1,'198.18.1.3','8.8.8.8',6,3,443,50,1,'gre-test')`,[bucket]) + VALUES(1,$1,'8.8.8.8','198.18.1.1',6,443,1,50,1,'gre-test')`,[bucket]) applyHeartbeatForTests({...heartbeat,dataQuality:createFlowIngestQuality().snapshot()}) try { const analytics = await fetch(endpoint+'/api/traffic/flow/analytics?range=5m&serverId=1') assert.equal(analytics.status,200,await analytics.clone().text()) - const shortReport = await analytics.json() as {bpsWire:number;bytesWire:number;sourceWindow?:unknown} + const shortReport = await analytics.json() as {bpsWire:number;bytesWire:number;sourceWindow?:unknown;rxSeries:number[];txSeries:number[]} assert.equal(shortReport.bpsWire,150030) assert.equal(shortReport.bytesWire,350070) assert.equal(shortReport.sourceWindow,undefined) + const report = await client.callTool({name:'mm_netflow_analytics_report_get',arguments:{serverId:1,minutes:5}}) + assert.ok(!report.isError,JSON.stringify(report)) + const mcp = (report.structuredContent as {result:{analytics:{rxSeries:number[];txSeries:number[]}}}).result.analytics + for (const series of [shortReport,mcp]) { + assert.equal(series.rxSeries.reduce((sum,value)=>sum+value,0),50*8/60/1e6) + assert.equal(series.txSeries.reduce((sum,value)=>sum+value,0),100*8/60/1e6) + } } finally { applyHeartbeatForTests(heartbeat) invalidateFlowAnalyticsCache() diff --git a/components/traffic/flow-analytics-panel.tsx b/components/traffic/flow-analytics-panel.tsx index 97dcbcef..54926842 100644 --- a/components/traffic/flow-analytics-panel.tsx +++ b/components/traffic/flow-analytics-panel.tsx @@ -475,6 +475,7 @@ export function FlowAnalyticsDetail({ {analytics && <> {analytics.sourceWindow &&

Данные за последние {analytics.sourceWindow.effectiveMinutes} мин из запрошенных {analytics.sourceWindow.requestedMinutes} мин.

} +

График RX/TX показывает потоки с известным направлением. Общий объём включает также транзит с неопределённым направлением.

networkMapFlowSchema.parse({...empty,totalBytes:undefined})) assert.throws(()=>networkMapFlowSchema.parse({...empty,namedBytes:1})) assert.throws(()=>networkMapFlowSchema.parse({...empty,rangeMinutes:10})) + const shorter={...empty,rangeMinutes:2,windowSec:120,sourceWindow:{requestedMinutes:5,effectiveMinutes:2}} + assert.equal(networkMapFlowSchema.parse(shorter).rangeMinutes,2) + assert.throws(()=>networkMapFlowSchema.parse({...shorter,sourceWindow:undefined})) + assert.throws(()=>networkMapFlowSchema.parse({...shorter,sourceWindow:{requestedMinutes:15,effectiveMinutes:2}})) + assert.throws(()=>networkMapFlowSchema.parse({...shorter,rangeMinutes:1,windowSec:60})) + assert.throws(()=>networkMapFlowSchema.parse({...shorter,windowSec:300})) }) test("map DNS matches exactly requested names and accepts explicit unresolved null",()=>{ assert.deepEqual(validateNetworkMapDns({results:{"router.example":null}},["router.example"]),{"router.example":null}) diff --git a/lib/network-map-response.ts b/lib/network-map-response.ts index d3944821..ecb07f2f 100644 --- a/lib/network-map-response.ts +++ b/lib/network-map-response.ts @@ -27,7 +27,13 @@ export function validateNetworkMapSnapshot(servers:unknown,gre:unknown,probes:un import {flowMapHopsDtoSchema} from "@mmapp/contracts/traffic-flow" import {isIpv4String} from "./gre-endpoint-resolve" -export const networkMapFlowSchema=flowMapHopsDtoSchema.required().refine(data=>data.rangeMinutes===5&&data.namedBytes<=data.totalBytes&&data.unclassifiedBytes<=data.totalBytes) +export const networkMapFlowSchema = flowMapHopsDtoSchema.required().partial({ sourceWindow: true }).refine(data => + (data.sourceWindow + ? data.sourceWindow.requestedMinutes === 5 && data.sourceWindow.effectiveMinutes === data.rangeMinutes + : data.rangeMinutes === 5) && + data.windowSec === data.rangeMinutes * 60 && + data.namedBytes <= data.totalBytes && data.unclassifiedBytes <= data.totalBytes +) export function validateNetworkMapDns(raw:unknown,hosts:string[]){ const data=z.object({results:z.record(z.string(),z.string().refine(isIpv4String).nullable())}).parse(raw) const requested=new Set(hosts.map(host=>host.toLowerCase())) diff --git a/tests/e2e/network-map-flow.spec.ts b/tests/e2e/network-map-flow.spec.ts index 4a89b4f5..a3b04c32 100644 --- a/tests/e2e/network-map-flow.spec.ts +++ b/tests/e2e/network-map-flow.spec.ts @@ -5,12 +5,13 @@ test.beforeEach(async({page})=>{await page.route("**/health",r=>r.fulfill({json: const server={id:1,name:"Router",host:"192.0.2.1",type:"home-router",country:"RU",enabled:true,status:"online"} for(const malformed of [false,true])test("Карта: NetFlow отказ и повтор "+malformed,async({page})=>{ let failing=true - await page.route("**/api/**",r=>{const p=new URL(r.request().url()).pathname;return r.fulfill(p===flowPath?(failing?(malformed?{json:{}}:{status:503,json:{error:"NetFlow недоступен"}}):{json:empty}):{json:p==="/api/servers"?[server]:p==="/api/filters/gre-tunnels"?{tunnels:[],failures:[]}:p==="/api/uptime/speed-probes"?{probes:[]}:p==="/api/auth/config"?{required:false}:{}})}) + const recovered=malformed?{...empty,rangeMinutes:2,windowSec:120,sourceWindow:{requestedMinutes:5,effectiveMinutes:2}}:empty + await page.route("**/api/**",r=>{const p=new URL(r.request().url()).pathname;return r.fulfill(p===flowPath?(failing?(malformed?{json:{}}:{status:503,json:{error:"NetFlow недоступен"}}):{json:recovered}):{json:p==="/api/servers"?[server]:p==="/api/filters/gre-tunnels"?{tunnels:[],failures:[]}:p==="/api/uptime/speed-probes"?{probes:[]}:p==="/api/auth/config"?{required:false}:{}})}) await page.goto("/network-map") await expect(page.getByRole("alert").filter({hasText:"Первоначальная загрузка NetFlow"})).toBeVisible() await expect(page.getByText("Данные NetFlow неизвестны",{exact:true})).toBeVisible() failing=false;await page.getByRole("button",{name:"Обновить",exact:true}).click() - await expect(page.getByText(/NetFlow · 5 мин · ответ/)).toContainText("приёмник выключен") + await expect(page.getByText(malformed?/NetFlow · 2 мин из 5 запрошенных · ответ/:/NetFlow · 5 мин · ответ/)).toContainText("приёмник выключен") await expect(page.getByRole("alert").filter({hasText:"NetFlow:"})).toHaveCount(0) }) test("Карта: NetFlow сохраняет сервис при фоновом сбое",async({page})=>{