Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87461daa42 | ||
|
|
e6b2108f79 | ||
|
|
bec8de7c95 |
No files matched your search
@@ -617,7 +617,7 @@ function NetflowRateBadge({
|
|||||||
onClick={(e) => { e.stopPropagation(); onOpen?.(e) }}
|
onClick={(e) => { e.stopPropagation(); onOpen?.(e) }}
|
||||||
>
|
>
|
||||||
<title>
|
<title>
|
||||||
Поток NetFlow между узлами (как в «Трафик»: 5 мин, без overlay/mesh). Скорость канала — отдельно.
|
Поток NetFlow между узлами за интервал карты, без overlay/mesh.
|
||||||
</title>
|
</title>
|
||||||
<rect className="network-map-focus-ring" x={-bw / 2 - 4} y={-bh / 2 - 4} width={bw + 8} height={bh + 8} rx="9" fill="none" stroke="var(--ring)" strokeWidth="2" />
|
<rect className="network-map-focus-ring" x={-bw / 2 - 4} y={-bh / 2 - 4} width={bw + 8} height={bh + 8} rx="9" fill="none" stroke="var(--ring)" strokeWidth="2" />
|
||||||
<rect
|
<rect
|
||||||
@@ -1442,7 +1442,7 @@ function NetworkMapContent() {
|
|||||||
if (cancelled||request!==epoch) return
|
if (cancelled||request!==epoch) return
|
||||||
const name = err instanceof Error ? err.name : ""
|
const name = err instanceof Error ? err.name : ""
|
||||||
if (name === "AbortError") return
|
if (name === "AbortError") return
|
||||||
setFlowError(err instanceof Error?(err.name==="ZodError"?"Некорректный ответ: требуется полный снимок за 5 минут":err.message):"Ошибка NetFlow")
|
setFlowError(err instanceof Error?(err.name==="ZodError"?"Ответ содержит неполные или противоречивые данные снимка":err.message):"Ошибка NetFlow")
|
||||||
}).finally(()=>{inFlight=false;if(!cancelled&&request===epoch)setFlowLoading(false)})
|
}).finally(()=>{inFlight=false;if(!cancelled&&request===epoch)setFlowLoading(false)})
|
||||||
}
|
}
|
||||||
queueMicrotask(()=>{if(!cancelled)tick()})
|
queueMicrotask(()=>{if(!cancelled)tick()})
|
||||||
@@ -2224,7 +2224,7 @@ function NetworkMapContent() {
|
|||||||
<span className="text-[10px] border border-border rounded px-1.5 py-0.5">
|
<span className="text-[10px] border border-border rounded px-1.5 py-0.5">
|
||||||
{useLiveData ? (readAt ? "Снимок API · ответ "+new Date(readAt).toLocaleTimeString("ru-RU")+" (время браузера)" : "Данные карты неизвестны") : "Демо · mock"}
|
{useLiveData ? (readAt ? "Снимок API · ответ "+new Date(readAt).toLocaleTimeString("ru-RU")+" (время браузера)" : "Данные карты неизвестны") : "Демо · mock"}
|
||||||
</span>
|
</span>
|
||||||
{useLiveData&&<span>{showNetflow||showServices?(flowReadAt?"NetFlow · 5 мин · ответ "+new Date(flowReadAt).toLocaleTimeString("ru-RU")+" (время браузера) · "+(flowListener?"приёмник включён":"приёмник выключен"):"Данные NetFlow неизвестны"):"NetFlow: отображение и обновление выключены"}</span>}
|
{useLiveData&&<span>{showNetflow||showServices?(flowReadAt?`NetFlow · ${mapWindowSec / 60} мин${mapWindowSec < 300 ? " из 5 запрошенных" : ""} · ответ `+new Date(flowReadAt).toLocaleTimeString("ru-RU")+" (время браузера) · "+(flowListener?"приёмник включён":"приёмник выключен"):"Данные NetFlow неизвестны"):"NetFlow: отображение и обновление выключены"}</span>}
|
||||||
{useLiveData&&flowLoading&&(showNetflow||showServices)&&<span role="status">Обновление NetFlow…</span>}
|
{useLiveData&&flowLoading&&(showNetflow||showServices)&&<span role="status">Обновление NetFlow…</span>}
|
||||||
{useLiveData&&flowError&&(showNetflow||showServices)&&<span role="alert" className="text-destructive [overflow-wrap:anywhere]">NetFlow: {flowError}. {flowReadAt?"Сохранён предыдущий снимок NetFlow; актуальность не подтверждена.":"Первоначальная загрузка NetFlow не подтверждена."}</span>}
|
{useLiveData&&flowError&&(showNetflow||showServices)&&<span role="alert" className="text-destructive [overflow-wrap:anywhere]">NetFlow: {flowError}. {flowReadAt?"Сохранён предыдущий снимок NetFlow; актуальность не подтверждена.":"Первоначальная загрузка NetFlow не подтверждена."}</span>}
|
||||||
{useLiveData&&dnsError&&<span role="alert" className="text-destructive [overflow-wrap:anywhere]">DNS: {dnsError}. Связи по неизвестному адресу не подтверждены.</span>}
|
{useLiveData&&dnsError&&<span role="alert" className="text-destructive [overflow-wrap:anywhere]">DNS: {dnsError}. Связи по неизвестному адресу не подтверждены.</span>}
|
||||||
@@ -2934,7 +2934,7 @@ function NetworkMapContent() {
|
|||||||
? "Ping и/или TX/RX — с последнего прогона speed-пробы; проба сопоставляется с этим GRE по WAN и интерфейсам. "
|
? "Ping и/или TX/RX — с последнего прогона speed-пробы; проба сопоставляется с этим GRE по WAN и интерфейсам. "
|
||||||
: useLiveData?"Измерения GRE неизвестны: нет подходящей пробы. ":"Измерения демонстрационные. "}
|
: useLiveData?"Измерения GRE неизвестны: нет подходящей пробы. ":"Измерения демонстрационные. "}
|
||||||
{spGr?"Проба "+spGr.id+" · время Ping: "+(spGr.lastPingAt?new Date(spGr.lastPingAt).toLocaleString("ru-RU"):"неизвестно")+" · время TX/RX неизвестно. ":""}
|
{spGr?"Проба "+spGr.id+" · время Ping: "+(spGr.lastPingAt?new Date(spGr.lastPingAt).toLocaleString("ru-RU"):"неизвестно")+" · время TX/RX неизвестно. ":""}
|
||||||
Поток — снимок NetFlow за 5 мин (как в «Трафик»: без overlay/mesh), не ёмкость канала.
|
Поток по данным NetFlow за {mapWindowSec / 60} мин, без overlay/mesh.
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -823,7 +823,7 @@ export default function TrafficPage() {
|
|||||||
const [flowScope, setFlowScope] = useState<FlowScope>("servers")
|
const [flowScope, setFlowScope] = useState<FlowScope>("servers")
|
||||||
const [flowRetry, setFlowRetry] = useState(0)
|
const [flowRetry, setFlowRetry] = useState(0)
|
||||||
const flowKey = JSON.stringify([isLive, groupMode, backendUrl, range, flowScope, flowRetry])
|
const flowKey = JSON.stringify([isLive, groupMode, backendUrl, range, flowScope, flowRetry])
|
||||||
const [flowResult, setFlowResult] = useState<{ key: string; receivedAt?: number; monthlyServers?: ServerRead[]; stats?: FlowStatsDto; exporters?: FlowEntityCard[]; clients?: FlowEntityCard[]; error?: string } | null>(null)
|
const [flowResult, setFlowResult] = useState<{ key: string; receivedAt?: number; monthlyServers?: ServerRead[]; stats?: FlowStatsDto; exporters?: FlowEntityCard[]; clients?: FlowEntityCard[]; cardWindow?: FlowStatsDto["sourceWindow"]; error?: string } | null>(null)
|
||||||
const currentFlows = flowResult?.key === flowKey ? flowResult : null
|
const currentFlows = flowResult?.key === flowKey ? flowResult : null
|
||||||
const liveBusy = isLive && groupMode === "flows" && !currentFlows
|
const liveBusy = isLive && groupMode === "flows" && !currentFlows
|
||||||
const liveError = currentFlows?.error
|
const liveError = currentFlows?.error
|
||||||
@@ -932,7 +932,8 @@ export default function TrafficPage() {
|
|||||||
if (cancelled) return
|
if (cancelled) return
|
||||||
const clients = await getFlowClients(backendUrl, range)
|
const clients = await getFlowClients(backendUrl, range)
|
||||||
if (cancelled) return
|
if (cancelled) return
|
||||||
setFlowResult({ key: flowKey, receivedAt: Date.now(), stats, exporters: exporters.exporters, clients: clients.clients })
|
setFlowResult({ key: flowKey, receivedAt: Date.now(), stats, exporters: exporters.exporters, clients: clients.clients,
|
||||||
|
cardWindow: flowScope === "users" ? clients.sourceWindow : exporters.sourceWindow })
|
||||||
setSelectedId(prev => {
|
setSelectedId(prev => {
|
||||||
const list = flowScope === "users" ? clients.clients : exporters.exporters
|
const list = flowScope === "users" ? clients.clients : exporters.exporters
|
||||||
return list.some(x => x.id === prev) ? prev : (list[0]?.id ?? "")
|
return list.some(x => x.id === prev) ? prev : (list[0]?.id ?? "")
|
||||||
@@ -1250,6 +1251,8 @@ export default function TrafficPage() {
|
|||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
{isLive && <TrafficDataTime label="Получение списка потоков" receivedAt={currentFlows?.receivedAt} source="API" />}
|
{isLive && <TrafficDataTime label="Получение списка потоков" receivedAt={currentFlows?.receivedAt} source="API" />}
|
||||||
|
{flowStats?.sourceWindow && <p role="status" className="text-sm text-muted-foreground">Сводка потоков за последние {flowStats.sourceWindow.effectiveMinutes} мин из запрошенных {flowStats.sourceWindow.requestedMinutes} мин.</p>}
|
||||||
|
{currentFlows?.cardWindow && <p role="status" className="text-sm text-muted-foreground">Данные карточек за последние {currentFlows.cardWindow.effectiveMinutes} мин из запрошенных {currentFlows.cardWindow.requestedMinutes} мин.</p>}
|
||||||
{liveBusy && <p role="status">Загрузка потоков…</p>}
|
{liveBusy && <p role="status">Загрузка потоков…</p>}
|
||||||
{liveError && <Alert variant="destructive" role="alert"><AlertTitle>Потоки недоступны</AlertTitle><AlertDescription>{liveError}<Button size="sm" variant="outline" onClick={() => setFlowRetry(value => value + 1)}>Повторить загрузку потоков</Button></AlertDescription></Alert>}
|
{liveError && <Alert variant="destructive" role="alert"><AlertTitle>Потоки недоступны</AlertTitle><AlertDescription>{liveError}<Button size="sm" variant="outline" onClick={() => setFlowRetry(value => value + 1)}>Повторить загрузку потоков</Button></AlertDescription></Alert>}
|
||||||
{flowError && !liveError && (
|
{flowError && !liveError && (
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ function evaluate(source,mocks={}){
|
|||||||
return module.exports
|
return module.exports
|
||||||
}
|
}
|
||||||
const contracts=evaluate(readFileSync(resolve(here,'../../packages/contracts/src/traffic-flow.ts'),'utf8'))
|
const contracts=evaluate(readFileSync(resolve(here,'../../packages/contracts/src/traffic-flow.ts'),'utf8'))
|
||||||
|
const readWindow=evaluate(readFileSync(resolve(here,'../src/services/traffic-flow-read-window.ts'),'utf8'))
|
||||||
let quality=null,slotReleases=0
|
let quality=null,slotReleases=0
|
||||||
const qualityHelper=evaluate(readFileSync(resolve(here,'../src/modules/mcp/flow-read-quality.ts'),'utf8'),{
|
const qualityHelper=evaluate(readFileSync(resolve(here,'../src/modules/mcp/flow-read-quality.ts'),'utf8'),{
|
||||||
'@mmapp/contracts/traffic-flow':contracts,'../../services/traffic-flow-ingest.js':{getFlowRuntimeCounters:async()=>({dataQuality:quality})},'./traffic-history-actions.js':{assertTrafficHistoryAccess:async()=>{}},
|
'@mmapp/contracts/traffic-flow':contracts,'../../services/traffic-flow-ingest.js':{getFlowRuntimeCounters:async()=>({dataQuality:quality})},'./traffic-history-actions.js':{assertTrafficHistoryAccess:async()=>{}},
|
||||||
@@ -21,6 +22,7 @@ const window={rows:[],topRowsPerBucketLimit:20,topConversationsLimit:20,since:'f
|
|||||||
class McpError extends Error{constructor(code,message,statusCode){super(message);this.code=code;this.statusCode=statusCode}}
|
class McpError extends Error{constructor(code,message,statusCode){super(message);this.code=code;this.statusCode=statusCode}}
|
||||||
const mocks={
|
const mocks={
|
||||||
'./flow-read-quality.js':qualityHelper,
|
'./flow-read-quality.js':qualityHelper,
|
||||||
|
'../../services/traffic-flow-read-window.js':readWindow,
|
||||||
'@mmapp/contracts/traffic-flow':{...contracts,flowAnalyticsDtoSchema:{parse:x=>x},flowMapHopsDtoSchema:{parse:x=>x}},
|
'@mmapp/contracts/traffic-flow':{...contracts,flowAnalyticsDtoSchema:{parse:x=>x},flowMapHopsDtoSchema:{parse:x=>x}},
|
||||||
'../../services/traffic-flow-analytics.js':{buildFlowAnalytics:async()=>dto,getFlowMonthly:async()=>({month:'2026-10',bytes:0,countries:[],services:[],asns:[]}),safeBuildLiveFlowSample:async()=>({event:'sample',data:dto}),LIVE_ANALYTICS_MINUTES:5},
|
'../../services/traffic-flow-analytics.js':{buildFlowAnalytics:async()=>dto,getFlowMonthly:async()=>({month:'2026-10',bytes:0,countries:[],services:[],asns:[]}),safeBuildLiveFlowSample:async()=>({event:'sample',data:dto}),LIVE_ANALYTICS_MINUTES:5},
|
||||||
'../../services/traffic-flow-map-hops.js':{buildFlowMapHops:async()=>({live:false,hops:[]})},
|
'../../services/traffic-flow-map-hops.js':{buildFlowMapHops:async()=>({live:false,hops:[]})},
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { z } from "zod"
|
|||||||
import { flowAnalyticsDtoSchema } from "@mmapp/contracts/traffic-flow"
|
import { flowAnalyticsDtoSchema } from "@mmapp/contracts/traffic-flow"
|
||||||
import { buildFlowAnalytics } from "../../services/traffic-flow-analytics.js"
|
import { buildFlowAnalytics } from "../../services/traffic-flow-analytics.js"
|
||||||
import { readFlowSourceLimits } from "../../services/traffic-flow-source-limits.js"
|
import { readFlowSourceLimits } from "../../services/traffic-flow-source-limits.js"
|
||||||
|
import { readFlowWindow } from "../../services/traffic-flow-read-window.js"
|
||||||
import { getBoundedFlowServerRowById as getServerRowById } from "../../services/traffic-flow-server-source.js"
|
import { getBoundedFlowServerRowById as getServerRowById } from "../../services/traffic-flow-server-source.js"
|
||||||
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
||||||
import { McpError } from "./security.js"
|
import { McpError } from "./security.js"
|
||||||
@@ -20,10 +21,10 @@ export function registerLegacyFlowAnalyticsAction(register: typeof add) {
|
|||||||
if (!server) throw new McpError("NOT_FOUND", "Сервер не найден", 404)
|
if (!server) throw new McpError("NOT_FOUND", "Сервер не найден", 404)
|
||||||
const authorize = () => assertTrafficHistorySource(context, server)
|
const authorize = () => assertTrafficHistorySource(context, server)
|
||||||
try {
|
try {
|
||||||
const result = await buildFlowAnalytics({ serverId: server.id, minutes: Number(input.minutes), iface: input.interface as string | undefined, skipHeavy: true },
|
const result = await readFlowWindow(Number(input.minutes), minutes => buildFlowAnalytics({ serverId: server.id, minutes, iface: input.interface as string | undefined, skipHeavy: true },
|
||||||
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize })
|
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize }), context.signal)
|
||||||
return {
|
return {
|
||||||
serverId: input.serverId, source: "PostgreSQL/flow_buckets", observedAt: new Date().toISOString(),
|
serverId: input.serverId, source: "PostgreSQL/flow_buckets", observedAt: new Date().toISOString(), sourceWindow: result.sourceWindow,
|
||||||
bpsNow: result.bpsNow, bytes: result.bytes, packets: result.packets, conversations: result.conversations,
|
bpsNow: result.bpsNow, bytes: result.bytes, packets: result.packets, conversations: result.conversations,
|
||||||
uniqueSrc: result.uniqueSrc, uniqueDst: result.uniqueDst, rxSeries: result.rxSeries, txSeries: result.txSeries,
|
uniqueSrc: result.uniqueSrc, uniqueDst: result.uniqueDst, rxSeries: result.rxSeries, txSeries: result.txSeries,
|
||||||
applications: result.applications, protocols: result.protocols, countries: result.countries,
|
applications: result.applications, protocols: result.protocols, countries: result.countries,
|
||||||
@@ -57,9 +58,10 @@ export function registerFlowAnalyticsActions(register: typeof add) {
|
|||||||
try {
|
try {
|
||||||
const limits = await readFlowSourceLimits()
|
const limits = await readFlowSourceLimits()
|
||||||
await authorize()
|
await authorize()
|
||||||
const result = flowAnalyticsDtoSchema.parse(await buildFlowAnalytics({ serverId: server.id, minutes: Number(input.minutes), userId: input.userId as string | undefined,
|
const { sourceWindow, ...raw } = await readFlowWindow(Number(input.minutes), minutes => buildFlowAnalytics({ serverId: server.id, minutes, userId: input.userId as string | undefined,
|
||||||
iface: input.interface as string | undefined, dedup: input.dedup === true, excludeMesh: input.excludeMesh === true, excludeOverlay: input.excludeOverlay === true },
|
iface: input.interface as string | undefined, dedup: input.dedup === true, excludeMesh: input.excludeMesh === true, excludeOverlay: input.excludeOverlay === true },
|
||||||
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize }))
|
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize }), context.signal)
|
||||||
|
const result = flowAnalyticsDtoSchema.parse(raw)
|
||||||
const offset = Number(input.offset), limit = Number(input.limit), pages: Record<string, { total: number; nextOffset: number | null }> = {}
|
const offset = Number(input.offset), limit = Number(input.limit), pages: Record<string, { total: number; nextOffset: number | null }> = {}
|
||||||
const analytics: Record<string, unknown> = { ...result }
|
const analytics: Record<string, unknown> = { ...result }
|
||||||
for (const [key, value] of Object.entries(result)) if (Array.isArray(value) && !["rxSeries", "txSeries"].includes(key)) {
|
for (const [key, value] of Object.entries(result)) if (Array.isArray(value) && !["rxSeries", "txSeries"].includes(key)) {
|
||||||
@@ -71,7 +73,7 @@ export function registerFlowAnalyticsActions(register: typeof add) {
|
|||||||
await authorize()
|
await authorize()
|
||||||
if ((await readFlowSourceLimits()).topN !== limits.topN) throw new McpError("STATE_CONFLICT", "Предел источника NetFlow изменился во время чтения", 409)
|
if ((await readFlowSourceLimits()).topN !== limits.topN) throw new McpError("STATE_CONFLICT", "Предел источника NetFlow изменился во время чтения", 409)
|
||||||
await authorize()
|
await authorize()
|
||||||
const output = { serverId: server.id, minutes: Number(input.minutes), interface: input.interface ?? null, userId: input.userId ?? null,
|
const output = { serverId: server.id, minutes: sourceWindow?.effectiveMinutes ?? Number(input.minutes), sourceWindow, interface: input.interface ?? null, userId: input.userId ?? null,
|
||||||
analytics: retained, pages, contextRates: { bpsNowEstimate: _bpsNow, bpsWireEstimate: _bpsWire, bpsOverlayEstimate: _bpsOverlay, bytesWireEstimate: _bytesWire,
|
analytics: retained, pages, contextRates: { bpsNowEstimate: _bpsNow, bpsWireEstimate: _bpsWire, bpsOverlayEstimate: _bpsOverlay, bytesWireEstimate: _bytesWire,
|
||||||
listenerBound: _live, userAttributionVerified: false, semantics: "REST estimates: current server/interface ring with retained-window fallback; wire projection uses current interface snapshots" },
|
listenerBound: _live, userAttributionVerified: false, semantics: "REST estimates: current server/interface ring with retained-window fallback; wire projection uses current interface snapshots" },
|
||||||
readStartedAt: startedAt, observedAt: new Date().toISOString(), live: false,
|
readStartedAt: startedAt, observedAt: new Date().toISOString(), live: false,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import { z } from "zod"
|
|||||||
import { readFlowExporterInputs, readFlowClientInputs, observeFlowExporter, observeFlowClient } from "../../services/traffic-flow-catalog-read.js"
|
import { readFlowExporterInputs, readFlowClientInputs, observeFlowExporter, observeFlowClient } from "../../services/traffic-flow-catalog-read.js"
|
||||||
import { getFlowMonthly } from "../../services/traffic-flow-analytics.js"
|
import { getFlowMonthly } from "../../services/traffic-flow-analytics.js"
|
||||||
import { readFlowCatalogWindow } from "../../services/traffic-flow-catalog-read.js"
|
import { readFlowCatalogWindow } from "../../services/traffic-flow-catalog-read.js"
|
||||||
|
import { readFlowWindow } from "../../services/traffic-flow-read-window.js"
|
||||||
import { projectFlowTalkers } from "../../services/traffic-flow-talkers-model.js"
|
import { projectFlowTalkers } from "../../services/traffic-flow-talkers-model.js"
|
||||||
import { resolveIfaceName } from "../../services/traffic-flow-ifindex.js"
|
import { resolveIfaceName } from "../../services/traffic-flow-ifindex.js"
|
||||||
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
||||||
@@ -21,9 +22,10 @@ export function registerFlowCatalogActions(register: typeof add) {
|
|||||||
await assertTrafficHistoryAccess(context)
|
await assertTrafficHistoryAccess(context)
|
||||||
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
||||||
try {
|
try {
|
||||||
const window = await readFlowCatalogWindow(Number(input.minutes), selected.map(row => row.id))
|
const requestedMinutes = Number(input.minutes)
|
||||||
|
const window = await readFlowWindow(requestedMinutes, minutes => readFlowCatalogWindow(minutes, selected.map(row => row.id)), context.signal)
|
||||||
const topN = window.topConversationsLimit ?? 20
|
const topN = window.topConversationsLimit ?? 20
|
||||||
const projection = projectFlowTalkers(window.rows, Number(input.minutes), topN, new Map(selected.map(row => [row.id, row.name])), resolveIfaceName)
|
const projection = projectFlowTalkers(window.rows, window.sourceWindow?.effectiveMinutes ?? requestedMinutes, topN, new Map(selected.map(row => [row.id, row.name])), resolveIfaceName)
|
||||||
const offset = Number(input.offset), limit = Number(input.limit), items = projection.talkers.slice(offset, offset + limit)
|
const offset = Number(input.offset), limit = Number(input.limit), items = projection.talkers.slice(offset, offset + limit)
|
||||||
for (const server of selected) await assertTrafficHistorySource(context, server)
|
for (const server of selected) await assertTrafficHistorySource(context, server)
|
||||||
if (selected.length && (await getTrafficFlowSettingsRow()).topN !== topN) throw new McpError("STATE_CONFLICT", "Предел источника NetFlow изменился во время чтения", 409)
|
if (selected.length && (await getTrafficFlowSettingsRow()).topN !== topN) throw new McpError("STATE_CONFLICT", "Предел источника NetFlow изменился во время чтения", 409)
|
||||||
@@ -33,7 +35,7 @@ export function registerFlowCatalogActions(register: typeof add) {
|
|||||||
retainedRowsBytes: projection.retainedRowsBytes, observedExporters: projection.observedExporters,
|
retainedRowsBytes: projection.retainedRowsBytes, observedExporters: projection.observedExporters,
|
||||||
uniqueSrc: projection.uniqueSrc, uniqueDst: projection.uniqueDst, topProto: projection.topProto,
|
uniqueSrc: projection.uniqueSrc, uniqueDst: projection.uniqueDst, topProto: projection.topProto,
|
||||||
topConversationsLimit: topN, topRowsPerBucketLimit: window.topRowsPerBucketLimit,
|
topConversationsLimit: topN, topRowsPerBucketLimit: window.topRowsPerBucketLimit,
|
||||||
incomplete: true, from: window.since, to: window.until,
|
incomplete: true, from: window.since, to: window.until, sourceWindow: window.sourceWindow,
|
||||||
source: "flow_buckets plus pending; scope before SQL and retained conversation aggregation",
|
source: "flow_buckets plus pending; scope before SQL and retained conversation aggregation",
|
||||||
bpsMeaning: "retained conversation bytes averaged over the entire requested window",
|
bpsMeaning: "retained conversation bytes averaged over the entire requested window",
|
||||||
...sourceSemantics },context)
|
...sourceSemantics },context)
|
||||||
@@ -47,27 +49,31 @@ export function registerFlowCatalogActions(register: typeof add) {
|
|||||||
register("traffic.flow.exporters.read", "mm_netflow_exporters_get", "NetFlow: экспортёры выбранной области", "traffic", "mm:traffic:read", "read", "servers", schema, {
|
register("traffic.flow.exporters.read", "mm_netflow_exporters_get", "NetFlow: экспортёры выбранной области", "traffic", "mm:traffic:read", "read", "servers", schema, {
|
||||||
read: async (input, context) => {
|
read: async (input, context) => {
|
||||||
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
||||||
const data = await readFlowExporterInputs(Number(input.minutes), selected.map(row => row.id))
|
const requestedMinutes = Number(input.minutes)
|
||||||
|
const data = await readFlowWindow(requestedMinutes, minutes => readFlowExporterInputs(minutes, selected.map(row => row.id)), context.signal)
|
||||||
const present = new Set([...data.window.rows.map(row => row.serverId), ...data.configuredIds])
|
const present = new Set([...data.window.rows.map(row => row.serverId), ...data.configuredIds])
|
||||||
const all = data.catalog.list.filter(row => present.has(row.id)).sort((a, b) => a.id - b.id)
|
const all = data.catalog.list.filter(row => present.has(row.id)).sort((a, b) => a.id - b.id)
|
||||||
const offset = Number(input.offset), limit = Number(input.limit)
|
const offset = Number(input.offset), limit = Number(input.limit)
|
||||||
return withCurrentFlowReadQuality({ ...page, exporters: all.slice(offset, offset + limit).map(row => observeFlowExporter(data, row.id, Number(input.minutes))), total: all.length,
|
for (const server of selected) await assertTrafficHistorySource(context, server)
|
||||||
nextOffset: offset + limit < all.length ? offset + limit : null, incomplete: page.incomplete || all.length > limit || offset > 0 || data.configurationMembershipVerified === false,
|
return withCurrentFlowReadQuality({ ...page, exporters: all.slice(offset, offset + limit).map(row => observeFlowExporter(data, row.id, data.sourceWindow?.effectiveMinutes ?? requestedMinutes)), total: all.length,
|
||||||
|
nextOffset: offset + limit < all.length ? offset + limit : null, incomplete: !!data.sourceWindow || page.incomplete || all.length > limit || offset > 0 || data.configurationMembershipVerified === false,
|
||||||
configurationMembershipVerified: data.configurationMembershipVerified, configurationEvidence: "stored-peer-records; device exporter settings and keys not verified", source: "flow_buckets plus pending; retained top rows per server/bucket",
|
configurationMembershipVerified: data.configurationMembershipVerified, configurationEvidence: "stored-peer-records; device exporter settings and keys not verified", source: "flow_buckets plus pending; retained top rows per server/bucket",
|
||||||
failures: data.configurationMembershipVerified === false ? [{ source: "selected-peer-membership", code: "UNVERIFIED_SCHEMA" }] : [],
|
failures: data.configurationMembershipVerified === false ? [{ source: "selected-peer-membership", code: "UNVERIFIED_SCHEMA" }] : [],
|
||||||
topRowsPerBucketLimit: data.window.topRowsPerBucketLimit, from: data.window.since, to: data.window.until, ...sourceSemantics },context)
|
topRowsPerBucketLimit: data.window.topRowsPerBucketLimit, from: data.window.since, to: data.window.until, sourceWindow: data.sourceWindow, ...sourceSemantics },context)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
register("traffic.flow.clients.read", "mm_netflow_clients_get", "NetFlow: клиенты по доступным привязкам", "traffic", "mm:traffic:read", "read", "servers", schema.safeExtend({ limit: z.number().int().min(1).max(50).default(20) }), {
|
register("traffic.flow.clients.read", "mm_netflow_clients_get", "NetFlow: клиенты по доступным привязкам", "traffic", "mm:traffic:read", "read", "servers", schema.safeExtend({ limit: z.number().int().min(1).max(50).default(20) }), {
|
||||||
read: async (input, context) => {
|
read: async (input, context) => {
|
||||||
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
const { selected, ...page } = await catalogDevices(input, context, { includeDisabled: true })
|
||||||
const offset = Number(input.offset), limit = Number(input.limit)
|
const offset = Number(input.offset), limit = Number(input.limit)
|
||||||
const data = await readFlowClientInputs(Number(input.minutes), { serverIds: selected.map(row => row.id), allowedServerIds: context.connection.policy.scope.serverIds, offset, limit })
|
const requestedMinutes = Number(input.minutes)
|
||||||
const clients = data.users.sort((a, b) => a.id.localeCompare(b.id)).map(user => observeFlowClient(user, data.window.rows, Number(input.minutes)))
|
const data = await readFlowWindow(requestedMinutes, minutes => readFlowClientInputs(minutes, { serverIds: selected.map(row => row.id), allowedServerIds: context.connection.policy.scope.serverIds, offset, limit }), context.signal)
|
||||||
|
const clients = data.users.sort((a, b) => a.id.localeCompare(b.id)).map(user => observeFlowClient(user, data.window.rows, data.sourceWindow?.effectiveMinutes ?? requestedMinutes))
|
||||||
|
for (const server of selected) await assertTrafficHistorySource(context, server)
|
||||||
return withCurrentFlowReadQuality({ ...page, clients, total: data.total, nextOffset: offset + clients.length < data.total ? offset + clients.length : null,
|
return withCurrentFlowReadQuality({ ...page, clients, total: data.total, nextOffset: offset + clients.length < data.total ? offset + clients.length : null,
|
||||||
incomplete: page.incomplete || clients.length < data.total || clients.some(row => !row.completeSourceSet || !row.completeAttribution),
|
incomplete: !!data.sourceWindow || page.incomplete || clients.length < data.total || clients.some(row => !row.completeSourceSet || !row.completeAttribution),
|
||||||
source: "retained ingress rows and current named whole-interface bindings", topRowsPerBucketLimit: data.window.topRowsPerBucketLimit,
|
source: "retained ingress rows and current named whole-interface bindings", topRowsPerBucketLimit: data.window.topRowsPerBucketLimit,
|
||||||
from: data.window.since, to: data.window.until, ...sourceSemantics },context)
|
from: data.window.since, to: data.window.until, sourceWindow: data.sourceWindow, ...sourceSemantics },context)
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
register("traffic.flow.monthly.read", "mm_netflow_monthly_report_get", "NetFlow: месячный отчёт выбранной области", "traffic", "mm:traffic:read", "read", "servers", catalogPaging.safeExtend({ month: z.string().regex(/^(?!0000)\d{4}-(0[1-9]|1[0-2])$/) }), {
|
register("traffic.flow.monthly.read", "mm_netflow_monthly_report_get", "NetFlow: месячный отчёт выбранной области", "traffic", "mm:traffic:read", "read", "servers", catalogPaging.safeExtend({ month: z.string().regex(/^(?!0000)\d{4}-(0[1-9]|1[0-2])$/) }), {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import { flowMapHopsDtoSchema } from "@mmapp/contracts/traffic-flow"
|
|||||||
import { isDeepStrictEqual } from "node:util"
|
import { isDeepStrictEqual } from "node:util"
|
||||||
import { buildFlowMapHops } from "../../services/traffic-flow-map-hops.js"
|
import { buildFlowMapHops } from "../../services/traffic-flow-map-hops.js"
|
||||||
import { readFlowMapSourceLimits } from "../../services/traffic-flow-source-limits.js"
|
import { readFlowMapSourceLimits } from "../../services/traffic-flow-source-limits.js"
|
||||||
|
import { readFlowWindow } from "../../services/traffic-flow-read-window.js"
|
||||||
import { getBoundedFlowServerRowById as getServerRowById } from "../../services/traffic-flow-server-source.js"
|
import { getBoundedFlowServerRowById as getServerRowById } from "../../services/traffic-flow-server-source.js"
|
||||||
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
import { assertTrafficHistoryAccess } from "./traffic-history-actions.js"
|
||||||
import { McpError } from "./security.js"
|
import { McpError } from "./security.js"
|
||||||
@@ -27,9 +28,10 @@ export function registerFlowMapActions(register: typeof add) {
|
|||||||
try {
|
try {
|
||||||
const limits = await readFlowMapSourceLimits()
|
const limits = await readFlowMapSourceLimits()
|
||||||
await authorize()
|
await authorize()
|
||||||
const result = flowMapHopsDtoSchema.parse(await buildFlowMapHops({ serverId: server.id, minutes: Number(input.minutes), userId: input.userId as string | undefined,
|
const { sourceWindow, ...raw } = await readFlowWindow(Number(input.minutes), minutes => buildFlowMapHops({ serverId: server.id, minutes, userId: input.userId as string | undefined,
|
||||||
iface: input.interface as string | undefined, dedup: input.dedup === true, excludeMesh: input.excludeMesh === true, excludeOverlay: input.excludeOverlay === true },
|
iface: input.interface as string | undefined, dedup: input.dedup === true, excludeMesh: input.excludeMesh === true, excludeOverlay: input.excludeOverlay === true },
|
||||||
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize }))
|
{ serverIds: context.connection.policy.scope.serverIds, maxRows: 20000, authorize }), context.signal)
|
||||||
|
const result = flowMapHopsDtoSchema.parse(raw)
|
||||||
const pages: Record<string, { total: number; nextOffset: number | null }> = {}
|
const pages: Record<string, { total: number; nextOffset: number | null }> = {}
|
||||||
const project = (value: unknown, path = "", depth = 0): unknown => {
|
const project = (value: unknown, path = "", depth = 0): unknown => {
|
||||||
if (Array.isArray(value)) {
|
if (Array.isArray(value)) {
|
||||||
@@ -45,7 +47,7 @@ export function registerFlowMapActions(register: typeof add) {
|
|||||||
await authorize()
|
await authorize()
|
||||||
if (!isDeepStrictEqual(limits, await readFlowMapSourceLimits())) throw new McpError("STATE_CONFLICT", "Пределы карты NetFlow изменились во время чтения", 409)
|
if (!isDeepStrictEqual(limits, await readFlowMapSourceLimits())) throw new McpError("STATE_CONFLICT", "Пределы карты NetFlow изменились во время чтения", 409)
|
||||||
await authorize()
|
await authorize()
|
||||||
const output = { serverId: server.id, userId: input.userId ?? null, interface: input.interface ?? null, map: projected, pages,
|
const output = { serverId: server.id, userId: input.userId ?? null, interface: input.interface ?? null, map: projected, pages, sourceWindow,
|
||||||
readStartedAt: startedAt, observedAt: new Date().toISOString(), live: false, listenerBound,
|
readStartedAt: startedAt, observedAt: new Date().toISOString(), live: false, listenerBound,
|
||||||
source: "retained flow_buckets plus pending; same REST map engine with scoped topology",
|
source: "retained flow_buckets plus pending; same REST map engine with scoped topology",
|
||||||
topRowsPerBucketLimit: Math.max(20, limits.topN), rowLimit: 20000, incomplete: true, graphComplete: false,
|
topRowsPerBucketLimit: Math.max(20, limits.topN), rowLimit: 20000, incomplete: true, graphComplete: false,
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"
|
|||||||
import { channel } from "node:diagnostics_channel"
|
import { channel } from "node:diagnostics_channel"
|
||||||
import { createFlowReadResponseOwner, writeBoundedFlowSse, withFlowReadSourceWindow } from "../services/traffic-flow-read-budget.js"
|
import { createFlowReadResponseOwner, writeBoundedFlowSse, withFlowReadSourceWindow } from "../services/traffic-flow-read-budget.js"
|
||||||
import { flowReadBudget, flowResponseReadLimits, FLOW_READ_RESERVATION_BYTES, FLOW_SSE_MAX_FRAME_BYTES } from "../services/traffic-flow-read-policy.js"
|
import { flowReadBudget, flowResponseReadLimits, FLOW_READ_RESERVATION_BYTES, FLOW_SSE_MAX_FRAME_BYTES } from "../services/traffic-flow-read-policy.js"
|
||||||
|
import { readFlowWindow } from "../services/traffic-flow-read-window.js"
|
||||||
import { updateTrafficFlowSettingsWithListener } from "../services/traffic-flow-settings-service.js"
|
import { updateTrafficFlowSettingsWithListener } from "../services/traffic-flow-settings-service.js"
|
||||||
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod"
|
||||||
import { mcpDatabaseRequestSignal, withDatabaseApiRequest } from "../modules/mcp/database-request-lease.js"
|
import { mcpDatabaseRequestSignal, withDatabaseApiRequest } from "../modules/mcp/database-request-lease.js"
|
||||||
@@ -98,7 +99,7 @@ function analyticsQuery(req: FastifyRequest) {
|
|||||||
|
|
||||||
async function sendFlowTalkers(req: FastifyRequest, reply: FastifyReply) {
|
async function sendFlowTalkers(req: FastifyRequest, reply: FastifyReply) {
|
||||||
const q = req.query as { range?: string }
|
const q = req.query as { range?: string }
|
||||||
return reply.send(await listFlowTalkers(rangeToMinutes(q.range)))
|
return reply.send(await readFlowWindow(rangeToMinutes(q.range), listFlowTalkers, mcpDatabaseRequestSignal()))
|
||||||
}
|
}
|
||||||
|
|
||||||
function requestPublicHost(req: FastifyRequest): string {
|
function requestPublicHost(req: FastifyRequest): string {
|
||||||
@@ -200,20 +201,22 @@ const trafficFlowRoutes: FastifyPluginAsyncZod = async (app) => {
|
|||||||
|
|
||||||
app.get("/traffic/flow/exporters", async (req, reply) => {
|
app.get("/traffic/flow/exporters", async (req, reply) => {
|
||||||
const q = req.query as { range?: string }
|
const q = req.query as { range?: string }
|
||||||
return reply.send(await listFlowExporters(rangeToMinutes(q.range)))
|
return reply.send(await readFlowWindow(rangeToMinutes(q.range), listFlowExporters, mcpDatabaseRequestSignal()))
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get("/traffic/flow/clients", async (req, reply) => {
|
app.get("/traffic/flow/clients", async (req, reply) => {
|
||||||
const q = req.query as { range?: string }
|
const q = req.query as { range?: string }
|
||||||
return reply.send(await listFlowClients(rangeToMinutes(q.range)))
|
return reply.send(await readFlowWindow(rangeToMinutes(q.range), listFlowClients, mcpDatabaseRequestSignal()))
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get("/traffic/flow/analytics", async (req, reply) => {
|
app.get("/traffic/flow/analytics", async (req, reply) => {
|
||||||
return reply.send(await buildFlowAnalytics(analyticsQuery(req)))
|
const query = analyticsQuery(req)
|
||||||
|
return reply.send(await readFlowWindow(query.minutes, minutes => buildFlowAnalytics({ ...query, minutes }), mcpDatabaseRequestSignal()))
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get("/traffic/flow/map-hops", async (req, reply) => {
|
app.get("/traffic/flow/map-hops", async (req, reply) => {
|
||||||
return reply.send(await buildFlowMapHops(analyticsQuery(req)))
|
const query = analyticsQuery(req)
|
||||||
|
return reply.send(await readFlowWindow(query.minutes, minutes => buildFlowMapHops({ ...query, minutes }), mcpDatabaseRequestSignal()))
|
||||||
})
|
})
|
||||||
|
|
||||||
app.get("/traffic/flow/monthly", async (req, reply) => {
|
app.get("/traffic/flow/monthly", async (req, reply) => {
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import assert from "node:assert/strict"
|
|||||||
import { rememberServerIfaces, resetIfaceCacheForTests } from "./traffic-flow-ifindex.js"
|
import { rememberServerIfaces, resetIfaceCacheForTests } from "./traffic-flow-ifindex.js"
|
||||||
import {
|
import {
|
||||||
ingestParsedFlowsForServerForTests,
|
ingestParsedFlowsForServerForTests,
|
||||||
|
getRingMbps,
|
||||||
resetFlowRingsForTests,
|
resetFlowRingsForTests,
|
||||||
} from "./traffic-flow-ingest.js"
|
} from "./traffic-flow-ingest.js"
|
||||||
import { buildFlowAnalytics, formatLiveSseFromBuilder, getFlowMonthly, listFlowClients, listFlowExporters, resetFlowAnalyticsCacheForTests } from "./traffic-flow-analytics.js"
|
import { buildFlowAnalytics, formatLiveSseFromBuilder, getFlowMonthly, listFlowClients, listFlowExporters, resetFlowAnalyticsCacheForTests } from "./traffic-flow-analytics.js"
|
||||||
@@ -98,6 +99,9 @@ ingestParsedFlowsForServerForTests(serverId, [
|
|||||||
])
|
])
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
const ring = getRingMbps(serverId)
|
||||||
|
assert.equal(ring.rxNow,0)
|
||||||
|
assert.equal(ring.txNow,12800 * 8 / 2 / 1e6)
|
||||||
const all = await buildFlowAnalytics({ minutes: 5, serverId })
|
const all = await buildFlowAnalytics({ minutes: 5, serverId })
|
||||||
assert.equal(all.applications[0]?.label, "HTTPS")
|
assert.equal(all.applications[0]?.label, "HTTPS")
|
||||||
assert.ok(all.protocols.some((p) => p.label === "TCP"))
|
assert.ok(all.protocols.some((p) => p.label === "TCP"))
|
||||||
@@ -119,6 +123,14 @@ try {
|
|||||||
|
|
||||||
const other = await buildFlowAnalytics({ minutes: 5, serverId: 99 })
|
const other = await buildFlowAnalytics({ minutes: 5, serverId: 99 })
|
||||||
assert.equal(other.conversations, 0)
|
assert.equal(other.conversations, 0)
|
||||||
|
|
||||||
|
ingestParsedFlowsForServerForTests(serverId, [
|
||||||
|
{ src:"10.1.1.9", dst:"9.9.9.9", proto:6, srcPort:50001, dstPort:443, inIface:"2", bytes:1200, packets:2 },
|
||||||
|
{ src:"8.8.4.4", dst:"9.9.9.9", proto:6, srcPort:50002, dstPort:443, inIface:"2", bytes:3400, packets:4 },
|
||||||
|
])
|
||||||
|
const mixed = await buildFlowAnalytics({ minutes:5, serverId })
|
||||||
|
assert.equal(mixed.bytes, 12800 + 1200 + 3400)
|
||||||
|
assert.equal(mixed.bpsNow, mixed.bytes * 8 / 300, "Unattributed transit remains in the total rate")
|
||||||
} finally {
|
} finally {
|
||||||
resetFlowRingsForTests()
|
resetFlowRingsForTests()
|
||||||
resetIfaceCacheForTests()
|
resetIfaceCacheForTests()
|
||||||
@@ -561,16 +573,21 @@ resetFlowRingsForTests()
|
|||||||
await dbQuery("DELETE FROM flow_buckets WHERE server_id=$1", [serverId])
|
await dbQuery("DELETE FROM flow_buckets WHERE server_id=$1", [serverId])
|
||||||
for (const minutesAgo of [2, 3, 4, 10]) {
|
for (const minutesAgo of [2, 3, 4, 10]) {
|
||||||
await dbQuery(`INSERT INTO flow_buckets (server_id,bucket_at,src,dst,in_iface,bytes,packets)
|
await dbQuery(`INSERT INTO flow_buckets (server_id,bucket_at,src,dst,in_iface,bytes,packets)
|
||||||
VALUES ($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'10.1.1.8','8.8.8.8','2',60000,10)`, [serverId, minutesAgo])
|
VALUES ($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'10.1.1.8','8.8.8.8','2',60000,10),
|
||||||
|
($1,date_trunc('minute',now())-($2::int*interval '1 minute'),'8.8.8.8','10.1.1.8','11',240000,40)`, [serverId, minutesAgo])
|
||||||
}
|
}
|
||||||
for (const minutes of [5, 15]) for (const dedup of [true, false]) {
|
for (const minutes of [5, 15]) for (const dedup of [true, false]) {
|
||||||
resetFlowAnalyticsCacheForTests()
|
resetFlowAnalyticsCacheForTests()
|
||||||
const history = await buildFlowAnalytics({ minutes, serverId, dedup, excludeMesh:false, excludeOverlay:false })
|
const history = await buildFlowAnalytics({ minutes, serverId, dedup, excludeMesh:false, excludeOverlay:false })
|
||||||
assert.equal(history.rxSeries.length, minutes)
|
assert.equal(history.rxSeries.length, minutes)
|
||||||
assert.equal(history.rxSeries.filter(v => v > 0).length, minutes === 5 ? 3 : 4)
|
assert.equal(history.rxSeries.filter(v => v > 0).length, minutes === 5 ? 3 : 4)
|
||||||
|
assert.deepEqual(history.rxSeries,history.txSeries.map(v => v * 4))
|
||||||
const missing = await buildFlowAnalytics({ minutes, serverId, iface:"not-an-interface" })
|
const missing = await buildFlowAnalytics({ minutes, serverId, iface:"not-an-interface" })
|
||||||
assert(missing.rxSeries.every(v => v === 0), "Historical chart respects interface filter")
|
assert(missing.rxSeries.every(v => v === 0), "Historical chart respects interface filter")
|
||||||
}
|
}
|
||||||
|
const historicalCard = (await listFlowExporters(5)).exporters.find(s => s.id === String(serverId))!
|
||||||
|
assert(historicalCard.txNow > 0)
|
||||||
|
assert.equal(historicalCard.rxNow,historicalCard.txNow * 4)
|
||||||
|
|
||||||
} finally {
|
} finally {
|
||||||
for (const table of ["flow_daily_dims", "flow_hour_facts", "flow_daily_facts", "flow_buckets", "traffic_samples", "user_interface_bindings", "server_snapshots"]) {
|
for (const table of ["flow_daily_dims", "flow_hour_facts", "flow_daily_facts", "flow_buckets", "traffic_samples", "user_interface_bindings", "server_snapshots"]) {
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { flowReadDbAll } from "./traffic-flow-read-policy.js"
|
import { flowReadDbAll } from "./traffic-flow-read-policy.js"
|
||||||
|
import { readFlowWindow } from "./traffic-flow-read-window.js"
|
||||||
import { isDeepStrictEqual } from "node:util"
|
import { isDeepStrictEqual } from "node:util"
|
||||||
import { readFlowSourceLimits } from "./traffic-flow-source-limits.js"
|
import { readFlowSourceLimits } from "./traffic-flow-source-limits.js"
|
||||||
import { readFlowExporterInputs, readFlowClientInputs, flowCounter, flowCounterSum } from "./traffic-flow-catalog-read.js"
|
import { readFlowExporterInputs, readFlowClientInputs, flowCounter, flowCounterSum } from "./traffic-flow-catalog-read.js"
|
||||||
@@ -28,7 +29,7 @@ import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js"
|
|||||||
import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js"
|
import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js"
|
||||||
import { enqueueRipeMisses } from "./traffic-flow-ripe.js"
|
import { enqueueRipeMisses } from "./traffic-flow-ripe.js"
|
||||||
import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js"
|
import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js"
|
||||||
import { resolveInternetDest } from "./traffic-flow-dest.js"
|
import { flowRateDirection, resolveInternetDest } from "./traffic-flow-dest.js"
|
||||||
import { refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
|
import { refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
|
||||||
import {
|
import {
|
||||||
enGreIfaceNames,
|
enGreIfaceNames,
|
||||||
@@ -105,22 +106,29 @@ async function userIfaceAllow(userId: string): Promise<Map<number, Set<string>>
|
|||||||
return allow
|
return allow
|
||||||
}
|
}
|
||||||
|
|
||||||
function seriesFromRows(rows: PendingFlowRow[], minutes: number): { rx: number[]; tx: number[] } {
|
function seriesFromRows(rows: PendingFlowRow[], minutes: number, direction: (row: PendingFlowRow) => "rx" | "tx" | "both" | null) {
|
||||||
const slots = Math.min(60, Math.max(5, minutes))
|
const slots = Math.min(60, Math.max(5, minutes))
|
||||||
const slotMs = (minutes * 60_000) / slots
|
const slotMs = (minutes * 60_000) / slots
|
||||||
const start = Date.now() - minutes * 60_000
|
const start = Date.now() - minutes * 60_000
|
||||||
const rx = Array(slots).fill(0) as number[]
|
const rx = Array(slots).fill(0) as number[]
|
||||||
const tx = Array(slots).fill(0) as number[]
|
const tx = Array(slots).fill(0) as number[]
|
||||||
|
let undirectedBytes = 0
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
const t = Date.parse(r.bucketAt)
|
const t = Date.parse(r.bucketAt)
|
||||||
if (!Number.isFinite(t)) continue
|
if (!Number.isFinite(t)) continue
|
||||||
const idx = Math.min(slots - 1, Math.max(0, Math.floor((t - start) / slotMs)))
|
const idx = Math.min(slots - 1, Math.max(0, Math.floor((t - start) / slotMs)))
|
||||||
rx[idx] += r.bytes
|
const side = direction(r)
|
||||||
|
if (side === null) undirectedBytes += r.bytes
|
||||||
|
if (side === "rx" || side === "both") rx[idx] += r.bytes
|
||||||
|
if (side === "tx" || side === "both") tx[idx] += r.bytes
|
||||||
}
|
}
|
||||||
const slotSec = Math.max(1, slotMs / 1000)
|
const slotSec = Math.max(1, slotMs / 1000)
|
||||||
return {
|
return {
|
||||||
rx: rx.map((b) => bpsToMbps((b * 8) / slotSec)),
|
rx: rx.map((b) => bpsToMbps((b * 8) / slotSec)),
|
||||||
tx,
|
tx: tx.map((b) => bpsToMbps((b * 8) / slotSec)),
|
||||||
|
rxNow: bpsToMbps(rx.reduce((a, b) => a + b, 0) * 8 / (minutes * 60)),
|
||||||
|
txNow: bpsToMbps(tx.reduce((a, b) => a + b, 0) * 8 / (minutes * 60)),
|
||||||
|
undirectedBytes,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -232,7 +240,8 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF
|
|||||||
for (const r of raw) {
|
for (const r of raw) {
|
||||||
const resolved = resolveIfaceName(r.serverId, r.inIface)
|
const resolved = resolveIfaceName(r.serverId, r.inIface)
|
||||||
const outResolved = resolveIfaceName(r.serverId, r.outIface)
|
const outResolved = resolveIfaceName(r.serverId, r.outIface)
|
||||||
if (!flowRowMatchesFilter(r, resolved.name, q, allow)) continue
|
if (!flowRowMatchesFilter(r, resolved.name, q, allow)
|
||||||
|
&& !flowRowMatchesFilter({ serverId: r.serverId, inIface: r.outIface }, outResolved.name, q, allow)) continue
|
||||||
ifacesForWire.add(resolved.name)
|
ifacesForWire.add(resolved.name)
|
||||||
if (outResolved.name && outResolved.name !== "—") ifacesForWire.add(outResolved.name)
|
if (outResolved.name && outResolved.name !== "—") ifacesForWire.add(outResolved.name)
|
||||||
const plane = classifyFlowPlane({
|
const plane = classifyFlowPlane({
|
||||||
@@ -440,7 +449,19 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF
|
|||||||
: { rx: Array(60).fill(0) as number[], tx: Array(60).fill(0) as number[], rxNow: 0, txNow: 0 }
|
: { rx: Array(60).fill(0) as number[], tx: Array(60).fill(0) as number[], rxNow: 0, txNow: 0 }
|
||||||
|
|
||||||
// The live ring covers about one minute; historical charts must use the requested window.
|
// The live ring covers about one minute; historical charts must use the requested window.
|
||||||
const fromBuckets = seriesFromRows(matched, q.minutes)
|
// Keep the winning interface's individual minute buckets when deduplicating.
|
||||||
|
const chosenIfaces = wantDedup ? new Map(working.map(row => [flowTupleKey(row), row.inIface])) : null
|
||||||
|
const chartRows = chosenIfaces ? matched.filter(r => chosenIfaces.get(flowTupleKey(r)) === r.inIface) : matched
|
||||||
|
const fromBuckets = seriesFromRows(chartRows, q.minutes, row => {
|
||||||
|
if (ifaceFilter || allow) {
|
||||||
|
const ingress = flowRowMatchesFilter(row, resolveIfaceName(row.serverId, row.inIface).name, q, allow)
|
||||||
|
const egress = flowRowMatchesFilter({ serverId: row.serverId, inIface: row.outIface }, resolveIfaceName(row.serverId, row.outIface).name, q, allow)
|
||||||
|
if (ingress && egress) return "both"
|
||||||
|
// At a client binding, ingress is the client's upload.
|
||||||
|
return allow && !ifaceFilter ? (ingress ? "tx" : "rx") : (ingress ? "rx" : "tx")
|
||||||
|
}
|
||||||
|
return flowRateDirection(row.serverId, row, topo, catalog.byId.get(row.serverId)?.host)
|
||||||
|
})
|
||||||
const rxSeries = fromBuckets.rx
|
const rxSeries = fromBuckets.rx
|
||||||
const txSeries = fromBuckets.tx
|
const txSeries = fromBuckets.tx
|
||||||
|
|
||||||
@@ -490,7 +511,7 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF
|
|||||||
if (scope && (!isDeepStrictEqual(topo, await loadFlowTopology(scope.serverIds, authorize)) || (await readFlowSourceLimits()).topN !== settings.topN)) throw new Error("Топология или предел источника NetFlow изменились во время чтения")
|
if (scope && (!isDeepStrictEqual(topo, await loadFlowTopology(scope.serverIds, authorize)) || (await readFlowSourceLimits()).topN !== settings.topN)) throw new Error("Топология или предел источника NetFlow изменились во время чтения")
|
||||||
await authorize()
|
await authorize()
|
||||||
return {
|
return {
|
||||||
bpsNow: (ring.rxNow + ring.txNow) * 1_000_000 || (totalBytes * 8) / windowSec,
|
bpsNow: (!fromBuckets.undirectedBytes && (ring.rxNow + ring.txNow) * 1_000_000) || (totalBytes * 8) / windowSec,
|
||||||
bytes: totalBytes,
|
bytes: totalBytes,
|
||||||
packets: totalPackets,
|
packets: totalPackets,
|
||||||
conversations: conv.size,
|
conversations: conv.size,
|
||||||
@@ -535,30 +556,32 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery, scope?: ScopedF
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function summarizeByServer(rows: PendingFlowRow[]) {
|
function groupRowsByServer(rows: PendingFlowRow[]) {
|
||||||
const bytes = new Map<number, number>()
|
const byServer = new Map<number, PendingFlowRow[]>()
|
||||||
const sessions = new Map<number, number>()
|
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
bytes.set(r.serverId, (bytes.get(r.serverId) ?? 0) + r.bytes)
|
const group = byServer.get(r.serverId)
|
||||||
sessions.set(r.serverId, (sessions.get(r.serverId) ?? 0) + 1)
|
if (group) group.push(r)
|
||||||
|
else byServer.set(r.serverId, [r])
|
||||||
}
|
}
|
||||||
return { bytes, sessions }
|
return byServer
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listFlowExporters(minutes: number): Promise<FlowExportersDto> {
|
export async function listFlowExporters(minutes: number): Promise<FlowExportersDto> {
|
||||||
const runtime = await getFlowRuntimeCounters()
|
const runtime = await getFlowRuntimeCounters()
|
||||||
const inputs = await readFlowExporterInputs(minutes)
|
const inputs = await readFlowExporterInputs(minutes)
|
||||||
const rows = inputs.window.rows
|
const rows = inputs.window.rows
|
||||||
const { bytes, sessions } = summarizeByServer(rows)
|
const byServer = groupRowsByServer(rows)
|
||||||
const ids = new Set<number>([...bytes.keys()])
|
const ids = new Set<number>(byServer.keys())
|
||||||
for (const id of inputs.configuredIds) ids.add(id)
|
for (const id of inputs.configuredIds) ids.add(id)
|
||||||
const catalog = inputs.catalog
|
const catalog = inputs.catalog
|
||||||
const emptySeries = Array(60).fill(0) as number[]
|
const topo = await loadFlowTopology()
|
||||||
const exporters = catalog.list
|
const exporters = catalog.list
|
||||||
.filter((s) => ids.has(s.id))
|
.filter((s) => ids.has(s.id))
|
||||||
.map((s) => {
|
.map((s) => {
|
||||||
const ring = getRingMbps(s.id, "__all__")
|
const ring = getRingMbps(s.id, "__all__")
|
||||||
const total = bytes.get(s.id) ?? 0
|
const records = byServer.get(s.id) ?? []
|
||||||
|
const rates = ring.rx.some(Boolean) || ring.tx.some(Boolean) ? ring
|
||||||
|
: seriesFromRows(records, minutes, row => flowRateDirection(s.id, row, topo, s.host))
|
||||||
return {
|
return {
|
||||||
id: String(s.id),
|
id: String(s.id),
|
||||||
name: s.name,
|
name: s.name,
|
||||||
@@ -566,12 +589,12 @@ export async function listFlowExporters(minutes: number): Promise<FlowExportersD
|
|||||||
site: s.site,
|
site: s.site,
|
||||||
country: s.country,
|
country: s.country,
|
||||||
status: snapshotStatus(s.id),
|
status: snapshotStatus(s.id),
|
||||||
rxNow: ring.rxNow || (total * 8) / Math.max(60, minutes * 60) / 1_000_000,
|
rxNow: rates.rxNow,
|
||||||
txNow: ring.txNow,
|
txNow: rates.txNow,
|
||||||
sessions: sessions.get(s.id) ?? 0,
|
sessions: records.length,
|
||||||
rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : emptySeries,
|
rxSeries: rates.rx,
|
||||||
txSeries: ring.tx,
|
txSeries: rates.tx,
|
||||||
bytes: total,
|
bytes: records.reduce((sum, row) => sum + row.bytes, 0),
|
||||||
} satisfies FlowEntityCard
|
} satisfies FlowEntityCard
|
||||||
})
|
})
|
||||||
.sort((a, b) => b.rxNow - a.rxNow)
|
.sort((a, b) => b.rxNow - a.rxNow)
|
||||||
@@ -598,8 +621,6 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id:
|
|||||||
byUser.set(b.userId, list)
|
byUser.set(b.userId, list)
|
||||||
}
|
}
|
||||||
const rows = inputs.window.rows
|
const rows = inputs.window.rows
|
||||||
const emptySeries = Array(60).fill(0) as number[]
|
|
||||||
const windowSec = Math.max(60, minutes * 60)
|
|
||||||
const clients: FlowEntityCard[] = []
|
const clients: FlowEntityCard[] = []
|
||||||
for (const u of users) {
|
for (const u of users) {
|
||||||
const userBinds = byUser.get(u.id) ?? []
|
const userBinds = byUser.get(u.id) ?? []
|
||||||
@@ -610,18 +631,15 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id:
|
|||||||
set.add(b.interfaceName)
|
set.add(b.interfaceName)
|
||||||
allow.set(b.serverId, set)
|
allow.set(b.serverId, set)
|
||||||
}
|
}
|
||||||
let total = 0
|
const bound = (id: number, iface: string) => {
|
||||||
let sessions = 0
|
const names = allow.get(id)
|
||||||
for (const r of rows) {
|
return names?.has(iface) || names?.has(resolveIfaceName(id, iface).name)
|
||||||
const resolved = resolveIfaceName(r.serverId, r.inIface)
|
|
||||||
const names = allow.get(r.serverId)
|
|
||||||
if (!names) continue
|
|
||||||
if (!names.has(resolved.name) && !names.has(r.inIface)) continue
|
|
||||||
total += r.bytes
|
|
||||||
sessions += 1
|
|
||||||
}
|
}
|
||||||
const firstServer = userBinds[0]?.serverId
|
const matched = rows.filter(r => bound(r.serverId, r.inIface) || bound(r.serverId, r.outIface))
|
||||||
const ring = firstServer ? getRingMbps(firstServer, "__all__") : { rx: emptySeries, tx: emptySeries, rxNow: 0, txNow: 0 }
|
const rates = seriesFromRows(matched, minutes, r => {
|
||||||
|
if (!bound(r.serverId, r.inIface)) return "rx"
|
||||||
|
return bound(r.serverId, r.outIface) ? "both" : "tx"
|
||||||
|
})
|
||||||
clients.push({
|
clients.push({
|
||||||
id: u.id,
|
id: u.id,
|
||||||
name: u.login,
|
name: u.login,
|
||||||
@@ -629,12 +647,12 @@ export async function listFlowClients(minutes: number): Promise<{ clients: { id:
|
|||||||
site: `${userBinds.length} ifaces`,
|
site: `${userBinds.length} ifaces`,
|
||||||
country: "UN",
|
country: "UN",
|
||||||
status: u.active ? "online" : "offline",
|
status: u.active ? "online" : "offline",
|
||||||
rxNow: (total * 8) / windowSec / 1_000_000 || ring.rxNow,
|
rxNow: rates.rxNow,
|
||||||
txNow: ring.txNow,
|
txNow: rates.txNow,
|
||||||
sessions,
|
sessions: matched.length,
|
||||||
rxSeries: ring.rx.some((v) => v > 0) ? ring.rx : emptySeries,
|
rxSeries: rates.rx,
|
||||||
txSeries: ring.tx,
|
txSeries: rates.tx,
|
||||||
bytes: total,
|
bytes: matched.reduce((sum, row) => sum + row.bytes, 0),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
clients.sort((a, b) => b.rxNow - a.rxNow)
|
clients.sort((a, b) => b.rxNow - a.rxNow)
|
||||||
@@ -664,7 +682,7 @@ export async function safeBuildLiveFlowSample(q: Omit<FlowAnalyticsQuery, "minut
|
|||||||
}> {
|
}> {
|
||||||
return formatLiveSseFromBuilder(async () => {
|
return formatLiveSseFromBuilder(async () => {
|
||||||
const skipHeavy = isFlowAnalyticsDegraded()
|
const skipHeavy = isFlowAnalyticsDegraded()
|
||||||
return await buildFlowAnalytics({ ...q, minutes: LIVE_ANALYTICS_MINUTES, skipHeavy }, scope)
|
return readFlowWindow(LIVE_ANALYTICS_MINUTES, minutes => buildFlowAnalytics({ ...q, minutes, skipHeavy }, scope))
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { getFlowReadResourceProfile } from "./traffic-flow-read-resource-profile.js"
|
import { getFlowReadResourceProfile } from "./traffic-flow-read-resource-profile.js"
|
||||||
import type { QueryResultRow } from "pg"
|
import type { QueryResultRow } from "pg"
|
||||||
import { trackFlowReadSource } from "./traffic-flow-read-budget.js"
|
import { trackFlowReadSource } from "./traffic-flow-read-budget.js"
|
||||||
|
import { FlowSqlSourceTooLarge } from "./traffic-flow-read-window.js"
|
||||||
|
|
||||||
export interface FlowBoundedQueryClient {
|
export interface FlowBoundedQueryClient {
|
||||||
query(sql: string, values: unknown[]): Promise<{ rows: Array<Record<string, unknown>>; fields?:Array<{name:string}> }>
|
query(sql: string, values: unknown[]): Promise<{ rows: Array<Record<string, unknown>>; fields?:Array<{name:string}> }>
|
||||||
@@ -24,7 +25,8 @@ export async function readBoundedFlowSql<T extends QueryResultRow>(client: FlowB
|
|||||||
const preflight=await client.query(`WITH flow_read_source AS MATERIALIZED (${query}) SELECT count(*)::text AS "__flowRowCount",COALESCE(sum(octet_length(row_to_json(s)::text)),0)::text AS "__flowEncodedBytes" FROM flow_read_source s`,values)
|
const preflight=await client.query(`WITH flow_read_source AS MATERIALIZED (${query}) SELECT count(*)::text AS "__flowRowCount",COALESCE(sum(octet_length(row_to_json(s)::text)),0)::text AS "__flowEncodedBytes" FROM flow_read_source s`,values)
|
||||||
const n=String(preflight.rows[0]?.__flowRowCount),b=String(preflight.rows[0]?.__flowEncodedBytes)
|
const n=String(preflight.rows[0]?.__flowRowCount),b=String(preflight.rows[0]?.__flowEncodedBytes)
|
||||||
const measuredBytes=Number(b)+2+Math.max(Number(n)-1,0)
|
const measuredBytes=Number(b)+2+Math.max(Number(n)-1,0)
|
||||||
if(!/^\d{1,15}$/.test(n)||!/^\d{1,15}$/.test(b)||Number(n)>limits.maxRows||measuredBytes>sourceByteLimit)throw Object.assign(Error("Источник NetFlow превышает допустимый объём; сократите область или окно"),{statusCode:409,code:"STATE_UNAVAILABLE"})
|
if(!/^\d{1,15}$/.test(n)||!/^\d{1,15}$/.test(b))throw Error("Invalid bounded NetFlow source size")
|
||||||
|
if(Number(n)>limits.maxRows||measuredBytes>sourceByteLimit)throw new FlowSqlSourceTooLarge()
|
||||||
const reserved=admit(Number(n),measuredBytes)
|
const reserved=admit(Number(n),measuredBytes)
|
||||||
const countIndex = values.length + 1, bytesIndex = values.length + 2
|
const countIndex = values.length + 1, bytesIndex = values.length + 2
|
||||||
const result = await client.query(`WITH flow_read_source AS MATERIALIZED (${query}),
|
const result = await client.query(`WITH flow_read_source AS MATERIALIZED (${query}),
|
||||||
@@ -36,7 +38,8 @@ export async function readBoundedFlowSql<T extends QueryResultRow>(client: FlowB
|
|||||||
if(result.fields && ["__flowRowCount","__flowEncodedBytes","__flowPresent"].some(name=>result.fields!.filter(field=>field.name===name).length!==1))throw Error("Reserved NetFlow source column collision")
|
if(result.fields && ["__flowRowCount","__flowEncodedBytes","__flowPresent"].some(name=>result.fields!.filter(field=>field.name===name).length!==1))throw Error("Reserved NetFlow source column collision")
|
||||||
const row = result.rows[0]
|
const row = result.rows[0]
|
||||||
const count = String(row?.__flowRowCount), bytes = String(row?.__flowEncodedBytes)
|
const count = String(row?.__flowRowCount), bytes = String(row?.__flowEncodedBytes)
|
||||||
if (!row || !/^\d{1,15}$/.test(count) || !/^\d{1,15}$/.test(bytes) || Number(count) > reserved.maxRows || Number(bytes) + 2 + Math.max(Number(count)-1,0) > reserved.maxEncodedBytes) throw Object.assign(Error("Источник NetFlow превышает допустимый объём; сократите область или окно"), { statusCode: 409, code: "STATE_UNAVAILABLE" })
|
if (!row || !/^\d{1,15}$/.test(count) || !/^\d{1,15}$/.test(bytes)) throw Error("Invalid bounded NetFlow source size")
|
||||||
|
if (Number(count) > reserved.maxRows || Number(bytes) + 2 + Math.max(Number(count)-1,0) > reserved.maxEncodedBytes) throw new FlowSqlSourceTooLarge()
|
||||||
if (Number(count) === 0) return []
|
if (Number(count) === 0) return []
|
||||||
if (result.rows.length !== Number(count) || result.rows.some(item => item.__flowPresent !== true)) throw Error("Invalid bounded NetFlow SQL result")
|
if (result.rows.length !== Number(count) || result.rows.some(item => item.__flowPresent !== true)) throw Error("Invalid bounded NetFlow SQL result")
|
||||||
// Preserve the normal pg column parsers (not JSONB numeric/timestamp coercion).
|
// Preserve the normal pg column parsers (not JSONB numeric/timestamp coercion).
|
||||||
|
|||||||
@@ -18,9 +18,7 @@ export async function readFlowCatalogWindow(minutes: number, serverIds?: readonl
|
|||||||
if (selected?.length === 0) return { rows: [] as PendingFlowRow[], since, until, topRowsPerBucketLimit: null, topConversationsLimit: null }
|
if (selected?.length === 0) return { rows: [] as PendingFlowRow[], since, until, topRowsPerBucketLimit: null, topConversationsLimit: null }
|
||||||
const settings = await getTrafficFlowSettingsRow()
|
const settings = await getTrafficFlowSettingsRow()
|
||||||
if (!Number.isSafeInteger(settings.topN) || settings.topN < 1 || settings.topN > 1000) throw fail("Не подтверждён topN источника NetFlow")
|
if (!Number.isSafeInteger(settings.topN) || settings.topN < 1 || settings.topN > 1000) throw fail("Не подтверждён topN источника NetFlow")
|
||||||
const rows:PendingFlowRow[]=[]
|
const rows = await listStoredFlowRows(since, undefined, { serverIds: selected, maxRows: 49999, untilIso: until })
|
||||||
if(selected===undefined)rows.push(...await listStoredFlowRows(since,undefined,{untilIso:until}))
|
|
||||||
else for(const serverId of selected)rows.push(...await listStoredFlowRows(since,serverId,{maxRows:Math.max(1,Math.floor(50000/selected.length)),untilIso:until}))
|
|
||||||
for (const row of rows) { flowCounter(row.bytes); flowCounter(row.packets) }
|
for (const row of rows) { flowCounter(row.bytes); flowCounter(row.packets) }
|
||||||
return { rows, since, until, topRowsPerBucketLimit: Math.max(20, settings.topN), topConversationsLimit: settings.topN }
|
return { rows, since, until, topRowsPerBucketLimit: Math.max(20, settings.topN), topConversationsLimit: settings.topN }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import {
|
|||||||
resetEngineForTests,
|
resetEngineForTests,
|
||||||
} from "./traffic-flow-engine.js"
|
} from "./traffic-flow-engine.js"
|
||||||
import { factsSnapshotForTests } from "./traffic-flow-facts.js"
|
import { factsSnapshotForTests } from "./traffic-flow-facts.js"
|
||||||
import { classifyInternetBrand, mapInternetBrand, resolveInternetDest } from "./traffic-flow-dest.js"
|
import { classifyInternetBrand, flowRateDirection, mapInternetBrand, resolveInternetDest } from "./traffic-flow-dest.js"
|
||||||
import { disableCatalogFetchForTests, resetFlowCatalogForTests } from "./traffic-flow-classify.js"
|
import { disableCatalogFetchForTests, resetFlowCatalogForTests } from "./traffic-flow-classify.js"
|
||||||
import {
|
import {
|
||||||
disableRipeEnqueueForTests,
|
disableRipeEnqueueForTests,
|
||||||
@@ -66,6 +66,11 @@ const topo: FlowTopology = {
|
|||||||
seedFlowTopologyForTests(topo)
|
seedFlowTopologyForTests(topo)
|
||||||
rememberServerIfaces(1, [{ name: "gre-client", ifindex: "2" }])
|
rememberServerIfaces(1, [{ name: "gre-client", ifindex: "2" }])
|
||||||
|
|
||||||
|
const publicFlow = { src: "95.167.1.10", dst: "8.8.8.8", inIface: "2", outIface: "ether1" }
|
||||||
|
assert.equal(flowRateDirection(1, publicFlow, topo), "tx")
|
||||||
|
assert.equal(flowRateDirection(1, { ...publicFlow, inIface: "ether1", outIface: "2" }, topo), "rx")
|
||||||
|
assert.equal(flowRateDirection(1, publicFlow, null), null, "Public transit has no inferred download or upload")
|
||||||
|
|
||||||
ingestParsedFlowsForServerForTests(1, [
|
ingestParsedFlowsForServerForTests(1, [
|
||||||
{
|
{
|
||||||
src: "95.167.1.10",
|
src: "95.167.1.10",
|
||||||
|
|||||||
@@ -4,6 +4,9 @@ import { classifyFlowDst, type FlowClassification } from "./traffic-flow-classif
|
|||||||
import { resolveFlowIp } from "./traffic-flow-geoip.js"
|
import { resolveFlowIp } from "./traffic-flow-geoip.js"
|
||||||
import { canonicalFactIface } from "./traffic-flow-ifindex.js"
|
import { canonicalFactIface } from "./traffic-flow-ifindex.js"
|
||||||
import {
|
import {
|
||||||
|
canonicalIp,
|
||||||
|
isNonPublicIp,
|
||||||
|
isUnspecifiedIp,
|
||||||
resolveFlowEndpoints,
|
resolveFlowEndpoints,
|
||||||
type FlowEndpoints,
|
type FlowEndpoints,
|
||||||
type InternetDestCtx,
|
type InternetDestCtx,
|
||||||
@@ -15,6 +18,28 @@ import {
|
|||||||
type FlowTopology,
|
type FlowTopology,
|
||||||
} from "./traffic-flow-topology.js"
|
} from "./traffic-flow-topology.js"
|
||||||
|
|
||||||
|
/** Aggregate DL/UL is relative to the exporter's WAN or its bound clients. */
|
||||||
|
export function flowRateDirection(
|
||||||
|
serverId: number,
|
||||||
|
row: { src: string; dst: string; inIface: string; outIface: string },
|
||||||
|
topo: FlowTopology | null,
|
||||||
|
exporterHost?: string,
|
||||||
|
): "rx" | "tx" | null {
|
||||||
|
const ingress = canonicalFactIface(serverId, row.inIface)
|
||||||
|
const egress = canonicalFactIface(serverId, row.outIface)
|
||||||
|
const wan = topo?.wanIfaces.get(serverId)
|
||||||
|
const clients = topo?.clientIfaces.get(serverId)
|
||||||
|
const download = Boolean(wan?.has(ingress) || clients?.has(egress))
|
||||||
|
const upload = Boolean(wan?.has(egress) || clients?.has(ingress))
|
||||||
|
if (download !== upload) return download ? "rx" : "tx"
|
||||||
|
if (isUnspecifiedIp(row.src) || isUnspecifiedIp(row.dst)) return null
|
||||||
|
const host = canonicalIp(exporterHost)
|
||||||
|
if (host && canonicalIp(row.src) === host && canonicalIp(row.dst) !== host) return "tx"
|
||||||
|
if (host && canonicalIp(row.dst) === host && canonicalIp(row.src) !== host) return "rx"
|
||||||
|
const srcLocal = isNonPublicIp(row.src), dstLocal = isNonPublicIp(row.dst)
|
||||||
|
return srcLocal === dstLocal ? null : srcLocal ? "tx" : "rx"
|
||||||
|
}
|
||||||
|
|
||||||
export interface InternetDestMeta {
|
export interface InternetDestMeta {
|
||||||
dest: string
|
dest: string
|
||||||
ripe: FlowIpMeta | null
|
ripe: FlowIpMeta | null
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ const dependencies:Record<string,unknown>={
|
|||||||
"./traffic-flow-ifaces.js":{maybeRefreshIfaces:async()=>{}},
|
"./traffic-flow-ifaces.js":{maybeRefreshIfaces:async()=>{}},
|
||||||
"./traffic-flow-ifindex.js":{canonicalFactIface:(_server:number,iface:string)=>iface},
|
"./traffic-flow-ifindex.js":{canonicalFactIface:(_server:number,iface:string)=>iface},
|
||||||
"./traffic-flow-facts-filter.js":{shouldWriteFlowFact:()=>true},
|
"./traffic-flow-facts-filter.js":{shouldWriteFlowFact:()=>true},
|
||||||
"./traffic-flow-dest.js":{resolveInternetDest:()=>({dest:"",ripe:null,classified:{category:"web",service:"https"},country:"XX",asn:0})},
|
"./traffic-flow-dest.js":{flowRateDirection:()=>"tx",resolveInternetDest:()=>({dest:"",ripe:null,classified:{category:"web",service:"https"},country:"XX",asn:0})},
|
||||||
"./traffic-flow-topology.js":{peekFlowTopology:()=>({}),peekServerCatalog:()=>({byId:new Map([[1,{type:"router"}]])}),getServerCatalog:()=>{throw Error("unexpected catalog SQL")},loadFlowTopology:()=>{throw Error("unexpected topology SQL")}},
|
"./traffic-flow-topology.js":{peekFlowTopology:()=>({}),peekServerCatalog:()=>({byId:new Map([[1,{type:"router"}]])}),getServerCatalog:()=>{throw Error("unexpected catalog SQL")},loadFlowTopology:()=>{throw Error("unexpected topology SQL")}},
|
||||||
"./traffic-flow-facts.js":{
|
"./traffic-flow-facts.js":{
|
||||||
bumpFlowFact:live.bump,hasPendingFlowFact:live.has,factsPendingSize:()=>live.size,
|
bumpFlowFact:live.bump,hasPendingFlowFact:live.has,factsPendingSize:()=>live.size,
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ import { maybeRefreshIfaces } from "./traffic-flow-ifaces.js"
|
|||||||
import { canonicalFactIface } from "./traffic-flow-ifindex.js"
|
import { canonicalFactIface } from "./traffic-flow-ifindex.js"
|
||||||
import { shouldWriteFlowFact } from "./traffic-flow-facts-filter.js"
|
import { shouldWriteFlowFact } from "./traffic-flow-facts-filter.js"
|
||||||
import { canonicalIp } from "./traffic-flow-ip.js"
|
import { canonicalIp } from "./traffic-flow-ip.js"
|
||||||
import { resolveInternetDest } from "./traffic-flow-dest.js"
|
import { flowRateDirection, resolveInternetDest } from "./traffic-flow-dest.js"
|
||||||
import {
|
import {
|
||||||
getServerCatalog,
|
getServerCatalog,
|
||||||
loadFlowTopology,
|
loadFlowTopology,
|
||||||
@@ -315,12 +315,12 @@ function bumpTick(key: string, inBytes: number, outBytes: number): void {
|
|||||||
tickAccum.set(key, prev)
|
tickAccum.set(key, prev)
|
||||||
}
|
}
|
||||||
|
|
||||||
function prepareFlowTicks(serverId:number,flow:ParsedFlow):Array<{key:string;inBytes:number;outBytes:number}> {
|
function prepareFlowTicks(serverId:number,flow:ParsedFlow,direction:"rx"|"tx"|null):Array<{key:string;inBytes:number;outBytes:number}> {
|
||||||
const plane=classifyFlowPlaneLite(flow)
|
const plane=classifyFlowPlaneLite(flow)
|
||||||
if(plane==="mgmt")return []
|
if(plane==="mgmt")return []
|
||||||
return [{key:ringKey(serverId,plane==="overlay"?RING_OVERLAY:plane==="client_mesh"?RING_MESH:RING_PAYLOAD),inBytes:flow.bytes,outBytes:0},
|
return [{key:ringKey(serverId,plane==="overlay"?RING_OVERLAY:plane==="client_mesh"?RING_MESH:RING_PAYLOAD),inBytes:direction==="rx"?flow.bytes:0,outBytes:direction==="tx"?flow.bytes:0},
|
||||||
...(flow.inIface?[{key:ringKey(serverId,flow.inIface),inBytes:flow.bytes,outBytes:0}]:[]),
|
...(flow.inIface?[{key:ringKey(serverId,flow.inIface),inBytes:flow.bytes,outBytes:0}]:[]),
|
||||||
...(flow.outIface&&flow.outIface!==flow.inIface?[{key:ringKey(serverId,flow.outIface),inBytes:0,outBytes:flow.bytes}]:[])]
|
...(flow.outIface?[{key:ringKey(serverId,flow.outIface),inBytes:0,outBytes:flow.bytes}]:[])]
|
||||||
}
|
}
|
||||||
|
|
||||||
function emptyRing(): { inBps: number[]; outBps: number[] } {
|
function emptyRing(): { inBps: number[]; outBps: number[] } {
|
||||||
@@ -426,7 +426,8 @@ export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): vo
|
|||||||
const topo = peekFlowTopology(), catalog = peekServerCatalog()
|
const topo = peekFlowTopology(), catalog = peekServerCatalog()
|
||||||
if (!topo) refreshColdTopology()
|
if (!topo) refreshColdTopology()
|
||||||
if (!catalog) refreshColdCatalog()
|
if (!catalog) refreshColdCatalog()
|
||||||
const serverType = catalog?.byId.get(serverId)?.type
|
const server = catalog?.byId.get(serverId)
|
||||||
|
const serverType = server?.type
|
||||||
for (const raw of flows) {
|
for (const raw of flows) {
|
||||||
const flow = normalizeParsedFlow(raw)
|
const flow = normalizeParsedFlow(raw)
|
||||||
const destMeta = resolveInternetDest({src:flow.src,dst:flow.dst,proto:flow.proto,srcPort:flow.srcPort,dstPort:flow.dstPort,
|
const destMeta = resolveInternetDest({src:flow.src,dst:flow.dst,proto:flow.proto,srcPort:flow.srcPort,dstPort:flow.dstPort,
|
||||||
@@ -440,7 +441,7 @@ export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): vo
|
|||||||
natSrc:flow.natSrc,natDst:flow.natDst,natSrcPort:flow.natSrcPort,natDstPort:flow.natDstPort})?
|
natSrc:flow.natSrc,natDst:flow.natDst,natSrcPort:flow.natSrcPort,natDstPort:flow.natDstPort})?
|
||||||
{serverId,bucketAt:hourAt,iface:canonicalFactIface(serverId,flow.inIface),country:country||"XX",service:classified.service,
|
{serverId,bucketAt:hourAt,iface:canonicalFactIface(serverId,flow.inIface),country:country||"XX",service:classified.service,
|
||||||
asn:destMeta.asn,bytes:flow.bytes,packets:flow.packets}:null
|
asn:destMeta.asn,bytes:flow.bytes,packets:flow.packets}:null
|
||||||
const ticks=prepareFlowTicks(serverId,flow),rollupCacheKey=rollupKey(serverId,bucketAt)
|
const ticks=prepareFlowTicks(serverId,flow,flowRateDirection(serverId,flow,topo,server?.host)),rollupCacheKey=rollupKey(serverId,bucketAt)
|
||||||
if(!admitAggregateRecord(rollupCacheKey,flow,dims,fact,ticks,observedAt))continue
|
if(!admitAggregateRecord(rollupCacheKey,flow,dims,fact,ticks,observedAt))continue
|
||||||
for(const tick of ticks)bumpTick(tick.key,tick.inBytes,tick.outBytes)
|
for(const tick of ticks)bumpTick(tick.key,tick.inBytes,tick.outBytes)
|
||||||
bumpRollup(serverId,bucketAt,flow,flow.bytes,flow.packets,rollupCacheKey)
|
bumpRollup(serverId,bucketAt,flow,flow.bytes,flow.packets,rollupCacheKey)
|
||||||
@@ -557,7 +558,7 @@ function pruneRecent(sinceMs = Date.now() - LIVE_WINDOW_MS): void {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PendingFlowReadOptions { serverId?:number; sinceIso?:string; untilIso?:string }
|
export interface PendingFlowReadOptions { serverId?:number; serverIds?:readonly number[]; sinceIso?:string; untilIso?:string }
|
||||||
export class FlowPendingReadUnconfirmed extends Error {
|
export class FlowPendingReadUnconfirmed extends Error {
|
||||||
readonly code = "FLOW_PENDING_UNCONFIRMED"
|
readonly code = "FLOW_PENDING_UNCONFIRMED"
|
||||||
constructor(readonly batchState:"running"|"unknown") { super(`NetFlow pending read unavailable: ${batchState} write batch`) }
|
constructor(readonly batchState:"running"|"unknown") { super(`NetFlow pending read unavailable: ${batchState} write batch`) }
|
||||||
@@ -566,8 +567,10 @@ function pendingReadMatches(options:PendingFlowReadOptions):(row:Pick<PendingFlo
|
|||||||
const since=options.sinceIso===undefined?-Infinity:typeof options.sinceIso==="string"?Date.parse(options.sinceIso):NaN
|
const since=options.sinceIso===undefined?-Infinity:typeof options.sinceIso==="string"?Date.parse(options.sinceIso):NaN
|
||||||
const until=options.untilIso===undefined?Infinity:typeof options.untilIso==="string"?Date.parse(options.untilIso):NaN
|
const until=options.untilIso===undefined?Infinity:typeof options.untilIso==="string"?Date.parse(options.untilIso):NaN
|
||||||
if(options.serverId!==undefined&&(!Number.isSafeInteger(options.serverId)||options.serverId<=0)||Number.isNaN(since)||Number.isNaN(until)||since>until)throw Error("Invalid pending flow read scope")
|
if(options.serverId!==undefined&&(!Number.isSafeInteger(options.serverId)||options.serverId<=0)||Number.isNaN(since)||Number.isNaN(until)||since>until)throw Error("Invalid pending flow read scope")
|
||||||
|
const serverIds = options.serverIds && new Set(options.serverIds)
|
||||||
return row=>{
|
return row=>{
|
||||||
if(options.serverId!==undefined&&row.serverId!==options.serverId)return false
|
if(options.serverId!==undefined&&row.serverId!==options.serverId)return false
|
||||||
|
if(serverIds&&!serverIds.has(row.serverId))return false
|
||||||
const time=Date.parse(row.bucketAt)
|
const time=Date.parse(row.bucketAt)
|
||||||
if(!Number.isFinite(time))throw Error("Invalid pending flow timestamp")
|
if(!Number.isFinite(time))throw Error("Invalid pending flow timestamp")
|
||||||
return time>=since&&time<=until
|
return time>=since&&time<=until
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { flowReadBucketSnapshot } from "./traffic-flow-read-policy.js"
|
import { flowReadBucketSnapshot } from "./traffic-flow-read-policy.js"
|
||||||
|
import { FlowSourceWindowTooLarge, FlowSqlSourceTooLarge } from "./traffic-flow-read-window.js"
|
||||||
import {flowIngestQualitySchema} from "@mmapp/contracts/traffic-flow"
|
import {flowIngestQualitySchema} from "@mmapp/contracts/traffic-flow"
|
||||||
import { flowLifecycleBoundary, type FlowLifecycleAuthorization } from "./traffic-flow-lifecycle-fence.js"
|
import { flowLifecycleBoundary, type FlowLifecycleAuthorization } from "./traffic-flow-lifecycle-fence.js"
|
||||||
import { flowCounter } from "./traffic-flow-observed-values.js"
|
import { flowCounter } from "./traffic-flow-observed-values.js"
|
||||||
@@ -475,12 +476,17 @@ interface StoredBucketRow {
|
|||||||
nat_dst_port: number | null
|
nat_dst_port: number | null
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function listStoredFlowRows(sinceIso: string, serverId?: number, options?: { maxRows?: number; untilIso?: string }): Promise<PendingFlowRow[]> {
|
export async function listStoredFlowRows(sinceIso: string, serverId?: number, options?: { maxRows?: number; untilIso?: string; serverIds?: readonly number[] }): Promise<PendingFlowRow[]> {
|
||||||
if (options?.maxRows !== undefined && (!Number.isSafeInteger(options.maxRows) || options.maxRows < 1 || options.maxRows > 100000)) throw new Error("Некорректный предел flow rows")
|
if (options?.maxRows !== undefined && (!Number.isSafeInteger(options.maxRows) || options.maxRows < 1 || options.maxRows > 100000)) throw new Error("Некорректный предел flow rows")
|
||||||
if (options?.untilIso && !Number.isFinite(Date.parse(options.untilIso))) throw new Error("Некорректная граница flow window")
|
if (options?.untilIso && !Number.isFinite(Date.parse(options.untilIso))) throw new Error("Некорректная граница flow window")
|
||||||
const settings = options?.maxRows === undefined ? await getTrafficFlowSettingsRow() : await readFlowSourceLimits()
|
const settings = options?.maxRows === undefined ? await getTrafficFlowSettingsRow() : await readFlowSourceLimits()
|
||||||
const keep = Math.max(20, settings.topN)
|
const keep = Math.max(20, settings.topN)
|
||||||
const parameters: Array<string | number> = serverId === undefined ? [sinceIso, keep] : [sinceIso, keep, serverId]
|
const serverIds = options?.serverIds
|
||||||
|
if (serverIds !== undefined && (serverId !== undefined || serverIds.some(id => !Number.isSafeInteger(id) || id < 1))) throw new Error("Некорректная область flow rows")
|
||||||
|
if (serverIds?.length === 0) return []
|
||||||
|
const parameters: Array<string | number | readonly number[]> = [sinceIso, keep]
|
||||||
|
if (serverId !== undefined || serverIds !== undefined) parameters.push(serverId ?? serverIds!)
|
||||||
|
const serverFilter = serverId !== undefined ? "AND server_id=$3" : serverIds ? "AND server_id=ANY($3::bigint[])" : ""
|
||||||
const untilParameter = options?.untilIso ? parameters.push(options.untilIso) : undefined
|
const untilParameter = options?.untilIso ? parameters.push(options.untilIso) : undefined
|
||||||
if (options?.maxRows !== undefined) parameters.push(options.maxRows + 1)
|
if (options?.maxRows !== undefined) parameters.push(options.maxRows + 1)
|
||||||
const stored = await flowReadBucketSnapshot<StoredBucketRow>(`
|
const stored = await flowReadBucketSnapshot<StoredBucketRow>(`
|
||||||
@@ -495,12 +501,15 @@ export async function listStoredFlowRows(sinceIso: string, serverId?: number, op
|
|||||||
PARTITION BY server_id, bucket_at ORDER BY bytes DESC
|
PARTITION BY server_id, bucket_at ORDER BY bytes DESC
|
||||||
) AS rn
|
) AS rn
|
||||||
FROM flow_buckets fb
|
FROM flow_buckets fb
|
||||||
WHERE bucket_at >= $1 ${serverId === undefined ? "" : "AND server_id=$3"} ${untilParameter ? `AND bucket_at <= $${untilParameter}` : ""}
|
WHERE bucket_at >= $1 ${serverFilter} ${untilParameter ? `AND bucket_at <= $${untilParameter}` : ""}
|
||||||
) ranked
|
) ranked
|
||||||
WHERE rn <= $2
|
WHERE rn <= $2
|
||||||
${options?.maxRows !== undefined ? `LIMIT $${parameters.length}` : ""}
|
${options?.maxRows !== undefined ? `LIMIT $${parameters.length}` : ""}
|
||||||
`, parameters)
|
`, parameters).catch(error => {
|
||||||
if (options?.maxRows !== undefined && stored.length > options.maxRows) throw new Error("Превышен предел flow rows; сократите окно или число серверов")
|
if (error instanceof FlowSqlSourceTooLarge) throw new FlowSourceWindowTooLarge()
|
||||||
|
throw error
|
||||||
|
})
|
||||||
|
if (options?.maxRows !== undefined && stored.length > options.maxRows) throw new FlowSourceWindowTooLarge()
|
||||||
const merged = new Map<string, PendingFlowRow>()
|
const merged = new Map<string, PendingFlowRow>()
|
||||||
for (const r of stored) {
|
for (const r of stored) {
|
||||||
mergeInto(merged, {
|
mergeInto(merged, {
|
||||||
@@ -525,14 +534,14 @@ export async function listStoredFlowRows(sinceIso: string, serverId?: number, op
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (!worker) {
|
if (!worker) {
|
||||||
for (const p of peekPendingFlows({serverId,sinceIso,untilIso:options?.untilIso})) {
|
for (const p of peekPendingFlows({serverId,serverIds,sinceIso,untilIso:options?.untilIso})) {
|
||||||
if (p.bucketAt < sinceIso || options?.untilIso && p.bucketAt > options.untilIso || serverId !== undefined && p.serverId !== serverId) continue
|
if (p.bucketAt < sinceIso || options?.untilIso && p.bucketAt > options.untilIso || serverId !== undefined && p.serverId !== serverId) continue
|
||||||
flowCounter(p.bytes); flowCounter(p.packets)
|
flowCounter(p.bytes); flowCounter(p.packets)
|
||||||
mergeInto(merged, p)
|
mergeInto(merged, p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const result = capFlowRowsPerServerBucket([...merged.values()], keep)
|
const result = capFlowRowsPerServerBucket([...merged.values()], keep)
|
||||||
if (options?.maxRows !== undefined && result.length > options.maxRows) throw new Error("Превышен предел flow rows с pending; сократите окно или число серверов")
|
if (options?.maxRows !== undefined && result.length > options.maxRows) throw new FlowSourceWindowTooLarge()
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import { resetFlowRingsForTests,applyHeartbeatForTests } from './traffic-flow-in
|
|||||||
import { createFlowIngestQuality } from './traffic-flow-ingest-quality.js'
|
import { createFlowIngestQuality } from './traffic-flow-ingest-quality.js'
|
||||||
import { disableRipeEnqueueForTests } from './traffic-flow-ripe.js'
|
import { disableRipeEnqueueForTests } from './traffic-flow-ripe.js'
|
||||||
import { readFlowCatalogWindow } from './traffic-flow-catalog-read.js'
|
import { readFlowCatalogWindow } from './traffic-flow-catalog-read.js'
|
||||||
|
import { invalidateFlowAnalyticsCache } from './traffic-flow-analytics.js'
|
||||||
import { sharedDatabaseGate } from './shared-database-gate.js'
|
import { sharedDatabaseGate } from './shared-database-gate.js'
|
||||||
import { getTrafficFlowSettingsRow,invalidateTrafficFlowSettingsCache } from './traffic-flow-settings.js'
|
import { getTrafficFlowSettingsRow,invalidateTrafficFlowSettingsCache } from './traffic-flow-settings.js'
|
||||||
import { createFlowReadBudget,createFlowReadResponseOwner } from './traffic-flow-read-budget.js'
|
import { createFlowReadBudget,createFlowReadResponseOwner } from './traffic-flow-read-budget.js'
|
||||||
@@ -35,12 +36,13 @@ try{
|
|||||||
await pool.query("INSERT INTO servers(id,name,host) VALUES(1,'visible','192.0.2.1'),(2,'FORBIDDEN_HIDDEN_SERVER','192.0.2.2')")
|
await pool.query("INSERT INTO servers(id,name,host) VALUES(1,'visible','192.0.2.1'),(2,'FORBIDDEN_HIDDEN_SERVER','192.0.2.2')")
|
||||||
const bucket=new Date(Date.now()-60000);await ensurePartitionFor(pool,'flow_buckets','day',bucket)
|
const bucket=new Date(Date.now()-60000);await ensurePartitionFor(pool,'flow_buckets','day',bucket)
|
||||||
await pool.query("INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface) VALUES(1,$1,'198.18.1.1','8.8.8.8',6,1,443,100,1,'ether1'),(2,$1,'198.18.2.1','8.8.8.8',6,1,443,999,1,'FORBIDDEN_HIDDEN_IFACE')",[bucket])
|
await pool.query("INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface) VALUES(1,$1,'198.18.1.1','8.8.8.8',6,1,443,100,1,'ether1'),(2,$1,'198.18.2.1','8.8.8.8',6,1,443,999,1,'FORBIDDEN_HIDDEN_IFACE')",[bucket])
|
||||||
const access=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned HTTP',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','traffic.flow.monthly.read','traffic.flow.live.read'],allowGlobal:false},limits:{callsPerMinute:240}}),actor)
|
const access=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned HTTP',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','traffic.flow.monthly.read','traffic.flow.live.read','traffic.flow.analytics.read'],allowGlobal:false},limits:{callsPerMinute:240}}),actor)
|
||||||
resetFlowRingsForTests();disableRipeEnqueueForTests()
|
resetFlowRingsForTests();disableRipeEnqueueForTests()
|
||||||
const collectorQuality=createFlowIngestQuality();collectorQuality.reject(987654,987,'aggregate-capacity')
|
const collectorQuality=createFlowIngestQuality();collectorQuality.reject(987654,987,'aggregate-capacity')
|
||||||
// Synthetic quality goes through the actual validated heartbeat receiver;
|
// Synthetic quality goes through the actual validated heartbeat receiver;
|
||||||
// this test starts no collector Worker or remote exporter.
|
// this test starts no collector Worker or remote exporter.
|
||||||
applyHeartbeatForTests({bound:false,address:null,packetsReceived:0,lastExporterIp:'',lastError:'',lastDatagramAt:null,pendingSize:0,dropped:0,rowsStored:0,workerAlive:true,rings:[],dataQuality:collectorQuality.snapshot()})
|
const heartbeat={bound:false,address:null,packetsReceived:0,lastExporterIp:'',lastError:'',lastDatagramAt:null,pendingSize:0,dropped:0,rowsStored:0,workerAlive:true,rings:[],dataQuality:collectorQuality.snapshot()}
|
||||||
|
applyHeartbeatForTests(heartbeat)
|
||||||
await readFlowCatalogWindow(5,[1])
|
await readFlowCatalogWindow(5,[1])
|
||||||
{
|
{
|
||||||
const budget=createFlowReadBudget({maxOwners:2,maxReservedBytes:200}),response=new EventEmitter() as ServerResponse,owner=createFlowReadResponseOwner(response,budget,{maxOwners:1,maxReservedBytes:100})
|
const budget=createFlowReadBudget({maxOwners:2,maxReservedBytes:200}),response=new EventEmitter() as ServerResponse,owner=createFlowReadResponseOwner(response,budget,{maxOwners:1,maxReservedBytes:100})
|
||||||
@@ -119,6 +121,52 @@ try{
|
|||||||
const scopedQuality=(result.structuredContent as {result:Record<string,unknown>}).result
|
const scopedQuality=(result.structuredContent as {result:Record<string,unknown>}).result
|
||||||
assert.equal(scopedQuality.dataQuality,null);assert.equal(scopedQuality.collectorQualityUnavailableReason,'global-telemetry-not-granted')
|
assert.equal(scopedQuality.dataQuality,null);assert.equal(scopedQuality.collectorQualityUnavailableReason,'global-telemetry-not-granted')
|
||||||
assert.ok(!JSON.stringify(result).includes('987654'),'Scoped read must not disclose collector-wide loss amounts')
|
assert.ok(!JSON.stringify(result).includes('987654'),'Scoped read must not disclose collector-wide loss amounts')
|
||||||
|
const oldFlowStart = new Date(Date.now() - 7 * 3600000)
|
||||||
|
await ensurePartitionFor(pool, 'flow_buckets', 'day', oldFlowStart)
|
||||||
|
await ensurePartitionFor(pool, 'flow_buckets', 'day', new Date(oldFlowStart.getTime() + 251 * 60000))
|
||||||
|
await pool.query(`INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface)
|
||||||
|
SELECT 1,$1::timestamptz + minutes * interval '1 minute','198.18.3.0'::inet + host, '8.8.4.4',6,1234,443,50,1,'ether1'
|
||||||
|
FROM generate_series(0,251) minutes CROSS JOIN generate_series(1,200) host`, [oldFlowStart])
|
||||||
|
const denseRest = await fetch(endpoint+'/api/traffic/flow?range=24h')
|
||||||
|
assert.equal(denseRest.status,200)
|
||||||
|
const restWindow = (await denseRest.json() as {sourceWindow:{requestedMinutes:number;effectiveMinutes:number}}).sourceWindow
|
||||||
|
assert.equal(restWindow.requestedMinutes,1440)
|
||||||
|
assert(restWindow.effectiveMinutes<1440)
|
||||||
|
const denseMcp = await client.callTool({name:'mm_netflow_talkers_get',arguments:{serverId:1,minutes:1440}})
|
||||||
|
assert.ok(!denseMcp.isError,JSON.stringify(denseMcp))
|
||||||
|
const mcpDay = (denseMcp.structuredContent as {result:{sourceWindow:{requestedMinutes:number;effectiveMinutes:number}; retainedRowsBytes:number; from:string;to:string}}).result
|
||||||
|
assert.equal(mcpDay.sourceWindow.requestedMinutes,1440)
|
||||||
|
assert(mcpDay.sourceWindow.effectiveMinutes<1440)
|
||||||
|
assert.equal(Date.parse(mcpDay.to)-Date.parse(mcpDay.from),mcpDay.sourceWindow.effectiveMinutes*60000)
|
||||||
|
assert(!JSON.stringify(denseMcp).includes('FORBIDDEN_'))
|
||||||
|
console.log('PASS dense REST and MCP NetFlow windows report the actual safely admitted interval')
|
||||||
|
const counterStart = new Date(Date.now()-10*60000)
|
||||||
|
await ensurePartitionFor(pool,'traffic_samples','week',counterStart)
|
||||||
|
await ensurePartitionFor(pool,'traffic_samples','week',new Date(counterStart.getTime()+50010))
|
||||||
|
await pool.query(`INSERT INTO traffic_samples(server_id,interface_name,sampled_at,rx_bps,tx_bps,rx_bytes,tx_bytes)
|
||||||
|
SELECT 1,'gre-test',$1::timestamptz + n * interval '1 millisecond',n,2*n,3*n,4*n FROM generate_series(1,50010) n`,[counterStart])
|
||||||
|
await pool.query(`INSERT INTO flow_buckets(server_id,bucket_at,src,dst,proto,src_port,dst_port,bytes,packets,in_iface)
|
||||||
|
VALUES(1,$1,'8.8.8.8','198.18.1.1',6,443,1,50,1,'gre-test')`,[bucket])
|
||||||
|
applyHeartbeatForTests({...heartbeat,dataQuality:createFlowIngestQuality().snapshot()})
|
||||||
|
try {
|
||||||
|
const analytics = await fetch(endpoint+'/api/traffic/flow/analytics?range=5m&serverId=1')
|
||||||
|
assert.equal(analytics.status,200,await analytics.clone().text())
|
||||||
|
const shortReport = await analytics.json() as {bpsWire:number;bytesWire:number;sourceWindow?:unknown;rxSeries:number[];txSeries:number[]}
|
||||||
|
assert.equal(shortReport.bpsWire,150030)
|
||||||
|
assert.equal(shortReport.bytesWire,350070)
|
||||||
|
assert.equal(shortReport.sourceWindow,undefined)
|
||||||
|
const report = await client.callTool({name:'mm_netflow_analytics_report_get',arguments:{serverId:1,minutes:5}})
|
||||||
|
assert.ok(!report.isError,JSON.stringify(report))
|
||||||
|
const mcp = (report.structuredContent as {result:{analytics:{rxSeries:number[];txSeries:number[]}}}).result.analytics
|
||||||
|
for (const series of [shortReport,mcp]) {
|
||||||
|
assert.equal(series.rxSeries.reduce((sum,value)=>sum+value,0),50*8/60/1e6)
|
||||||
|
assert.equal(series.txSeries.reduce((sum,value)=>sum+value,0),100*8/60/1e6)
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
applyHeartbeatForTests(heartbeat)
|
||||||
|
invalidateFlowAnalyticsCache()
|
||||||
|
}
|
||||||
|
console.log('PASS five-minute REST analysis uses one latest counter per interface despite 50,010 historical samples')
|
||||||
const globalAccess=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned global telemetry',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','netflow.settings.read'],allowGlobal:true},limits:{callsPerMinute:240}}),actor)
|
const globalAccess=await createMcpConnection(mcpConnectionCreateSchema.parse({name:'owned global telemetry',mode:'observer',permissions:['mm:traffic:read'],expiresAt:new Date(Date.now()+600000).toISOString(),scope:{serverIds:[1],operationIds:['traffic.flow.talkers.read','netflow.settings.read'],allowGlobal:true},limits:{callsPerMinute:240}}),actor)
|
||||||
const globalClient=new Client({name:'owned-global-quality',version:'1'})
|
const globalClient=new Client({name:'owned-global-quality',version:'1'})
|
||||||
try{
|
try{
|
||||||
|
|||||||
@@ -0,0 +1,27 @@
|
|||||||
|
export class FlowSqlSourceTooLarge extends Error {
|
||||||
|
readonly statusCode = 409
|
||||||
|
readonly code = "STATE_UNAVAILABLE"
|
||||||
|
|
||||||
|
constructor() {
|
||||||
|
super("Источник NetFlow превышает допустимый объём; сократите область или окно")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
export class FlowSourceWindowTooLarge extends FlowSqlSourceTooLarge {}
|
||||||
|
|
||||||
|
type FlowReadWindow = { requestedMinutes: number; effectiveMinutes: number }
|
||||||
|
|
||||||
|
/** Retrying a smaller historical window keeps the same response reservation.
|
||||||
|
* Only an explicit source-size refusal allows fallback; other errors propagate. */
|
||||||
|
export async function readFlowWindow<T extends object>(requestedMinutes: number, read: (minutes: number) => Promise<T>, signal?: AbortSignal): Promise<T & { sourceWindow?: FlowReadWindow }> {
|
||||||
|
let effectiveMinutes = requestedMinutes
|
||||||
|
for (;;) {
|
||||||
|
signal?.throwIfAborted()
|
||||||
|
try {
|
||||||
|
const result = await read(effectiveMinutes)
|
||||||
|
return effectiveMinutes === requestedMinutes ? result : { ...result, sourceWindow: { requestedMinutes, effectiveMinutes } }
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof FlowSourceWindowTooLarge) || effectiveMinutes <= 1) throw error
|
||||||
|
effectiveMinutes = Math.max(1, Math.floor(effectiveMinutes / (effectiveMinutes <= 15 ? 2 : 4)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -247,7 +247,6 @@ export function enGreIfaceNames(topo: FlowTopology, serverId: number, ifaceNames
|
|||||||
export async function latestWireBps(serverId: number, ifaceNames: string[], bounded = false): Promise<{ bps: number; bytes: number }> {
|
export async function latestWireBps(serverId: number, ifaceNames: string[], bounded = false): Promise<{ bps: number; bytes: number }> {
|
||||||
if (!ifaceNames.length) return { bps: 0, bytes: 0 }
|
if (!ifaceNames.length) return { bps: 0, bytes: 0 }
|
||||||
if (bounded && ifaceNames.length > 1000) throw new Error("Превышен предел физических интерфейсов NetFlow")
|
if (bounded && ifaceNames.length > 1000) throw new Error("Превышен предел физических интерфейсов NetFlow")
|
||||||
const placeholders = ifaceNames.map(() => "?").join(",")
|
|
||||||
const rows = await flowReadDbAll<{
|
const rows = await flowReadDbAll<{
|
||||||
name: string
|
name: string
|
||||||
rxBps: number
|
rxBps: number
|
||||||
@@ -255,18 +254,18 @@ export async function latestWireBps(serverId: number, ifaceNames: string[], boun
|
|||||||
rxBytes: number
|
rxBytes: number
|
||||||
txBytes: number
|
txBytes: number
|
||||||
}>(`
|
}>(`
|
||||||
SELECT ${bounded ? "DISTINCT ON (interface_name)" : ""} interface_name AS name, rx_bps AS "rxBps", tx_bps AS "txBps", rx_bytes AS "rxBytes", tx_bytes AS "txBytes"
|
SELECT iface.name, sample.rx_bps AS "rxBps", sample.tx_bps AS "txBps", sample.rx_bytes AS "rxBytes", sample.tx_bytes AS "txBytes"
|
||||||
FROM traffic_samples
|
FROM (SELECT DISTINCT unnest(?::text[]) AS name) iface
|
||||||
WHERE server_id = ? AND interface_name IN (${placeholders})
|
CROSS JOIN LATERAL (
|
||||||
ORDER BY ${bounded ? "interface_name," : ""} sampled_at DESC
|
SELECT rx_bps, tx_bps, rx_bytes, tx_bytes FROM traffic_samples
|
||||||
`, [serverId, ...ifaceNames])
|
WHERE server_id = ? AND interface_name = iface.name
|
||||||
const seen = new Set<string>()
|
ORDER BY sampled_at DESC LIMIT 1
|
||||||
|
) sample
|
||||||
|
`, [ifaceNames, serverId])
|
||||||
let bps = 0
|
let bps = 0
|
||||||
let bytes = 0
|
let bytes = 0
|
||||||
for (const r of rows) {
|
for (const r of rows) {
|
||||||
if (bounded && ([r.rxBps,r.txBps].some(value => value === null || !Number.isFinite(Number(value)) || Number(value) < 0 || Number(value) > Number.MAX_SAFE_INTEGER) || [r.rxBytes,r.txBytes].some(value => value === null || !Number.isSafeInteger(Number(value)) || Number(value) < 0))) throw new Error("Не подтверждены физические счётчики NetFlow")
|
if (bounded && ([r.rxBps,r.txBps].some(value => value === null || !Number.isFinite(Number(value)) || Number(value) < 0 || Number(value) > Number.MAX_SAFE_INTEGER) || [r.rxBytes,r.txBytes].some(value => value === null || !Number.isSafeInteger(Number(value)) || Number(value) < 0))) throw new Error("Не подтверждены физические счётчики NetFlow")
|
||||||
if (seen.has(r.name)) continue
|
|
||||||
seen.add(r.name)
|
|
||||||
bps += (Number(r.rxBps) || 0) + (Number(r.txBps) || 0)
|
bps += (Number(r.rxBps) || 0) + (Number(r.txBps) || 0)
|
||||||
bytes += (Number(r.rxBytes) || 0) + (Number(r.txBytes) || 0)
|
bytes += (Number(r.rxBytes) || 0) + (Number(r.txBytes) || 0)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -70,15 +70,17 @@ export function TrafficRxTxChart({
|
|||||||
rx,
|
rx,
|
||||||
tx,
|
tx,
|
||||||
range = "1h",
|
range = "1h",
|
||||||
|
rangeMinutes: actualMinutes,
|
||||||
embedded = false,
|
embedded = false,
|
||||||
}: {
|
}: {
|
||||||
rx: (number|null)[]
|
rx: (number|null)[]
|
||||||
tx: (number|null)[]
|
tx: (number|null)[]
|
||||||
range?: string
|
range?: string
|
||||||
|
rangeMinutes?: number
|
||||||
/** Skip outer Frame when already inside OpsPanel / Frame. */
|
/** Skip outer Frame when already inside OpsPanel / Frame. */
|
||||||
embedded?: boolean
|
embedded?: boolean
|
||||||
}) {
|
}) {
|
||||||
const rangeMinutes = TRAFFIC_RANGE_MINUTES[range] ?? 60
|
const rangeMinutes = actualMinutes ?? TRAFFIC_RANGE_MINUTES[range] ?? 60
|
||||||
const data = toChartData(rx, tx, rangeMinutes)
|
const data = toChartData(rx, tx, rangeMinutes)
|
||||||
const tickEvery = Math.max(1, Math.ceil(data.length / 6))
|
const tickEvery = Math.max(1, Math.ceil(data.length / 6))
|
||||||
|
|
||||||
|
|||||||
@@ -474,10 +474,13 @@ export function FlowAnalyticsDetail({
|
|||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
{analytics && <>
|
{analytics && <>
|
||||||
|
{analytics.sourceWindow && <p role="status" className="mb-3 text-sm text-muted-foreground">Данные за последние {analytics.sourceWindow.effectiveMinutes} мин из запрошенных {analytics.sourceWindow.requestedMinutes} мин.</p>}
|
||||||
|
<p className="mb-3 text-sm text-muted-foreground">График RX/TX показывает потоки с известным направлением. Общий объём включает также транзит с неопределённым направлением.</p>
|
||||||
<TrafficRxTxChart
|
<TrafficRxTxChart
|
||||||
rx={analytics?.rxSeries ?? card?.rxSeries ?? []}
|
rx={analytics?.rxSeries ?? card?.rxSeries ?? []}
|
||||||
tx={analytics?.txSeries ?? card?.txSeries ?? []}
|
tx={analytics?.txSeries ?? card?.txSeries ?? []}
|
||||||
range={range}
|
range={range}
|
||||||
|
rangeMinutes={analytics.sourceWindow?.effectiveMinutes}
|
||||||
/>
|
/>
|
||||||
|
|
||||||
<div className="mt-4 pt-4 border-t">
|
<div className="mt-4 pt-4 border-t">
|
||||||
|
|||||||
@@ -39,6 +39,12 @@ test("map Flow envelope retains zeros and refuses missing metadata and contradic
|
|||||||
assert.throws(()=>networkMapFlowSchema.parse({...empty,totalBytes:undefined}))
|
assert.throws(()=>networkMapFlowSchema.parse({...empty,totalBytes:undefined}))
|
||||||
assert.throws(()=>networkMapFlowSchema.parse({...empty,namedBytes:1}))
|
assert.throws(()=>networkMapFlowSchema.parse({...empty,namedBytes:1}))
|
||||||
assert.throws(()=>networkMapFlowSchema.parse({...empty,rangeMinutes:10}))
|
assert.throws(()=>networkMapFlowSchema.parse({...empty,rangeMinutes:10}))
|
||||||
|
const shorter={...empty,rangeMinutes:2,windowSec:120,sourceWindow:{requestedMinutes:5,effectiveMinutes:2}}
|
||||||
|
assert.equal(networkMapFlowSchema.parse(shorter).rangeMinutes,2)
|
||||||
|
assert.throws(()=>networkMapFlowSchema.parse({...shorter,sourceWindow:undefined}))
|
||||||
|
assert.throws(()=>networkMapFlowSchema.parse({...shorter,sourceWindow:{requestedMinutes:15,effectiveMinutes:2}}))
|
||||||
|
assert.throws(()=>networkMapFlowSchema.parse({...shorter,rangeMinutes:1,windowSec:60}))
|
||||||
|
assert.throws(()=>networkMapFlowSchema.parse({...shorter,windowSec:300}))
|
||||||
})
|
})
|
||||||
test("map DNS matches exactly requested names and accepts explicit unresolved null",()=>{
|
test("map DNS matches exactly requested names and accepts explicit unresolved null",()=>{
|
||||||
assert.deepEqual(validateNetworkMapDns({results:{"router.example":null}},["router.example"]),{"router.example":null})
|
assert.deepEqual(validateNetworkMapDns({results:{"router.example":null}},["router.example"]),{"router.example":null})
|
||||||
|
|||||||
@@ -27,7 +27,13 @@ export function validateNetworkMapSnapshot(servers:unknown,gre:unknown,probes:un
|
|||||||
|
|
||||||
import {flowMapHopsDtoSchema} from "@mmapp/contracts/traffic-flow"
|
import {flowMapHopsDtoSchema} from "@mmapp/contracts/traffic-flow"
|
||||||
import {isIpv4String} from "./gre-endpoint-resolve"
|
import {isIpv4String} from "./gre-endpoint-resolve"
|
||||||
export const networkMapFlowSchema=flowMapHopsDtoSchema.required().refine(data=>data.rangeMinutes===5&&data.namedBytes<=data.totalBytes&&data.unclassifiedBytes<=data.totalBytes)
|
export const networkMapFlowSchema = flowMapHopsDtoSchema.required().partial({ sourceWindow: true }).refine(data =>
|
||||||
|
(data.sourceWindow
|
||||||
|
? data.sourceWindow.requestedMinutes === 5 && data.sourceWindow.effectiveMinutes === data.rangeMinutes
|
||||||
|
: data.rangeMinutes === 5) &&
|
||||||
|
data.windowSec === data.rangeMinutes * 60 &&
|
||||||
|
data.namedBytes <= data.totalBytes && data.unclassifiedBytes <= data.totalBytes
|
||||||
|
)
|
||||||
export function validateNetworkMapDns(raw:unknown,hosts:string[]){
|
export function validateNetworkMapDns(raw:unknown,hosts:string[]){
|
||||||
const data=z.object({results:z.record(z.string(),z.string().refine(isIpv4String).nullable())}).parse(raw)
|
const data=z.object({results:z.record(z.string(),z.string().refine(isIpv4String).nullable())}).parse(raw)
|
||||||
const requested=new Set(hosts.map(host=>host.toLowerCase()))
|
const requested=new Set(hosts.map(host=>host.toLowerCase()))
|
||||||
|
|||||||
@@ -163,7 +163,13 @@ export const flowTalkerDtoSchema = z.object({
|
|||||||
plane: z.string().optional(),
|
plane: z.string().optional(),
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const flowSourceWindowSchema = z.object({
|
||||||
|
requestedMinutes: z.number().int().positive(),
|
||||||
|
effectiveMinutes: z.number().int().positive(),
|
||||||
|
}).strict().refine(value => value.effectiveMinutes < value.requestedMinutes)
|
||||||
|
|
||||||
export const flowStatsDtoSchema = z.object({
|
export const flowStatsDtoSchema = z.object({
|
||||||
|
sourceWindow: flowSourceWindowSchema.optional(),
|
||||||
dataQuality:flowIngestQualitySchema.nullable().optional(),
|
dataQuality:flowIngestQualitySchema.nullable().optional(),
|
||||||
exportersOnline: z.number().int().nonnegative(),
|
exportersOnline: z.number().int().nonnegative(),
|
||||||
bytesPerMin: z.number().nonnegative(),
|
bytesPerMin: z.number().nonnegative(),
|
||||||
@@ -246,6 +252,7 @@ export const flowPathRowSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export const flowAnalyticsDtoSchema = z.object({
|
export const flowAnalyticsDtoSchema = z.object({
|
||||||
|
sourceWindow: flowSourceWindowSchema.optional(),
|
||||||
bpsNow: z.number().nonnegative(),
|
bpsNow: z.number().nonnegative(),
|
||||||
bytes: z.number().nonnegative(),
|
bytes: z.number().nonnegative(),
|
||||||
packets: z.number().nonnegative(),
|
packets: z.number().nonnegative(),
|
||||||
@@ -284,6 +291,7 @@ export const flowAnalyticsDtoSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export const flowExportersDtoSchema = z.object({
|
export const flowExportersDtoSchema = z.object({
|
||||||
|
sourceWindow: flowSourceWindowSchema.optional(),
|
||||||
exporters: z.array(flowEntityCardSchema),
|
exporters: z.array(flowEntityCardSchema),
|
||||||
lastExporterIp: z.string().nullable().optional(),
|
lastExporterIp: z.string().nullable().optional(),
|
||||||
lastError: z.string().nullable().optional(),
|
lastError: z.string().nullable().optional(),
|
||||||
@@ -294,6 +302,7 @@ export const flowExportersDtoSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export const flowClientsDtoSchema = z.object({
|
export const flowClientsDtoSchema = z.object({
|
||||||
|
sourceWindow: flowSourceWindowSchema.optional(),
|
||||||
clients: z.array(flowEntityCardSchema),
|
clients: z.array(flowEntityCardSchema),
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -390,6 +399,7 @@ export const flowMapCountryServiceGroupDtoSchema = z.object({
|
|||||||
})
|
})
|
||||||
|
|
||||||
export const flowMapHopsDtoSchema = z.object({
|
export const flowMapHopsDtoSchema = z.object({
|
||||||
|
sourceWindow: flowSourceWindowSchema.optional(),
|
||||||
hops: z.array(flowMapHopDtoSchema),
|
hops: z.array(flowMapHopDtoSchema),
|
||||||
live: z.boolean(),
|
live: z.boolean(),
|
||||||
rangeMinutes: z.number().int().positive(),
|
rangeMinutes: z.number().int().positive(),
|
||||||
|
|||||||
@@ -5,12 +5,13 @@ test.beforeEach(async({page})=>{await page.route("**/health",r=>r.fulfill({json:
|
|||||||
const server={id:1,name:"Router",host:"192.0.2.1",type:"home-router",country:"RU",enabled:true,status:"online"}
|
const server={id:1,name:"Router",host:"192.0.2.1",type:"home-router",country:"RU",enabled:true,status:"online"}
|
||||||
for(const malformed of [false,true])test("Карта: NetFlow отказ и повтор "+malformed,async({page})=>{
|
for(const malformed of [false,true])test("Карта: NetFlow отказ и повтор "+malformed,async({page})=>{
|
||||||
let failing=true
|
let failing=true
|
||||||
await page.route("**/api/**",r=>{const p=new URL(r.request().url()).pathname;return r.fulfill(p===flowPath?(failing?(malformed?{json:{}}:{status:503,json:{error:"NetFlow недоступен"}}):{json:empty}):{json:p==="/api/servers"?[server]:p==="/api/filters/gre-tunnels"?{tunnels:[],failures:[]}:p==="/api/uptime/speed-probes"?{probes:[]}:p==="/api/auth/config"?{required:false}:{}})})
|
const recovered=malformed?{...empty,rangeMinutes:2,windowSec:120,sourceWindow:{requestedMinutes:5,effectiveMinutes:2}}:empty
|
||||||
|
await page.route("**/api/**",r=>{const p=new URL(r.request().url()).pathname;return r.fulfill(p===flowPath?(failing?(malformed?{json:{}}:{status:503,json:{error:"NetFlow недоступен"}}):{json:recovered}):{json:p==="/api/servers"?[server]:p==="/api/filters/gre-tunnels"?{tunnels:[],failures:[]}:p==="/api/uptime/speed-probes"?{probes:[]}:p==="/api/auth/config"?{required:false}:{}})})
|
||||||
await page.goto("/network-map")
|
await page.goto("/network-map")
|
||||||
await expect(page.getByRole("alert").filter({hasText:"Первоначальная загрузка NetFlow"})).toBeVisible()
|
await expect(page.getByRole("alert").filter({hasText:"Первоначальная загрузка NetFlow"})).toBeVisible()
|
||||||
await expect(page.getByText("Данные NetFlow неизвестны",{exact:true})).toBeVisible()
|
await expect(page.getByText("Данные NetFlow неизвестны",{exact:true})).toBeVisible()
|
||||||
failing=false;await page.getByRole("button",{name:"Обновить",exact:true}).click()
|
failing=false;await page.getByRole("button",{name:"Обновить",exact:true}).click()
|
||||||
await expect(page.getByText(/NetFlow · 5 мин · ответ/)).toContainText("приёмник выключен")
|
await expect(page.getByText(malformed?/NetFlow · 2 мин из 5 запрошенных · ответ/:/NetFlow · 5 мин · ответ/)).toContainText("приёмник выключен")
|
||||||
await expect(page.getByRole("alert").filter({hasText:"NetFlow:"})).toHaveCount(0)
|
await expect(page.getByRole("alert").filter({hasText:"NetFlow:"})).toHaveCount(0)
|
||||||
})
|
})
|
||||||
test("Карта: NetFlow сохраняет сервис при фоновом сбое",async({page})=>{
|
test("Карта: NetFlow сохраняет сервис при фоновом сбое",async({page})=>{
|
||||||
|
|||||||
@@ -13,7 +13,9 @@ for (const mode of ['servers','flows'] as const) for (const minutes of [5,15]) {
|
|||||||
const url=new URL(r.request().url()),path=url.pathname,span=url.searchParams.get('range')==='5m'?5:url.searchParams.get('range')==='15m'?15:60
|
const url=new URL(r.request().url()),path=url.pathname,span=url.searchParams.get('range')==='5m'?5:url.searchParams.get('range')==='15m'?15:60
|
||||||
const built=history(span),server={id:'1',name,site:'TEST',country:'RU',status:'online',...built,rxTotal:1,txTotal:1,volumeUnit:'GiB'}
|
const built=history(span),server={id:'1',name,site:'TEST',country:'RU',status:'online',...built,rxTotal:1,txTotal:1,volumeUnit:'GiB'}
|
||||||
const card={id:'1',name,subtitle:'',site:'TEST',country:'RU',status:'online',rxNow:1,txNow:0,sessions:1,bytes:60000,rxSeries:[1,2,1],txSeries:[0,0,0]}
|
const card={id:'1',name,subtitle:'',site:'TEST',country:'RU',status:'online',rxNow:1,txNow:0,sessions:1,bytes:60000,rxSeries:[1,2,1],txSeries:[0,0,0]}
|
||||||
const analytics={bpsNow:1000000,bytes:60000,packets:10,conversations:1,uniqueSrc:1,uniqueDst:1,topProto:'TCP',rxSeries:Array.from({length:span},(_,i)=>1+i%3),txSeries:Array(span).fill(0),applications:[],protocols:[],sources:[],destinations:[],interfaces:[],conversationsList:[],ifaces:[],live:false}
|
const sourceWindow=mode==='flows'&&span===15?{requestedMinutes:15,effectiveMinutes:7}:undefined
|
||||||
|
const chartMinutes=sourceWindow?.effectiveMinutes??span
|
||||||
|
const analytics={bpsNow:1000000,bytes:60000,packets:10,conversations:1,uniqueSrc:1,uniqueDst:1,topProto:'TCP',rxSeries:Array.from({length:chartMinutes},(_,i)=>1+i%3),txSeries:Array(chartMinutes).fill(0),applications:[],protocols:[],sources:[],destinations:[],interfaces:[],conversationsList:[],ifaces:[],live:false,sourceWindow}
|
||||||
if(path.endsWith('/live'))return r.fulfill({status:503,json:{error:'Live выключен в проверке истории'}})
|
if(path.endsWith('/live'))return r.fulfill({status:503,json:{error:'Live выключен в проверке истории'}})
|
||||||
const json=path==='/api/auth/config'?{required:false}:path==='/api/traffic/servers'?{servers:[server]}:
|
const json=path==='/api/auth/config'?{required:false}:path==='/api/traffic/servers'?{servers:[server]}:
|
||||||
path==='/api/traffic/servers/1'?{server}:path==='/api/traffic/flow/exporters'?{exporters:[card]}:
|
path==='/api/traffic/servers/1'?{server}:path==='/api/traffic/flow/exporters'?{exporters:[card]}:
|
||||||
@@ -28,6 +30,7 @@ for (const mode of ['servers','flows'] as const) for (const minutes of [5,15]) {
|
|||||||
await page.getByRole('button',{name:`${minutes}м`,exact:true}).click()
|
await page.getByRole('button',{name:`${minutes}м`,exact:true}).click()
|
||||||
const chart=page.getByRole('region',{name:'История RX/TX'})
|
const chart=page.getByRole('region',{name:'История RX/TX'})
|
||||||
await expect(chart).toBeVisible()
|
await expect(chart).toBeVisible()
|
||||||
|
if(mode==='flows'&&minutes===15)await expect(page.getByText('Данные за последние 7 мин из запрошенных 15 мин.')).toBeVisible()
|
||||||
const curves=chart.locator('path.recharts-line-curve')
|
const curves=chart.locator('path.recharts-line-curve')
|
||||||
await expect(curves).toHaveCount(2)
|
await expect(curves).toHaveCount(2)
|
||||||
await expect.poll(async()=>((await curves.first().getAttribute('d'))?.match(/[CL]/g)||[]).length).toBeGreaterThan(2)
|
await expect.poll(async()=>((await curves.first().getAttribute('d'))?.match(/[CL]/g)||[]).length).toBeGreaterThan(2)
|
||||||
|
|||||||
Reference in new issue
Block a user