697 lines
27 KiB
TypeScript
697 lines
27 KiB
TypeScript
import { canonicalIp, isUnspecifiedIp } from "./traffic-flow-ip.js"
|
|
|
|
export interface ParsedFlow {
|
|
src: string
|
|
dst: string
|
|
proto: number
|
|
srcPort: number
|
|
dstPort: number
|
|
bytes: number
|
|
packets: number
|
|
inIface: string
|
|
outIface: string
|
|
nextHop: string
|
|
flowStartMs: number
|
|
flowEndMs: number
|
|
natSrc: string
|
|
natDst: string
|
|
natSrcPort: number
|
|
natDstPort: number
|
|
}
|
|
|
|
export type ParsedFlowInput = Partial<ParsedFlow> & Pick<ParsedFlow, "src" | "dst" | "proto" | "bytes">
|
|
|
|
export function emptyParsedFlow(): ParsedFlow {
|
|
return {
|
|
src: "",
|
|
dst: "",
|
|
proto: 0,
|
|
srcPort: 0,
|
|
dstPort: 0,
|
|
bytes: 0,
|
|
packets: 0,
|
|
inIface: "",
|
|
outIface: "",
|
|
nextHop: "",
|
|
flowStartMs: 0,
|
|
flowEndMs: 0,
|
|
natSrc: "",
|
|
natDst: "",
|
|
natSrcPort: 0,
|
|
natDstPort: 0,
|
|
}
|
|
}
|
|
|
|
export function normalizeParsedFlow(flow: ParsedFlowInput): ParsedFlow {
|
|
return {
|
|
...emptyParsedFlow(),
|
|
...flow,
|
|
src: canonicalIp(flow.src) || flow.src || "",
|
|
dst: canonicalIp(flow.dst) || flow.dst || "",
|
|
nextHop: canonicalIp(flow.nextHop ?? ""),
|
|
flowStartMs: flow.flowStartMs ?? 0,
|
|
flowEndMs: flow.flowEndMs ?? 0,
|
|
natSrc: canonicalIp(flow.natSrc ?? ""),
|
|
natDst: canonicalIp(flow.natDst ?? ""),
|
|
natSrcPort: flow.natSrcPort ?? 0,
|
|
natDstPort: flow.natDstPort ?? 0,
|
|
inIface: flow.inIface ?? "",
|
|
outIface: flow.outIface ?? "",
|
|
srcPort: flow.srcPort ?? 0,
|
|
dstPort: flow.dstPort ?? 0,
|
|
packets: flow.packets ?? 0,
|
|
}
|
|
}
|
|
|
|
interface FieldSpec {
|
|
type: number
|
|
length: number
|
|
}
|
|
|
|
/** Trusted local receiver context, never read from the exported packet.
|
|
* collectorSocketId identifies a receiver socket incarnation. A wildcard-bound
|
|
* socket does not establish the packet's precise local destination IP. */
|
|
export interface FlowDatagramTransport { sourcePort: number; collectorSocketId: string }
|
|
function templateScope(version: number, domain: number, transport?: FlowDatagramTransport): string {
|
|
if (transport === undefined) return JSON.stringify([version, domain, "unknown-transport"])
|
|
if (!transport || !Number.isInteger(transport.sourcePort) || transport.sourcePort < 1 || transport.sourcePort > 65535
|
|
|| typeof transport.collectorSocketId !== "string" || transport.collectorSocketId.length < 1 || transport.collectorSocketId.length > 128
|
|
|| !/^[A-Za-z0-9_-]+$/.test(transport.collectorSocketId)) throw new Error("Invalid flow datagram transport context")
|
|
return JSON.stringify([version, domain, "udp-socket", transport.sourcePort, transport.collectorSocketId])
|
|
}
|
|
function scopedTemplateId(scope: string, id: number): string { return JSON.stringify([scope, id]) }
|
|
|
|
interface Template {
|
|
fields: FieldSpec[]
|
|
refreshedAtMs: number
|
|
v9Clock?: V9Clock
|
|
v9ValidFrom?: V9Clock
|
|
v9Conflicted?: true
|
|
ipfixExportTime?: number
|
|
ipfixValidFrom?: number
|
|
ipfixConflicted?: true
|
|
ipfixScope?: string
|
|
ipfixResetMessage?: object
|
|
}
|
|
|
|
type V9Clock = { exportedAt: number; uptime: number; sequence: number }
|
|
type IpfixClock = { exportTime: number; sequence: number }
|
|
type IpfixScopeState = { clock?: IpfixClock; resetAt?: number }
|
|
|
|
// RFC 3954 uses uint32 milliseconds for uptime and whole seconds for export time.
|
|
function differentV9Boot(first: V9Clock, second: V9Clock): boolean {
|
|
const elapsedExportMs = (second.exportedAt - first.exportedAt) * 1000
|
|
// Signed modular distance handles wrap and reordered datagrams.
|
|
return Math.abs((second.uptime - first.uptime - elapsedExportMs) | 0) > 2000
|
|
}
|
|
|
|
function compareV9Clock(first: V9Clock, second: V9Clock): number {
|
|
return first.exportedAt - second.exportedAt ||
|
|
((first.uptime - second.uptime) | 0) || ((first.sequence - second.sequence) | 0)
|
|
}
|
|
|
|
// RouterOS refreshes reuse IDs; its configurable record fields number in tens.
|
|
// These generous retention budgets bound objects, not an asserted V8 byte size.
|
|
export const FLOW_TEMPLATE_LIMITS = Object.freeze({ exporters: 256, perExporterTemplates: 256, templates: 4096,
|
|
perTemplateFields: 256, perExporterFields: 8192, fields: 65536 })
|
|
// RFC7011 8.4: UDP lifetime is deployment-specific; use three times the
|
|
// 30-minute refresh interval in the MikroTik MUM configuration example.
|
|
// Longer exporter refresh intervals require an explicit collector override.
|
|
// This is not active-flow-timeout, and data traffic never refreshes a template.
|
|
export const FLOW_TEMPLATE_LIFETIME_MS = Number(process.env.FLOW_TEMPLATE_LIFETIME_MS ?? 90 * 60 * 1000)
|
|
if (!Number.isSafeInteger(FLOW_TEMPLATE_LIFETIME_MS) || FLOW_TEMPLATE_LIFETIME_MS <= 0) {
|
|
throw new Error("FLOW_TEMPLATE_LIFETIME_MS must be a positive safe integer in milliseconds")
|
|
}
|
|
const MAX_TEMPLATE_EXPORTERS = FLOW_TEMPLATE_LIMITS.exporters
|
|
const templatesByExporter = new Map<string, Map<string, Template>>()
|
|
// Each scope exists only while its exporter retains at least one template.
|
|
const ipfixScopes = new WeakMap<Map<string, Template>, Map<string, IpfixScopeState>>()
|
|
const fieldsByExporter = new Map<string, number>()
|
|
let templateCount = 0, fieldCount = 0, refusedTemplates = 0
|
|
export class FlowTemplateRefused extends Error {
|
|
readonly code = "FLOW_TEMPLATE_REFUSED"
|
|
constructor(readonly reason: "fields" | "exporter-templates" | "templates" | "exporter-fields" | "total-fields" | "truncated" | "missing-template") {
|
|
super("NetFlow/IPFIX template refused: " + reason)
|
|
}
|
|
}
|
|
function refuseTemplate(reason: FlowTemplateRefused["reason"]): never { refusedTemplates++; throw new FlowTemplateRefused(reason) }
|
|
function forgetExporter(exporter: string): void {
|
|
const previous = templatesByExporter.get(exporter)
|
|
if (!previous) return
|
|
templateCount -= previous.size; fieldCount -= fieldsByExporter.get(exporter) ?? 0
|
|
fieldsByExporter.delete(exporter); templatesByExporter.delete(exporter)
|
|
}
|
|
|
|
/** Lazy expiry uses local monotonic time, never exporter timestamps. Retention
|
|
* remains bounded by existing caps; no sweeper timer, pending data or tombstones.
|
|
* Empty exporter maps remain in the existing bounded LRU until normal eviction. */
|
|
function expireTemplate(exporter: string, map: Map<string, Template>, id: string, now: number): void {
|
|
const retained = map.get(id)
|
|
if (!retained || now - retained.refreshedAtMs < FLOW_TEMPLATE_LIFETIME_MS) return
|
|
map.delete(id); templateCount--; fieldCount -= retained.fields.length
|
|
const scope = retained.ipfixScope
|
|
if (scope !== undefined && ![...map.values()].some(template => template.ipfixScope === scope)) {
|
|
ipfixScopes.get(map)?.delete(scope)
|
|
}
|
|
fieldsByExporter.set(exporter, (fieldsByExporter.get(exporter) ?? 0) - retained.fields.length)
|
|
}
|
|
function dataTemplate(exporter: string, id: string, clock?: V9Clock, ipfixExportTime?: number, resetMessage?: object): Template {
|
|
const map = templatesByExporter.get(exporter)
|
|
if (map) expireTemplate(exporter, map, id, performance.now())
|
|
const retained = map?.get(id)
|
|
// The existing ingress catches FlowTemplateRefused, exposes an error and
|
|
// refuses publication. An undecodable datagram is not an observed zero.
|
|
if (!retained || retained.v9Conflicted || clock && retained.v9Clock && differentV9Boot(retained.v9Clock, clock))
|
|
refuseTemplate("missing-template")
|
|
if (clock && retained.v9ValidFrom && compareV9Clock(clock, retained.v9ValidFrom) < 0)
|
|
refuseTemplate("missing-template")
|
|
if (ipfixExportTime !== undefined && retained.ipfixValidFrom !== undefined && ipfixExportTime < retained.ipfixValidFrom)
|
|
refuseTemplate("missing-template")
|
|
const resetAt = retained.ipfixScope === undefined ? undefined : ipfixScopes.get(map!)?.get(retained.ipfixScope)?.resetAt
|
|
if (retained.ipfixConflicted) refuseTemplate("missing-template")
|
|
if (resetAt !== undefined) {
|
|
const fresh = retained.ipfixExportTime! > resetAt ||
|
|
retained.ipfixExportTime === resetAt && retained.ipfixResetMessage !== undefined
|
|
const inResetMessage = resetMessage !== undefined && retained.ipfixResetMessage === resetMessage
|
|
if (!fresh || ipfixExportTime! <= resetAt && !inResetMessage) refuseTemplate("missing-template")
|
|
}
|
|
return retained
|
|
}
|
|
|
|
function templatesForExporter(exporter: string): Map<string, Template> {
|
|
const existing = templatesByExporter.get(exporter)
|
|
if (existing) {
|
|
templatesByExporter.delete(exporter)
|
|
templatesByExporter.set(exporter, existing)
|
|
return existing
|
|
}
|
|
const created = new Map<string, Template>()
|
|
templatesByExporter.set(exporter, created)
|
|
fieldsByExporter.set(exporter, 0)
|
|
while (templatesByExporter.size > MAX_TEMPLATE_EXPORTERS) {
|
|
const oldest = templatesByExporter.keys().next().value
|
|
if (oldest == null || oldest === exporter) break
|
|
forgetExporter(oldest)
|
|
}
|
|
return created
|
|
}
|
|
|
|
/** Check retention budgets after bounded field staging and body validation.
|
|
* The per-template count is checked before staging. A rejected replacement leaves
|
|
* the complete old template and all accounting intact. No await splits commit. */
|
|
function admitTemplate(exporter: string, map: Map<string, Template>, id: string, count: number): void {
|
|
if (count > FLOW_TEMPLATE_LIMITS.perTemplateFields) refuseTemplate("fields")
|
|
const now = performance.now()
|
|
for (const retainedId of map.keys()) expireTemplate(exporter, map, retainedId, now)
|
|
const previous = map.get(id), added = previous ? 0 : 1, delta = count - (previous?.fields.length ?? 0)
|
|
// Preview eviction without changing any retained state until validation succeeds.
|
|
const victim = !templatesByExporter.has(exporter) && templatesByExporter.size >= MAX_TEMPLATE_EXPORTERS
|
|
? templatesByExporter.keys().next().value : undefined
|
|
const releasedTemplates = victim === undefined ? 0 : templatesByExporter.get(victim)?.size ?? 0
|
|
const releasedFields = victim === undefined ? 0 : fieldsByExporter.get(victim) ?? 0
|
|
if (map.size + added > FLOW_TEMPLATE_LIMITS.perExporterTemplates) refuseTemplate("exporter-templates")
|
|
if (templateCount - releasedTemplates + added > FLOW_TEMPLATE_LIMITS.templates) refuseTemplate("templates")
|
|
if ((fieldsByExporter.get(exporter) ?? 0) + delta > FLOW_TEMPLATE_LIMITS.perExporterFields) refuseTemplate("exporter-fields")
|
|
if (fieldCount - releasedFields + delta > FLOW_TEMPLATE_LIMITS.fields) refuseTemplate("total-fields")
|
|
}
|
|
function storeTemplate(exporter: string, map: Map<string, Template>, id: string, fields: FieldSpec[], v9Clock?: V9Clock, ipfix?: { scope: string; exportTime: number; resetMessage?: object }): void {
|
|
const previous = map.get(id), delta = fields.length - (previous?.fields.length ?? 0)
|
|
const ipfixExportTime = ipfix?.exportTime
|
|
const resetAt = ipfix && ipfixScopes.get(map)?.get(ipfix.scope)?.resetAt
|
|
if (v9Clock && previous?.v9Clock && previous.v9Clock.exportedAt > v9Clock.exportedAt) return
|
|
if (ipfixExportTime !== undefined && previous?.ipfixExportTime !== undefined && previous.ipfixExportTime > ipfixExportTime) return
|
|
if (ipfixExportTime !== undefined && resetAt !== undefined && ipfixExportTime <= resetAt && !ipfix?.resetMessage) return
|
|
const sameLayout = previous && delta === 0 && fields.every((field, index) =>
|
|
field.type === previous.fields[index].type && field.length === previous.fields[index].length)
|
|
if (v9Clock && previous?.v9Clock?.exportedAt === v9Clock.exportedAt) {
|
|
if (previous.v9Conflicted || previous.fields.length > 0 && fields.length > 0 &&
|
|
differentV9Boot(previous.v9Clock, v9Clock)) {
|
|
previous.v9Conflicted = true
|
|
return
|
|
}
|
|
if (compareV9Clock(v9Clock, previous.v9Clock) < 0) return
|
|
}
|
|
// Export Time has one-second resolution. Conflicting definitions within
|
|
// that second cannot be ordered using the message timestamp.
|
|
if (previous && ipfixExportTime !== undefined && previous.ipfixExportTime === ipfixExportTime &&
|
|
(!sameLayout || previous.ipfixConflicted)) {
|
|
previous.ipfixConflicted = true
|
|
return
|
|
}
|
|
if (!previous) templateCount++
|
|
fieldCount += delta; fieldsByExporter.set(exporter, (fieldsByExporter.get(exporter) ?? 0) + delta)
|
|
if (ipfix) {
|
|
let scopes = ipfixScopes.get(map)
|
|
if (!scopes) ipfixScopes.set(map, scopes = new Map())
|
|
if (!scopes.has(ipfix.scope)) scopes.set(ipfix.scope, {})
|
|
}
|
|
// A refresh of the same layout must not move the first compatible packet.
|
|
const v9ValidFrom = v9Clock && previous?.v9Clock
|
|
? sameLayout && !previous.v9Conflicted && !differentV9Boot(previous.v9Clock, v9Clock)
|
|
? previous.v9ValidFrom : v9Clock
|
|
: undefined
|
|
map.set(id, { fields, refreshedAtMs: performance.now(), v9Clock, v9ValidFrom, ipfixExportTime,
|
|
ipfixScope: ipfix?.scope,
|
|
ipfixResetMessage: ipfix?.resetMessage,
|
|
ipfixValidFrom: sameLayout && !previous.ipfixConflicted &&
|
|
(resetAt === undefined || previous.ipfixExportTime! > resetAt) ? previous.ipfixValidFrom : ipfixExportTime })
|
|
}
|
|
function templateFields(buf: Buffer, offset: number, end: number, count: number, enterprise: boolean) {
|
|
const fields: FieldSpec[] = []
|
|
for (let index = 0; index < count; index++) {
|
|
if (offset + 4 > end) refuseTemplate("truncated")
|
|
const type = buf.readUInt16BE(offset), length = buf.readUInt16BE(offset + 2)
|
|
offset += 4
|
|
if (enterprise && type & 0x8000) {
|
|
if (offset + 4 > end) refuseTemplate("truncated")
|
|
offset += 4
|
|
}
|
|
fields.push({ type: enterprise ? type & 0x7fff : type, length })
|
|
}
|
|
return { fields, next: offset }
|
|
}
|
|
|
|
function ipv4(buf: Buffer, offset: number): string {
|
|
return `${buf[offset]}.${buf[offset + 1]}.${buf[offset + 2]}.${buf[offset + 3]}`
|
|
}
|
|
|
|
function usableIpfixIp(ip: string): boolean {
|
|
const t = String(ip ?? "").trim()
|
|
if (!t) return false
|
|
return !isUnspecifiedIp(t)
|
|
}
|
|
|
|
function ipv6(buf: Buffer, offset: number): string {
|
|
const parts: string[] = []
|
|
for (let i = 0; i < 8; i++) parts.push(buf.readUInt16BE(offset + i * 2).toString(16))
|
|
return parts.join(":")
|
|
}
|
|
|
|
const VAR_LEN = 0xffff
|
|
|
|
function consumeField(
|
|
buf: Buffer,
|
|
off: number,
|
|
length: number,
|
|
limit: number,
|
|
): { data: Buffer; next: number } | null {
|
|
if (length === VAR_LEN) {
|
|
if (off >= limit) return null
|
|
const first = buf[off]!
|
|
if (first < 255) {
|
|
const end = off + 1 + first
|
|
if (end > limit) return null
|
|
return { data: buf.subarray(off + 1, end), next: end }
|
|
}
|
|
if (off + 3 > limit) return null
|
|
const len = buf.readUInt16BE(off + 1)
|
|
const end = off + 3 + len
|
|
if (end > limit) return null
|
|
return { data: buf.subarray(off + 3, end), next: end }
|
|
}
|
|
const end = off + length
|
|
if (end > limit) return null
|
|
return { data: buf.subarray(off, end), next: end }
|
|
}
|
|
|
|
function fixedRecordSize(fields: FieldSpec[]): number | null {
|
|
let n = 0
|
|
for (const f of fields) {
|
|
if (f.length === VAR_LEN) return null
|
|
n += f.length
|
|
}
|
|
return n
|
|
}
|
|
|
|
function readUint(buf: Buffer, offset: number, length: number): number {
|
|
if (length === 1) return buf.readUInt8(offset)
|
|
if (length === 2) return buf.readUInt16BE(offset)
|
|
if (length === 4) return buf.readUInt32BE(offset)
|
|
if (length === 8) {
|
|
const big = buf.readBigUInt64BE(offset)
|
|
const n = Number(big)
|
|
return Number.isFinite(n) ? n : 0
|
|
}
|
|
let v = 0
|
|
for (let i = 0; i < length; i++) v = (v << 8) + buf[offset + i]
|
|
return v >>> 0
|
|
}
|
|
|
|
function parseNetflowV5(buf: Buffer): ParsedFlow[] {
|
|
if (buf.length < 24) return []
|
|
const count = buf.readUInt16BE(2)
|
|
const out: ParsedFlow[] = []
|
|
let off = 24
|
|
for (let i = 0; i < count && off + 48 <= buf.length; i++) {
|
|
out.push(normalizeParsedFlow({
|
|
src: ipv4(buf, off),
|
|
dst: ipv4(buf, off + 4),
|
|
packets: buf.readUInt32BE(off + 16),
|
|
bytes: buf.readUInt32BE(off + 20),
|
|
srcPort: buf.readUInt16BE(off + 32),
|
|
dstPort: buf.readUInt16BE(off + 34),
|
|
proto: buf.readUInt8(off + 38),
|
|
inIface: String(buf.readUInt16BE(off + 12)),
|
|
outIface: String(buf.readUInt16BE(off + 14)),
|
|
}))
|
|
off += 48
|
|
}
|
|
return out
|
|
}
|
|
|
|
function parseIpfixTemplates(exporter: string, buf: Buffer, setStart: number, setEnd: number, setId: number, scope: string, exportTime: number, resetMessage?: object) {
|
|
let off = setStart + 4
|
|
let map: Map<string, Template> | undefined
|
|
while (off + 4 <= setEnd) {
|
|
const templateId = buf.readUInt16BE(off)
|
|
const fieldCount = buf.readUInt16BE(off + 2)
|
|
off += 4
|
|
// This collector is UDP-only: RFC7011 section8.4 requires ignoring
|
|
// withdrawals, including the four-byte Options withdrawal record.
|
|
if (fieldCount === 0) continue
|
|
if (fieldCount > FLOW_TEMPLATE_LIMITS.perTemplateFields) refuseTemplate("fields")
|
|
if (setId === 3) {
|
|
// options template: skip scope count
|
|
if (off + 2 > setEnd) refuseTemplate("truncated")
|
|
off += 2
|
|
}
|
|
const parsed = templateFields(buf, off, setEnd, fieldCount, true)
|
|
off = parsed.next
|
|
if (templateId >= 256) {
|
|
admitTemplate(exporter, map ?? templatesByExporter.get(exporter) ?? new Map(), scopedTemplateId(scope, templateId), fieldCount)
|
|
map ??= templatesForExporter(exporter)
|
|
storeTemplate(exporter, map, scopedTemplateId(scope, templateId), parsed.fields, undefined, { scope, exportTime, resetMessage })
|
|
}
|
|
}
|
|
}
|
|
|
|
function recordFromFields(
|
|
fields: FieldSpec[],
|
|
buf: Buffer,
|
|
offset: number,
|
|
limit: number,
|
|
): { flow: ParsedFlow; next: number } | null {
|
|
let off = offset
|
|
let src = ""
|
|
let dst = ""
|
|
let proto = 0
|
|
let srcPort = 0
|
|
let dstPort = 0
|
|
let bytes = 0
|
|
let packets = 0
|
|
let inIface = ""
|
|
let outIface = ""
|
|
let ifaceName = ""
|
|
let nextHop = ""
|
|
let flowStartMs = 0
|
|
let flowEndMs = 0
|
|
let natSrc = ""
|
|
let natDst = ""
|
|
let natSrcPort = 0
|
|
let natDstPort = 0
|
|
for (const f of fields) {
|
|
const field = consumeField(buf, off, f.length, limit)
|
|
if (!field) return null
|
|
const { data } = field
|
|
switch (f.type) {
|
|
case 8:
|
|
if (data.length === 4) {
|
|
const value = ipv4(data, 0)
|
|
if (usableIpfixIp(value) || !usableIpfixIp(src)) src = value
|
|
}
|
|
break
|
|
case 12:
|
|
if (data.length === 4) {
|
|
const value = ipv4(data, 0)
|
|
if (usableIpfixIp(value) || !usableIpfixIp(dst)) dst = value
|
|
}
|
|
break
|
|
case 27:
|
|
if (data.length === 16 && !usableIpfixIp(src)) src = ipv6(data, 0)
|
|
break
|
|
case 28:
|
|
if (data.length === 16 && !usableIpfixIp(dst)) dst = ipv6(data, 0)
|
|
break
|
|
case 15:
|
|
if (data.length === 4 && !nextHop) nextHop = ipv4(data, 0)
|
|
break
|
|
case 18:
|
|
if (data.length === 4 && !nextHop) nextHop = ipv4(data, 0)
|
|
break
|
|
case 62:
|
|
if (data.length === 16 && !nextHop) nextHop = ipv6(data, 0)
|
|
break
|
|
case 225:
|
|
if (data.length === 4) {
|
|
const value = ipv4(data, 0)
|
|
if (usableIpfixIp(value) || !usableIpfixIp(natSrc)) natSrc = value
|
|
}
|
|
break
|
|
case 226:
|
|
if (data.length === 4) {
|
|
const value = ipv4(data, 0)
|
|
if (usableIpfixIp(value) || !usableIpfixIp(natDst)) natDst = value
|
|
}
|
|
break
|
|
case 281:
|
|
if (data.length === 16 && !usableIpfixIp(natSrc)) {
|
|
const value = ipv6(data, 0)
|
|
if (usableIpfixIp(value)) natSrc = value
|
|
}
|
|
break
|
|
case 282:
|
|
if (data.length === 16 && !usableIpfixIp(natDst)) {
|
|
const value = ipv6(data, 0)
|
|
if (usableIpfixIp(value)) natDst = value
|
|
}
|
|
break
|
|
case 227:
|
|
natSrcPort = readUint(data, 0, data.length)
|
|
break
|
|
case 228:
|
|
natDstPort = readUint(data, 0, data.length)
|
|
break
|
|
case 4:
|
|
proto = readUint(data, 0, data.length)
|
|
break
|
|
case 7:
|
|
srcPort = readUint(data, 0, data.length)
|
|
break
|
|
case 11:
|
|
dstPort = readUint(data, 0, data.length)
|
|
break
|
|
case 1:
|
|
bytes = readUint(data, 0, data.length)
|
|
break
|
|
case 2:
|
|
packets = readUint(data, 0, data.length)
|
|
break
|
|
case 85:
|
|
if (!bytes) bytes = readUint(data, 0, data.length)
|
|
break
|
|
case 86:
|
|
if (!packets) packets = readUint(data, 0, data.length)
|
|
break
|
|
case 10:
|
|
inIface = String(readUint(data, 0, data.length))
|
|
break
|
|
case 14:
|
|
outIface = String(readUint(data, 0, data.length))
|
|
break
|
|
case 21:
|
|
if (!flowEndMs) flowEndMs = readUint(data, 0, data.length)
|
|
break
|
|
case 22:
|
|
if (!flowStartMs) flowStartMs = readUint(data, 0, data.length)
|
|
break
|
|
case 150:
|
|
if (!flowStartMs) flowStartMs = readUint(data, 0, data.length) * 1000
|
|
break
|
|
case 151:
|
|
if (!flowEndMs) flowEndMs = readUint(data, 0, data.length) * 1000
|
|
break
|
|
case 152:
|
|
flowStartMs = readUint(data, 0, data.length)
|
|
break
|
|
case 153:
|
|
flowEndMs = readUint(data, 0, data.length)
|
|
break
|
|
case 82:
|
|
ifaceName = data.toString("utf8").replace(/\0/g, "").trim()
|
|
break
|
|
default:
|
|
break
|
|
}
|
|
off = field.next
|
|
}
|
|
if (ifaceName && !inIface) inIface = ifaceName
|
|
if (natSrc && !usableIpfixIp(src) && usableIpfixIp(natSrc)) src = natSrc
|
|
if (natDst && !usableIpfixIp(dst) && usableIpfixIp(natDst)) dst = natDst
|
|
return {
|
|
flow: normalizeParsedFlow({
|
|
src, dst, proto, srcPort, dstPort, bytes, packets, inIface, outIface, nextHop, flowStartMs, flowEndMs, natSrc, natDst, natSrcPort, natDstPort,
|
|
}),
|
|
next: off,
|
|
}
|
|
}
|
|
|
|
function parseDataRecords(
|
|
tpl: Template,
|
|
buf: Buffer,
|
|
recOff: number,
|
|
setEnd: number,
|
|
out: ParsedFlow[],
|
|
) {
|
|
const size = fixedRecordSize(tpl.fields)
|
|
while (recOff + 1 < setEnd) {
|
|
if (size != null && recOff + size > setEnd) break
|
|
const parsed = recordFromFields(tpl.fields, buf, recOff, setEnd)
|
|
if (!parsed) break
|
|
if (parsed.flow.src || parsed.flow.dst) out.push(parsed.flow)
|
|
if (parsed.next <= recOff) break
|
|
recOff = parsed.next
|
|
}
|
|
}
|
|
|
|
function parseIpfix(buf: Buffer, exporter: string, scope: string): ParsedFlow[] {
|
|
if (buf.length < 16) return []
|
|
const total = buf.readUInt16BE(2)
|
|
const exportTime = buf.readUInt32BE(4)
|
|
const sequence = buf.readUInt32BE(8)
|
|
const map = templatesByExporter.get(exporter)
|
|
const state = map && ipfixScopes.get(map)?.get(scope)
|
|
const previous = state?.clock
|
|
const delta = previous ? (sequence - previous.sequence) >>> 0 : 0
|
|
const regressed = state && previous && exportTime > previous.exportTime && delta > 0x80000000
|
|
const end = Math.min(buf.length, total)
|
|
let resetMessage: object | undefined
|
|
if (regressed) {
|
|
for (let start = 16; start + 4 <= end;) {
|
|
const length = buf.readUInt16BE(start + 2)
|
|
if (length < 4 || start + length > end) break
|
|
if (buf.readUInt16BE(start) >= 256) {
|
|
resetMessage = {}
|
|
break
|
|
}
|
|
start += length
|
|
}
|
|
}
|
|
if (regressed && resetMessage) {
|
|
state.resetAt = exportTime
|
|
state.clock = undefined
|
|
}
|
|
let off = 16
|
|
const out: ParsedFlow[] = []
|
|
let hasData = false
|
|
while (off + 4 <= end) {
|
|
const setId = buf.readUInt16BE(off)
|
|
const setLen = buf.readUInt16BE(off + 2)
|
|
if (setLen < 4 || off + setLen > end) break
|
|
const setEnd = off + setLen
|
|
if (setId === 2 || setId === 3) {
|
|
parseIpfixTemplates(exporter, buf, off, setEnd, setId, scope, exportTime, resetMessage)
|
|
} else if (setId >= 256) {
|
|
const tpl = dataTemplate(exporter, scopedTemplateId(scope, setId), undefined, exportTime, resetMessage)
|
|
parseDataRecords(tpl, buf, off + 4, setEnd, out)
|
|
hasData = true
|
|
}
|
|
off = setEnd
|
|
}
|
|
const currentMap = templatesByExporter.get(exporter)
|
|
const current = currentMap && ipfixScopes.get(currentMap)?.get(scope)
|
|
if (hasData && current && (!current.clock ||
|
|
exportTime >= current.clock.exportTime && ((sequence - current.clock.sequence) >>> 0) < 0x80000000)) {
|
|
current.clock = { exportTime, sequence }
|
|
}
|
|
return out
|
|
}
|
|
|
|
function parseNetflowV9(buf: Buffer, exporter: string, scope: string): ParsedFlow[] {
|
|
if (buf.length < 20) return []
|
|
const count = buf.readUInt16BE(2)
|
|
const clock = { exportedAt: buf.readUInt32BE(8), uptime: buf.readUInt32BE(4), sequence: buf.readUInt32BE(12) }
|
|
let off = 20
|
|
const out: ParsedFlow[] = []
|
|
let map = templatesByExporter.get(exporter)
|
|
if (map) templatesForExporter(exporter)
|
|
for (let s = 0; s < count && off + 4 <= buf.length; s++) {
|
|
const setId = buf.readUInt16BE(off)
|
|
const setLen = buf.readUInt16BE(off + 2)
|
|
if (setLen < 4 || off + setLen > buf.length) break
|
|
const setEnd = off + setLen
|
|
if (setId === 0) {
|
|
let tOff = off + 4
|
|
while (tOff + 4 <= setEnd) {
|
|
const templateId = buf.readUInt16BE(tOff)
|
|
const fieldCount = buf.readUInt16BE(tOff + 2)
|
|
tOff += 4
|
|
// Preserve v9's existing zero-field replacement semantics. RFC7011's
|
|
// UDP withdrawal-ignore rule applies to IPFIX, not the v9 format.
|
|
if (fieldCount === 0) {
|
|
if (templateId >= 256) {
|
|
admitTemplate(exporter, map ?? new Map(), scopedTemplateId(scope, templateId), 0)
|
|
map ??= templatesForExporter(exporter); storeTemplate(exporter, map, scopedTemplateId(scope, templateId), [], clock)
|
|
}
|
|
continue
|
|
}
|
|
if (fieldCount > FLOW_TEMPLATE_LIMITS.perTemplateFields) refuseTemplate("fields")
|
|
const parsed = templateFields(buf, tOff, setEnd, fieldCount, false)
|
|
tOff = parsed.next
|
|
if (templateId >= 256) {
|
|
admitTemplate(exporter, map ?? new Map(), scopedTemplateId(scope, templateId), fieldCount)
|
|
map ??= templatesForExporter(exporter); storeTemplate(exporter, map, scopedTemplateId(scope, templateId), parsed.fields, clock)
|
|
}
|
|
}
|
|
if (map) templatesForExporter(exporter)
|
|
} else if (setId >= 256) {
|
|
const tpl = dataTemplate(exporter, scopedTemplateId(scope, setId), clock)
|
|
parseDataRecords(tpl, buf, off + 4, setEnd, out)
|
|
}
|
|
off = setEnd
|
|
}
|
|
return out
|
|
}
|
|
|
|
export function parseFlowPacket(buf: Buffer, exporterIp: string, transport?: FlowDatagramTransport): ParsedFlow[] {
|
|
if (buf.length < 2) return []
|
|
const version = buf.readUInt16BE(0)
|
|
if (version === 5) return parseNetflowV5(buf)
|
|
// Validate receiver context before ANY template access/LRU touch.
|
|
if (version === 9 && buf.length >= 20) return parseNetflowV9(buf, exporterIp, templateScope(version, buf.readUInt32BE(16), transport))
|
|
if (version === 10 && buf.length >= 16) return parseIpfix(buf, exporterIp, templateScope(version, buf.readUInt32BE(12), transport))
|
|
return []
|
|
}
|
|
|
|
export function protoName(proto: number): string {
|
|
switch (proto) {
|
|
case 1: return "ICMP"
|
|
case 6: return "TCP"
|
|
case 17: return "UDP"
|
|
case 47: return "GRE"
|
|
case 50: return "ESP"
|
|
case 58: return "ICMPv6"
|
|
case 89: return "OSPF"
|
|
default: return String(proto)
|
|
}
|
|
}
|
|
|
|
/** One collector socket per isolate. Call only after invalidating its predecessor's
|
|
* callbacks; parsed flow rows and owned SQL batches do not retain templates. */
|
|
export function clearFlowTemplateCache(): void {
|
|
templatesByExporter.clear()
|
|
fieldsByExporter.clear(); templateCount = 0; fieldCount = 0
|
|
}
|
|
export function resetFlowTemplatesForTests(): void {
|
|
clearFlowTemplateCache(); refusedTemplates = 0
|
|
}
|
|
|
|
export function templateCacheUsageForTests(exporter?: string) {
|
|
return { exporters: templatesByExporter.size, templates: templateCount, fields: fieldCount, refusedTemplates,
|
|
exporterTemplates: exporter === undefined ? null : templatesByExporter.get(exporter)?.size ?? 0,
|
|
exporterFields: exporter === undefined ? null : fieldsByExporter.get(exporter) ?? 0 }
|
|
}
|
|
|
|
export function templateExporterCountForTests(): number {
|
|
return templatesByExporter.size
|
|
}
|