feat(netflow): локальные GeoLite2-базы для стран и ASN потоков
Docker images / prepare-release (push) Successful in 13s
Docker images / backend-test (push) Successful in 2m13s
Docker images / frontend-image (push) Successful in 4m31s
Docker images / updater-image (push) Successful in 48s
Docker images / backend-image (push) Successful in 3m1s
Docker images / notify-webhook (push) Skipped
Docker images / publish-release (push) Successful in 15s

Lookup страны/ASN при ingest теперь идёт сначала по локальным mmdb
MaxMind GeoLite2 (Country + ASN, зеркало P3TERX, без ключей), с
мгновенным синхронным ответом для IPv4/IPv6; RIPEstat остаётся
fallback до первой загрузки баз и при промахе.

- backend/src/services/traffic-flow-geoip.ts: ридеры maxmind, фасад
  resolveFlowIp (geoip-first → RIPE-кэш), статус ридеров
- geoip-update-collector.ts: conditional GET по ETag, валидация пробоем
  8.8.8.8 (US / AS15169), атомарная подмена с .prev-откатом
- джоба планировщика geoip_update (по умолчанию раз в 7 дней),
  настройки geoip_settings + миграция 0004, API /api/geoip
  (GET/PUT/update), контракты @mmapp/contracts/geoip
- engine/analytics/map-hops переведены на resolveFlowIp
- секция «GeoIP-базы» в настройках NetFlow, метки джобы на странице
  сбора данных, тесты test:geoip
This commit is contained in:
Denozordec
2026-09-09 23:33:31 +07:00
parent d39e3454aa
commit a80caf5676
25 changed files with 1114 additions and 23 deletions
+100
View File
@@ -0,0 +1,100 @@
import { eq } from "drizzle-orm"
import { db } from "../db/index.js"
import { geoipSettings } from "../db/schema.js"
import type { GeoipSettingsDto, GeoipSettingsPatch } from "@mmapp/contracts/geoip"
const SETTINGS_ID = 1
const DEFAULT_INTERVAL_SEC = 604800
let dbEnabled = true
/** Тесты без PostgreSQL: геттеры отдают дефолты, touch/update — no-op. */
export function disableGeoipDbForTests(): void {
dbEnabled = false
}
export function resetGeoipSettingsForTests(): void {
dbEnabled = true
}
type GeoipSettingsRow = typeof geoipSettings.$inferSelect
async function getGeoipSettingsRow(): Promise<GeoipSettingsRow | undefined> {
if (!dbEnabled) return undefined
return (
(await db.select().from(geoipSettings).where(eq(geoipSettings.id, SETTINGS_ID)).limit(1))[0]
)
}
function toDto(row: GeoipSettingsRow | undefined): GeoipSettingsDto {
return {
enabled: row?.enabled ?? true,
updateIntervalSec: row?.updateIntervalSec ?? DEFAULT_INTERVAL_SEC,
lastCheckAt: row?.lastCheckAt ?? null,
lastSuccessAt: row?.lastSuccessAt ?? null,
lastError: row?.lastError ?? null,
countryBuildAt: row?.countryBuildAt ?? null,
asnBuildAt: row?.asnBuildAt ?? null,
updatedAt: row?.updatedAt ?? new Date().toISOString(),
}
}
export async function getGeoipSettings(): Promise<GeoipSettingsDto> {
return toDto(await getGeoipSettingsRow())
}
/** ETag'и зеркала для conditional GET (ключ — имя файла базы). */
export async function getGeoipEtags(): Promise<Record<string, string>> {
const row = await getGeoipSettingsRow()
if (!row) return {}
const raw = row?.etagsJson
if (!raw || typeof raw !== "object") return {}
return Object.fromEntries(
Object.entries(raw as Record<string, unknown>).filter(
(entry): entry is [string, string] => typeof entry[1] === "string",
),
)
}
export async function updateGeoipSettings(patch: GeoipSettingsPatch): Promise<GeoipSettingsDto> {
const prev = await getGeoipSettingsRow()
const next = {
enabled: patch.enabled ?? prev?.enabled ?? true,
updateIntervalSec: patch.updateIntervalSec ?? prev?.updateIntervalSec ?? DEFAULT_INTERVAL_SEC,
updatedAt: new Date().toISOString(),
}
if (prev) {
await db.update(geoipSettings).set(next).where(eq(geoipSettings.id, SETTINGS_ID))
} else {
await db.insert(geoipSettings).values({ id: SETTINGS_ID, ...next })
}
return getGeoipSettings()
}
export async function touchGeoipRunMeta(patch: {
lastCheckAt?: string
lastSuccessAt?: string | null
lastError?: string | null
countryBuildAt?: string | null
asnBuildAt?: string | null
etags?: Record<string, string>
}): Promise<void> {
const prev = await getGeoipSettingsRow()
const set: Partial<typeof geoipSettings.$inferInsert> = {
updatedAt: new Date().toISOString(),
}
if (patch.lastCheckAt !== undefined) set.lastCheckAt = patch.lastCheckAt
if (patch.lastSuccessAt !== undefined) set.lastSuccessAt = patch.lastSuccessAt
if (patch.lastError !== undefined) set.lastError = patch.lastError
if (patch.countryBuildAt !== undefined) set.countryBuildAt = patch.countryBuildAt
if (patch.asnBuildAt !== undefined) set.asnBuildAt = patch.asnBuildAt
if (patch.etags !== undefined) {
const prevEtags = (prev?.etagsJson as Record<string, string> | null) ?? {}
set.etagsJson = { ...prevEtags, ...patch.etags }
}
if (prev) {
await db.update(geoipSettings).set(set).where(eq(geoipSettings.id, SETTINGS_ID))
} else {
await db.insert(geoipSettings).values({ id: SETTINGS_ID, ...set })
}
}
@@ -0,0 +1,198 @@
import { rename, rm, mkdir, writeFile } from "node:fs/promises"
import path from "node:path"
import { open, type AsnResponse, type CountryResponse } from "maxmind"
import type { GeoipUpdateRunSnapshot } from "../types/scheduler-run-snapshot.js"
import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js"
import { getGeoipEtags, getGeoipSettings, touchGeoipRunMeta } from "./geoip-settings.js"
import {
GEOIP_ASN_FILE,
GEOIP_COUNTRY_FILE,
geoipDir,
reloadGeoipReaders,
} from "./traffic-flow-geoip.js"
/** Зеркало GeoLite2 без регистрации и ключей (см. README: GeoIP). */
const MIRROR_BASE = "https://github.com/P3TERX/GeoLite.mmdb/raw/download"
const DOWNLOAD_TIMEOUT_MS = 120_000
/** Пробный IP для валидации скачанной базы: Google DNS. */
const PROBE_IP = "8.8.8.8"
type GeoipDbKind = "country" | "asn"
let updating = false
let fetchImpl: typeof fetch = globalThis.fetch.bind(globalThis)
export function getGeoipUpdateState(): { running: boolean } {
return { running: updating }
}
async function validateCountryFile(filePath: string): Promise<string> {
const reader = await open<CountryResponse>(filePath)
const rec = reader.get(PROBE_IP)
const iso = rec?.country?.iso_code ?? rec?.registered_country?.iso_code ?? ""
if (iso !== "US") {
throw new Error(`база Country не распознала ${PROBE_IP} как US (${iso || "нет записи"})`)
}
return reader.metadata.buildEpoch.toISOString()
}
async function validateAsnFile(filePath: string): Promise<string> {
const reader = await open<AsnResponse>(filePath)
const rec = reader.get(PROBE_IP)
const asn = rec?.autonomous_system_number ?? 0
if (asn !== 15169) {
throw new Error(`база ASN не распознала ${PROBE_IP} как AS15169 (${asn ? `AS${asn}` : "нет записи"})`)
}
return reader.metadata.buildEpoch.toISOString()
}
let validateCountry = validateCountryFile
let validateAsn = validateAsnFile
export function setGeoipFetchForTests(fn: typeof fetch): void {
fetchImpl = fn
}
export function setGeoipValidateForTests(opts: {
country?: (filePath: string) => Promise<string>
asn?: (filePath: string) => Promise<string>
}): void {
validateCountry = opts.country ?? validateCountryFile
validateAsn = opts.asn ?? validateAsnFile
}
export function resetGeoipUpdateForTests(): void {
updating = false
fetchImpl = globalThis.fetch.bind(globalThis)
validateCountry = validateCountryFile
validateAsn = validateAsnFile
}
/**
* Разовая проверка/доставка баз с зеркала P3TERX. Conditional GET по ETag
* (304 = не меняем файл), валидация пробоем 8.8.8.8, атомарная подмена через rename.
*/
export async function collectGeoipUpdateOnce(
opts: { force?: boolean } = {},
): Promise<GeoipUpdateRunSnapshot> {
const sampledAt = new Date().toISOString()
if (updating) {
if (opts.force) {
throw Object.assign(new Error("Обновление GeoIP уже выполняется"), { statusCode: 409 })
}
return emptySnapshot(sampledAt, true)
}
const settings = await getGeoipSettings()
if (!settings.enabled && !opts.force) {
return emptySnapshot(sampledAt, true)
}
updating = true
const snapshot: GeoipUpdateRunSnapshot = {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "geoip_update",
sampledAt,
checked: 0,
downloaded: 0,
skippedUnchanged: 0,
bytes: 0,
errors: [],
}
try {
const dir = geoipDir()
await mkdir(dir, { recursive: true })
const storedEtags = await getGeoipEtags()
const etags: Record<string, string> = {}
const buildAt: Partial<Record<GeoipDbKind, string>> = {}
for (const kind of ["country", "asn"] as const) {
snapshot.checked += 1
const file = kind === "country" ? GEOIP_COUNTRY_FILE : GEOIP_ASN_FILE
const target = path.join(dir, file)
const tmp = `${target}.tmp`
const prevEtag = storedEtags[file]
try {
const ac = new AbortController()
const timer = setTimeout(() => ac.abort(), DOWNLOAD_TIMEOUT_MS)
let res: Response
try {
res = await fetchImpl(`${MIRROR_BASE}/${file}`, {
headers: prevEtag ? { "If-None-Match": prevEtag } : {},
signal: ac.signal,
})
} finally {
clearTimeout(timer)
}
if (res.status === 304) {
snapshot.skippedUnchanged += 1
if (prevEtag) etags[file] = prevEtag
continue
}
if (!res.ok) throw new Error(`HTTP ${res.status}`)
const etag = res.headers.get("etag") ?? ""
const body = Buffer.from(await res.arrayBuffer())
snapshot.bytes += body.byteLength
await writeFile(tmp, body)
buildAt[kind] =
kind === "country" ? await validateCountry(tmp) : await validateAsn(tmp)
const prevFile = `${target}.prev`
await rm(prevFile, { force: true })
await rename(target, prevFile).catch(() => {
/* текущего файла могло ещё не быть */
})
await rename(tmp, target)
snapshot.downloaded += 1
if (etag) etags[file] = etag
} catch (e) {
await rm(tmp, { force: true }).catch(() => {
/* best-effort */
})
const message = e instanceof Error ? e.message : String(e)
snapshot.errors.push(`${file}: ${message}`)
}
}
if (snapshot.downloaded > 0) {
await reloadGeoipReaders()
}
await touchGeoipRunMeta({
lastCheckAt: sampledAt,
lastSuccessAt: snapshot.errors.length ? null : sampledAt,
lastError: snapshot.errors.length ? snapshot.errors.join("; ") : null,
countryBuildAt: buildAt.country,
asnBuildAt: buildAt.asn,
etags,
})
return snapshot
} catch (e) {
const message = e instanceof Error ? e.message : String(e)
snapshot.fatalError = message
await touchGeoipRunMeta({
lastCheckAt: sampledAt,
lastError: message,
}).catch(() => {
/* best-effort */
})
return snapshot
} finally {
updating = false
}
}
function emptySnapshot(sampledAt: string, skipped: boolean): GeoipUpdateRunSnapshot {
return {
v: SCHEDULER_RUN_SNAPSHOT_VERSION,
job: "geoip_update",
sampledAt,
skipped,
checked: 0,
downloaded: 0,
skippedUnchanged: 0,
bytes: 0,
errors: [],
}
}
+20
View File
@@ -46,6 +46,8 @@ import { collectCertificatesRenewOnce } from "./certificate-renew-collector.js"
import { getCertificateRenewSettings } from "./certificates-service.js"
import { collectScheduledBackupsOnce } from "./backup-scheduler-collector.js"
import { getBackupScheduleSettings } from "./backup-service.js"
import { collectGeoipUpdateOnce } from "./geoip-update-collector.js"
import { getGeoipSettings } from "./geoip-settings.js"
import {
endSchedulerJob,
isSchedulerJobRunning,
@@ -63,6 +65,7 @@ export const JOB_KEYS = [
"gre_bgp",
"certificates_renew",
"backups",
"geoip_update",
"alert_engine",
] as const
export type SchedulerJobKey = (typeof JOB_KEYS)[number]
@@ -148,6 +151,9 @@ async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise<void> {
case "backups":
snapshot = await collectScheduledBackupsOnce()
break
case "geoip_update":
snapshot = await collectGeoipUpdateOnce()
break
case "alert_engine": {
const r = await runAlertEngineOnce()
snapshot = {
@@ -377,6 +383,18 @@ export async function refreshScheduler(): Promise<void> {
)
}
const geoip = await getGeoipSettings()
if (geoip.enabled) {
const geoipMs = Math.max(6 * 3600_000, geoip.updateIntervalSec * 1000)
void executeSchedulerJob("geoip_update").catch(() => {})
timers.set(
"geoip_update",
setInterval(() => {
void executeSchedulerJob("geoip_update").catch(() => {})
}, geoipMs),
)
}
const alertMs = 20_000
void executeSchedulerJob("alert_engine").catch(() => {})
timers.set(
@@ -409,6 +427,7 @@ export async function getSchedulerStatus() {
const internetPath = await getInternetPathSettings()
const certRenew = await getCertificateRenewSettings()
const backupSchedule = await getBackupScheduleSettings()
const geoip = await getGeoipSettings()
const resOn = uptime.resourcesEnabled ?? uptime.enabled
const pingOn = uptime.pingEnabled ?? uptime.enabled
@@ -424,6 +443,7 @@ export async function getSchedulerStatus() {
gre_bgp: { enabled: true, intervalSec: 30 },
certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec },
backups: { enabled: backupSchedule.enabled, intervalSec: 60 },
geoip_update: { enabled: geoip.enabled, intervalSec: geoip.updateIntervalSec },
alert_engine: { enabled: true, intervalSec: 20 },
}
@@ -26,7 +26,8 @@ import { resolveIfaceName } from "./traffic-flow-ifaces.js"
import { getTrafficFlowSettingsRow, listHostPeers } from "./traffic-flow-settings.js"
import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js"
import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js"
import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js"
import { enqueueRipeMisses } from "./traffic-flow-ripe.js"
import { resolveFlowIp } from "./traffic-flow-geoip.js"
import { classifyFlowDst, refreshFlowCatalogInBackground } from "./traffic-flow-classify.js"
import { isIsoCountry } from "./traffic-flow-brands.js"
import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js"
@@ -255,7 +256,7 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery): Promise<FlowAn
const peer = pickInternetPeer(r.src, r.dst, r.srcPort, r.dstPort)
peers.add(peer)
const app = applicationName(r.proto, r.dstPort, r.srcPort)
const ripe = lookupRipeCached(peer)
const ripe = resolveFlowIp(peer)
const classified = classifyFlowDst(peer, r.proto, r.dstPort, r.srcPort, ripe)
bump(applications, app, r.bytes, r.packets)
bump(protocols, protoName(r.proto), r.bytes, r.packets)
+25 -2
View File
@@ -5,7 +5,8 @@ import { classifyFlowPlaneLite } from "./traffic-flow-planes.js"
import { pickServerIdForExporter, type OverlayPeerRef } from "./traffic-flow-map-exporter.js"
import { applicationName } from "./traffic-flow-apps.js"
import { classifyFlowDst } from "./traffic-flow-classify.js"
import { enqueueRipeMisses, lookupRipeCached, pruneRipeSqlite } from "./traffic-flow-ripe.js"
import { enqueueRipeMisses, pruneRipeSqlite } from "./traffic-flow-ripe.js"
import { resolveFlowIp } from "./traffic-flow-geoip.js"
import { invalidateTrafficFlowSettingsCache } from "./traffic-flow-settings.js"
import { isIsoCountry } from "./traffic-flow-brands.js"
import { maybeRefreshIfaces } from "./traffic-flow-ifaces.js"
@@ -333,7 +334,7 @@ export function queueParsedFlows(serverId: number, flows: ParsedFlowInput[]): vo
addToTick(serverId, flow, flow.bytes)
bumpRollup(serverId, bucketAt, flow, flow.bytes, flow.packets)
const peer = pickInternetPeer(flow.src, flow.dst, flow.srcPort, flow.dstPort)
const ripe = lookupRipeCached(peer)
const ripe = resolveFlowIp(peer)
if (peer && !ripe) ripeMisses.push(peer)
const classified = classifyFlowDst(peer, flow.proto, flow.dstPort, flow.srcPort, ripe)
const app = applicationName(flow.proto, flow.dstPort, flow.srcPort)
@@ -939,3 +940,25 @@ export function pendingSizeForTests(): number {
export function droppedForTests(): number {
return dropped
}
/** Снимок минутных dims (dim → key → bytes) для тестов обогащения потоков. */
export function minuteDimsSnapshotForTests(): Map<string, Map<string, { bytes: number; packets: number }>> {
const out = new Map<string, Map<string, { bytes: number; packets: number }>>()
for (const [k, acc] of minuteDims) {
// dimKey: serverId\0bucketAt\0dim\0key
const parts = k.split("\0")
const dim = parts[2] ?? ""
const key = parts.slice(3).join("\0")
let byKey = out.get(dim)
if (!byKey) {
byKey = new Map()
out.set(dim, byKey)
}
const prev = byKey.get(key)
byKey.set(key, {
bytes: (prev?.bytes ?? 0) + acc.bytes,
packets: (prev?.packets ?? 0) + acc.packets,
})
}
return out
}
@@ -0,0 +1,186 @@
import assert from "node:assert/strict"
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"
import { tmpdir } from "node:os"
import path from "node:path"
import type { AsnResponse, CountryResponse, Reader } from "maxmind"
import {
disableRipeEnqueueForTests,
disableRipePersistForTests,
resetRipeCacheForTests,
seedRipeCacheForTests,
} from "./traffic-flow-ripe.js"
import {
lookupGeoip,
resetGeoipForTests,
resolveFlowIp,
setGeoipReadersForTests,
} from "./traffic-flow-geoip.js"
import {
resetEngineForTests,
ingestParsedFlowsForServerForTests,
minuteDimsSnapshotForTests,
} from "./traffic-flow-engine.js"
import { classifyFlowDst } from "./traffic-flow-classify.js"
import { disableGeoipDbForTests } from "./geoip-settings.js"
import {
collectGeoipUpdateOnce,
resetGeoipUpdateForTests,
setGeoipFetchForTests,
setGeoipValidateForTests,
} from "./geoip-update-collector.js"
disableRipePersistForTests()
disableRipeEnqueueForTests()
resetRipeCacheForTests()
resetGeoipForTests()
// ── lookupGeoip: приватные IP → negative без ридеров ──────────────────────────
assert.equal(lookupGeoip("10.1.1.8")?.ok, false)
assert.equal(lookupGeoip("192.168.0.1")?.prefix, "192.168.0.1/32")
assert.equal(lookupGeoip("100.64.1.2")?.ok, false)
assert.equal(lookupGeoip("fe80::1")?.prefix, "fe80::1/128")
// ── без ридеров публичный IP → null, resolveFlowIp уходит в RIPE-кэш ─────────
assert.equal(lookupGeoip("1.2.3.10"), null)
seedRipeCacheForTests({
prefix: "1.2.3.0/24",
asn: 64500,
country: "NL",
lat: null,
lng: null,
holder: "TEST",
ok: true,
fetchedAt: Date.now(),
})
assert.equal(resolveFlowIp("1.2.3.10")?.country, "NL")
assert.equal(resolveFlowIp("1.2.3.10")?.asn, 64500)
// ── fake-ридеры: geoip приоритетнее RIPE ──────────────────────────────────────
function fakeCountryReader(byIp: Record<string, string>): Reader<CountryResponse> {
return {
get(ip: string) {
const iso = byIp[ip]
return iso ? ({ country: { iso_code: iso } } as CountryResponse) : null
},
metadata: { buildEpoch: new Date("2026-09-02T00:00:00Z") },
} as unknown as Reader<CountryResponse>
}
function fakeAsnReader(byIp: Record<string, { asn: number; org: string }>): Reader<AsnResponse> {
return {
get(ip: string) {
const hit = byIp[ip]
return hit
? ({ autonomous_system_number: hit.asn, autonomous_system_organization: hit.org } as AsnResponse)
: null
},
metadata: { buildEpoch: new Date("2026-09-02T00:00:00Z") },
} as unknown as Reader<AsnResponse>
}
setGeoipReadersForTests({
country: fakeCountryReader({ "8.8.8.8": "US", "6.6.6.6": "EU" }),
asn: fakeAsnReader({
"8.8.8.8": { asn: 15169, org: "GOOGLE" },
"6.6.6.6": { asn: 15169, org: "GOOGLE" },
}),
})
const hit = resolveFlowIp("8.8.8.8")
assert.equal(hit?.country, "US")
assert.equal(hit?.asn, 15169)
assert.equal(hit?.holder, "GOOGLE")
assert.equal(hit?.ok, true)
// 1.2.3.10 в fake-ридерах нет — по-прежнему из RIPE-кэша
assert.equal(resolveFlowIp("1.2.3.10")?.asn, 64500)
// EU не ISO-страна: отфильтрована, страна выведена из ASN (HQ Google → US)
assert.equal(lookupGeoip("6.6.6.6")?.country, "US")
// geoip-мета совместима с classifyFlowDst (бренд по ASN 15169)
const classified = classifyFlowDst("8.8.8.8", 6, 443, 51504, hit)
assert.equal(classified.service, "Google")
// ── движок: dims country/asn наполняются из geoip-ридеров ────────────────────
resetEngineForTests()
ingestParsedFlowsForServerForTests(1, [{
src: "192.168.88.10",
dst: "8.8.8.8",
proto: 6,
srcPort: 51504,
dstPort: 443,
bytes: 1000,
packets: 10,
inIface: "wg-flow",
outIface: "",
nextHop: "",
flowStartMs: 0,
flowEndMs: 0,
natSrc: "",
natDst: "",
}])
const dims = minuteDimsSnapshotForTests()
assert.equal(dims.get("country")?.get("US")?.bytes, 1000)
assert.equal(dims.get("asn")?.get("15169")?.bytes, 1000)
// ── коллектор: 304 → обе базы без изменений ───────────────────────────────────
disableGeoipDbForTests()
resetGeoipUpdateForTests()
const geoipDir = mkdtempSync(path.join(tmpdir(), "mm-geoip-test-"))
process.env.GEOIP_DIR = geoipDir
setGeoipFetchForTests(async () => new Response(null, { status: 304 }))
let snap = await collectGeoipUpdateOnce({ force: true })
assert.equal(snap.skippedUnchanged, 2)
assert.equal(snap.downloaded, 0)
assert.equal(existsSync(path.join(geoipDir, "GeoLite2-Country.mmdb")), false)
// ── коллектор: 200 + валидация ok → подмена, старый файл в .prev ─────────────
const countryPath = path.join(geoipDir, "GeoLite2-Country.mmdb")
const asnPath = path.join(geoipDir, "GeoLite2-ASN.mmdb")
writeFileSync(countryPath, "old-country")
setGeoipFetchForTests(async () =>
new Response(new Uint8Array([1, 2, 3]), { status: 200, headers: { etag: '"v1"' } }))
setGeoipValidateForTests({
country: async (p) => {
assert.ok(p.endsWith(".tmp"), "валидация должна идти по tmp-файлу")
return "2026-09-08T00:00:00.000Z"
},
asn: async () => "2026-09-08T00:00:00.000Z",
})
snap = await collectGeoipUpdateOnce({ force: true })
assert.equal(snap.downloaded, 2)
assert.equal(snap.errors.length, 0)
assert.deepEqual(readFileSync(countryPath), Buffer.from([1, 2, 3]))
assert.equal(readFileSync(`${countryPath}.prev`, "utf8"), "old-country")
assert.equal(existsSync(`${asnPath}.prev`), false, "prev у asn не бывает при первой загрузке")
assert.equal(existsSync(`${countryPath}.tmp`), false)
// ── коллектор: битая база → подмены нет, старый файл цел, tmp удалён ─────────
writeFileSync(asnPath, "good-asn")
setGeoipFetchForTests(async () =>
new Response(new Uint8Array([9, 9]), { status: 200 }))
setGeoipValidateForTests({
country: async () => {
throw new Error("битая база")
},
asn: async () => {
throw new Error("битая база")
},
})
snap = await collectGeoipUpdateOnce({ force: true })
assert.equal(snap.downloaded, 0)
assert.equal(snap.errors.length, 2)
assert.deepEqual(readFileSync(countryPath), Buffer.from([1, 2, 3]), "country не тронута")
assert.equal(readFileSync(asnPath, "utf8"), "good-asn", "asn не тронут")
assert.equal(existsSync(`${countryPath}.tmp`), false)
assert.equal(existsSync(`${asnPath}.tmp`), false)
rmSync(geoipDir, { recursive: true, force: true })
delete process.env.GEOIP_DIR
resetGeoipUpdateForTests()
resetGeoipForTests()
console.log("traffic-flow-geoip.test.ts: ok")
+162
View File
@@ -0,0 +1,162 @@
import { existsSync } from "node:fs"
import path from "node:path"
import { open, type AsnResponse, type CountryResponse, type Reader } from "maxmind"
import { isNonPublicIp } from "./traffic-flow-ip.js"
import { isIsoCountry, resolveRipeCountry } from "./traffic-flow-brands.js"
import { lookupRipeCached, type FlowIpMeta } from "./traffic-flow-ripe.js"
export const GEOIP_COUNTRY_FILE = "GeoLite2-Country.mmdb"
export const GEOIP_ASN_FILE = "GeoLite2-ASN.mmdb"
/** Каталог баз: `storage/geoip` рядом со storage/backups; переопределяется GEOIP_DIR. */
export function geoipDir(): string {
return path.resolve(process.env.GEOIP_DIR ?? path.join(process.cwd(), "storage", "geoip"))
}
export function geoipCountryPath(): string {
return path.join(geoipDir(), GEOIP_COUNTRY_FILE)
}
export function geoipAsnPath(): string {
return path.join(geoipDir(), GEOIP_ASN_FILE)
}
export interface GeoipReaders {
country: Reader<CountryResponse> | null
asn: Reader<AsnResponse> | null
}
let readers: GeoipReaders = { country: null, asn: null }
let initPromise: Promise<GeoipReaders> | null = null
/** Открывает оба файла best-effort: отсутствующий/битый файл не мешает второму. */
export async function openGeoipReaders(dir = geoipDir()): Promise<GeoipReaders> {
const next: GeoipReaders = { country: null, asn: null }
if (existsSync(path.join(dir, GEOIP_COUNTRY_FILE))) {
try {
next.country = await open<CountryResponse>(path.join(dir, GEOIP_COUNTRY_FILE))
} catch {
/* битый файл — работаем без country */
}
}
if (existsSync(path.join(dir, GEOIP_ASN_FILE))) {
try {
next.asn = await open<AsnResponse>(path.join(dir, GEOIP_ASN_FILE))
} catch {
/* битый файл — работаем без ASN */
}
}
return next
}
/** Открывает ридеры при старте; файлы есть — работают, нет — lookup уходит в RIPE-fallback. */
export async function initGeoip(): Promise<GeoipReaders> {
if (!initPromise) {
initPromise = openGeoipReaders().then((next) => {
readers = next
return next
})
}
return initPromise
}
/** Переоткрывает ридеры после обновления файлов (атомарная замена ссылок). */
export async function reloadGeoipReaders(): Promise<GeoipReaders> {
const next = await openGeoipReaders()
readers = next
initPromise = Promise.resolve(next)
return next
}
export function setGeoipReadersForTests(next: Partial<GeoipReaders>): void {
readers = { country: next.country ?? null, asn: next.asn ?? null }
}
export function resetGeoipForTests(): void {
readers = { country: null, asn: null }
initPromise = null
}
function negativeMeta(ip: string): FlowIpMeta {
const v6 = ip.includes(":")
return {
prefix: `${ip}/${v6 ? 128 : 32}`,
asn: 0,
country: "—",
lat: null,
lng: null,
holder: "",
ok: false,
fetchedAt: Date.now(),
}
}
function safeCountryIso(reader: Reader<CountryResponse>, ip: string): string {
try {
const rec = reader.get(ip)
return rec?.country?.iso_code ?? rec?.registered_country?.iso_code ?? ""
} catch {
return ""
}
}
function safeAsn(reader: Reader<AsnResponse>, ip: string): { asn: number; holder: string } {
try {
const rec = reader.get(ip)
return {
asn: rec?.autonomous_system_number ?? 0,
holder: rec?.autonomous_system_organization ?? "",
}
} catch {
return { asn: 0, holder: "" }
}
}
/**
* Синхронный lookup по локальным GeoLite2. Возвращает FlowIpMeta в семантике RIPE-кэша
* (ok=true когда есть страна или ASN; null — данных нет, пусть пробует RIPE).
*/
export function lookupGeoip(ip: string): FlowIpMeta | null {
const trimmed = String(ip ?? "").trim()
if (!trimmed) return null
if (isNonPublicIp(trimmed)) return negativeMeta(trimmed)
const { country: countryReader, asn: asnReader } = readers
if (!countryReader && !asnReader) return null
const iso = countryReader ? safeCountryIso(countryReader, trimmed) : ""
const country = iso && isIsoCountry(iso) ? iso : ""
const { asn, holder } = asnReader ? safeAsn(asnReader, trimmed) : { asn: 0, holder: "" }
if (!asn && !country) return null
return {
prefix: `${trimmed}/${trimmed.includes(":") ? 128 : 32}`,
asn,
country: resolveRipeCountry(country, asn, holder) || "—",
lat: null,
lng: null,
holder,
ok: true,
fetchedAt: Date.now(),
}
}
/** Главный вход для потребителей пайплайна: локальные базы первыми, RIPE-кэш fallback. */
export function resolveFlowIp(ip: string): FlowIpMeta | null {
return lookupGeoip(ip) ?? lookupRipeCached(ip)
}
export interface GeoipReadersStatus {
countryLoaded: boolean
asnLoaded: boolean
countryBuildAt: string | null
asnBuildAt: string | null
dir: string
}
export function geoipReadersStatus(): GeoipReadersStatus {
return {
countryLoaded: Boolean(readers.country),
asnLoaded: Boolean(readers.asn),
countryBuildAt: readers.country?.metadata.buildEpoch.toISOString() ?? null,
asnBuildAt: readers.asn?.metadata.buildEpoch.toISOString() ?? null,
dir: geoipDir(),
}
}
@@ -13,7 +13,8 @@ import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-inge
import { resolveIfaceName } from "./traffic-flow-ifaces.js"
import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js"
import { pickInternetPeer } from "./traffic-flow-ip.js"
import { lookupRipeCached, type FlowIpMeta } from "./traffic-flow-ripe.js"
import { type FlowIpMeta } from "./traffic-flow-ripe.js"
import { resolveFlowIp } from "./traffic-flow-geoip.js"
import { getTrafficFlowSettingsRow } from "./traffic-flow-settings.js"
import { loadFlowTopology, resolveClient, resolveEn, getServerCatalog } from "./traffic-flow-topology.js"
import { flowDataEpoch } from "./traffic-flow-engine.js"
@@ -394,7 +395,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number
}
for (const [dst, acc] of dstAcc) {
const ripe = lookupRipeCached(dst)
const ripe = resolveFlowIp(dst)
const classified = classifyMapDstLite(dst, acc.proto, acc.dstPort, acc.srcPort, ripe)
if (!classified) continue
const toId = mapServiceNodeId(classified.service)