From a80caf5676c301c668cbc282235713335a59e4f1 Mon Sep 17 00:00:00 2001 From: Denozordec Date: Wed, 9 Sep 2026 23:33:31 +0700 Subject: [PATCH] =?UTF-8?q?feat(netflow):=20=D0=BB=D0=BE=D0=BA=D0=B0=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=8B=D0=B5=20GeoLite2-=D0=B1=D0=B0=D0=B7=D1=8B=20?= =?UTF-8?q?=D0=B4=D0=BB=D1=8F=20=D1=81=D1=82=D1=80=D0=B0=D0=BD=20=D0=B8=20?= =?UTF-8?q?ASN=20=D0=BF=D0=BE=D1=82=D0=BE=D0=BA=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lookup страны/ASN при ingest теперь идёт сначала по локальным mmdb MaxMind GeoLite2 (Country + ASN, зеркало P3TERX, без ключей), с мгновенным синхронным ответом для IPv4/IPv6; RIPEstat остаётся fallback до первой загрузки баз и при промахе. - backend/src/services/traffic-flow-geoip.ts: ридеры maxmind, фасад resolveFlowIp (geoip-first → RIPE-кэш), статус ридеров - geoip-update-collector.ts: conditional GET по ETag, валидация пробоем 8.8.8.8 (US / AS15169), атомарная подмена с .prev-откатом - джоба планировщика geoip_update (по умолчанию раз в 7 дней), настройки geoip_settings + миграция 0004, API /api/geoip (GET/PUT/update), контракты @mmapp/contracts/geoip - engine/analytics/map-hops переведены на resolveFlowIp - секция «GeoIP-базы» в настройках NetFlow, метки джобы на странице сбора данных, тесты test:geoip --- README.md | 8 + app/(main)/data-collection/page.tsx | 36 ++++ backend/.gitignore | 1 + backend/drizzle/0004_geoip_settings.sql | 19 ++ backend/package.json | 6 +- backend/src/db/schema.ts | 14 ++ backend/src/index.ts | 4 + backend/src/routes/geoip.ts | 57 +++++ backend/src/services/geoip-settings.ts | 100 +++++++++ .../src/services/geoip-update-collector.ts | 198 ++++++++++++++++++ backend/src/services/scheduler.ts | 20 ++ .../src/services/traffic-flow-analytics.ts | 5 +- backend/src/services/traffic-flow-engine.ts | 27 ++- .../src/services/traffic-flow-geoip.test.ts | 186 ++++++++++++++++ backend/src/services/traffic-flow-geoip.ts | 162 ++++++++++++++ backend/src/services/traffic-flow-map-hops.ts | 5 +- backend/src/types/scheduler-run-snapshot.ts | 14 ++ components/traffic/netflow-settings-panel.tsx | 125 +++++++++++ lib/scheduler-run-snapshot.ts | 14 ++ lib/scheduler-settings.ts | 4 + package-lock.json | 49 +++-- packages/contracts/package.json | 4 + packages/contracts/src/geoip.ts | 51 +++++ packages/contracts/src/index.ts | 1 + shared/api/geoip.ts | 27 +++ 25 files changed, 1114 insertions(+), 23 deletions(-) create mode 100644 backend/drizzle/0004_geoip_settings.sql create mode 100644 backend/src/routes/geoip.ts create mode 100644 backend/src/services/geoip-settings.ts create mode 100644 backend/src/services/geoip-update-collector.ts create mode 100644 backend/src/services/traffic-flow-geoip.test.ts create mode 100644 backend/src/services/traffic-flow-geoip.ts create mode 100644 packages/contracts/src/geoip.ts create mode 100644 shared/api/geoip.ts diff --git a/README.md b/README.md index 3cf691c..f467292 100644 --- a/README.md +++ b/README.md @@ -190,6 +190,14 @@ npm run build -w @mmapp/contracts npm --prefix backend run db:migrate-from-sqlite ``` +### GeoIP-базы GeoLite2 (страны и ASN для NetFlow) + +Backend держит локальные mmdb-базы MaxMind GeoLite2 (Country + ASN) в `backend/storage/geoip/` и скачивает их с зеркала [P3TERX/GeoLite.mmdb](https://github.com/P3TERX/GeoLite.mmdb) — без регистрации и ключей. Lookup страны/ASN потока при ingest становится мгновенным (включая IPv6) и не упирается в лимиты RIPEstat; пока базы не скачаны или lookup промахнулся, работает прежний RIPE-fallback. + +Управление — секция «GeoIP-базы (GeoLite2)» в настройках NetFlow (страница «Сбор данных»): автообновление (по умолчанию проверка раз в 7 дней, upstream обновляется еженедельно), статус сборки баз и кнопка «Обновить сейчас». Джоба планировщика — `geoip_update`. Атрибуция: данные MaxMind GeoLite2, CC BY-SA 4.0. + +Примечание для Docker: каталог `storage/geoip` внутри контейнера ephemeral — без смонтированного volume базы (~17 МБ) перекачаются после пересоздания контейнера. Каталог переопределяется переменной `GEOIP_DIR`. + ## CI/CD (Gitea Actions) Файл: `.gitea/workflows/docker.yml` (имя workflow: **Docker images**). diff --git a/app/(main)/data-collection/page.tsx b/app/(main)/data-collection/page.tsx index 51e4518..53a05e5 100644 --- a/app/(main)/data-collection/page.tsx +++ b/app/(main)/data-collection/page.tsx @@ -34,6 +34,7 @@ import { type InternetPathRunSnapshot, type CertificatesRenewRunSnapshot, type BackupsRunSnapshot, + type GeoipUpdateRunSnapshot, type PingRunSnapshot, type ResourcesRunSnapshot, type SchedulerRunSnapshot, @@ -340,6 +341,41 @@ function SnapshotTables({ snap }: { snap: SchedulerRunSnapshot }) { ) } + if (snap.job === "geoip_update") { + const g = snap as GeoipUpdateRunSnapshot + return ( +
+ {g.skipped ? ( +

Прогон пропущен: обновление уже выполнялось или задача отключена.

+ ) : null} +
+
+
Баз проверено
+
{g.checked}
+
+
+
Скачано
+
{g.downloaded}
+
+
+
Без изменений
+
{g.skippedUnchanged}
+
+
+
Объём, МБ
+
{(g.bytes / 1024 / 1024).toFixed(1)}
+
+
+ {g.errors.length ? ( +
+ {g.errors.map((e, i) => ( +

{e}

+ ))} +
+ ) : null} +
+ ) + } if (snap.job === "alert_engine") { const a = snap as AlertEngineRunSnapshot return ( diff --git a/backend/.gitignore b/backend/.gitignore index 5460d7e..920f11d 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -5,3 +5,4 @@ dist/ *.db-wal .env storage/backups/ +storage/geoip/ diff --git a/backend/drizzle/0004_geoip_settings.sql b/backend/drizzle/0004_geoip_settings.sql new file mode 100644 index 0000000..7ecf4b5 --- /dev/null +++ b/backend/drizzle/0004_geoip_settings.sql @@ -0,0 +1,19 @@ +-- GeoLite2 mmdb (страна/ASN для netflow): настройки автообновления зеркала P3TERX + +CREATE TABLE IF NOT EXISTS geoip_settings ( + id BIGINT PRIMARY KEY CHECK (id = 1), + enabled BOOLEAN NOT NULL DEFAULT TRUE, + update_interval_sec INTEGER NOT NULL DEFAULT 604800, + last_check_at TIMESTAMPTZ, + last_success_at TIMESTAMPTZ, + last_error TEXT, + country_build_at TIMESTAMPTZ, + asn_build_at TIMESTAMPTZ, + etags_json JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +INSERT INTO geoip_settings (id) +VALUES (1) +ON CONFLICT (id) DO NOTHING; diff --git a/backend/package.json b/backend/package.json index 8a1c704..a3b7a78 100644 --- a/backend/package.json +++ b/backend/package.json @@ -15,11 +15,12 @@ "test:auth": "tsx src/lib/permissions.test.ts && tsx src/plugins/auth.smoke.test.ts", "test:wireguard": "npx tsx src/services/wireguard-config.test.ts", "test:traffic-rate": "tsx src/services/traffic-rate.test.ts", - "test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-planes.test.ts && tsx src/services/traffic-flow-ip.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-brands.test.ts && tsx src/services/traffic-flow-ingest.test.ts && tsx src/services/traffic-flow-analytics.test.ts && tsx src/services/traffic-flow-map-hops.test.ts && tsx src/services/traffic-flow-purge.test.ts", + "test:traffic-flow": "tsx src/services/traffic-flow-parse.test.ts && tsx src/services/traffic-flow-map-exporter.test.ts && tsx src/services/traffic-flow-ifaces.test.ts && tsx src/services/traffic-flow-dedup.test.ts && tsx src/services/traffic-flow-planes.test.ts && tsx src/services/traffic-flow-ip.test.ts && tsx src/services/traffic-flow-classify.test.ts && tsx src/services/traffic-flow-ripe.test.ts && tsx src/services/traffic-flow-brands.test.ts && tsx src/services/traffic-flow-ingest.test.ts && tsx src/services/traffic-flow-analytics.test.ts && tsx src/services/traffic-flow-map-hops.test.ts && tsx src/services/traffic-flow-purge.test.ts && tsx src/services/traffic-flow-geoip.test.ts", "test:users": "tsx src/modules/users/iface-type.test.ts && tsx src/modules/users/bindings.test.ts", "test:pg": "tsx src/db/sql-bind.test.ts && tsx src/db/sqlite-json.test.ts && tsx src/db/traffic-flags.test.ts && tsx src/db/pg-schema.test.ts", "test:backups": "tsx src/services/s3-backup-client.test.ts", - "test": "npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups" + "test": "npm run test:alert-engine && npm run test:auth && npm run test:wireguard && npm run test:traffic-rate && npm run test:traffic-flow && npm run test:users && npm run test:pg && npm run test:backups", + "test:geoip": "tsx src/services/traffic-flow-geoip.test.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.888.0", @@ -33,6 +34,7 @@ "drizzle-orm": "^0.45.2", "fastify": "^5.8.5", "fastify-plugin": "^5.1.0", + "maxmind": "^5.0.7", "pg": "^8.23.0", "undici": "^8.1.0", "zod": "^4.4.1" diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts index 4c119d8..f20e88b 100644 --- a/backend/src/db/schema.ts +++ b/backend/src/db/schema.ts @@ -231,6 +231,20 @@ export const flowBuckets = pgTable("flow_buckets", { index("idx_flow_buckets_server_time").on(t.serverId, t.bucketAt), ]) +export const geoipSettings = pgTable("geoip_settings", { + id: idSingleton(), + enabled: boolean("enabled").notNull().default(true), + updateIntervalSec: integer("update_interval_sec").notNull().default(604800), + lastCheckAt: ts("last_check_at"), + lastSuccessAt: ts("last_success_at"), + lastError: text("last_error"), + countryBuildAt: ts("country_build_at"), + asnBuildAt: ts("asn_build_at"), + etagsJson: jsonb("etags_json").notNull().default(sql`'{}'::jsonb`), + createdAt: ts("created_at").notNull().defaultNow(), + updatedAt: ts("updated_at").notNull().defaultNow(), +}) + export const flowIpMeta = pgTable("flow_ip_meta", { prefix: text("prefix").primaryKey(), asn: integer("asn").notNull().default(0), diff --git a/backend/src/index.ts b/backend/src/index.ts index e891e37..c0dd3f6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -14,6 +14,7 @@ import filtersRoutes from "./routes/filters.js" import recursiveRoutes from "./routes/recursive-routes.js" import trafficRoutes from "./routes/traffic.js" import trafficFlowRoutes from "./routes/traffic-flow.js" +import geoipRoutes from "./routes/geoip.js" import serversApiPingRoutes from "./routes/servers-api-ping.js" import uptimeRoutes from "./routes/uptime.js" import networkRoutes from "./routes/network.js" @@ -32,6 +33,7 @@ import firewallRoutes from "./routes/firewall.js" import usersRoutes from "./routes/users.js" import { refreshScheduler, stopScheduler } from "./services/scheduler.js" import { getFlowWorkerHealth, startTrafficFlowListener, stopTrafficFlowListener } from "./services/traffic-flow-ingest.js" +import { initGeoip } from "./services/traffic-flow-geoip.js" const eventLoopDelay = monitorEventLoopDelay({ resolution: 20 }) eventLoopDelay.enable() @@ -116,6 +118,7 @@ export async function buildApp(opts?: { await app.register(recursiveRoutes, { prefix: "/api" }) await app.register(trafficRoutes, { prefix: "/api" }) await app.register(trafficFlowRoutes, { prefix: "/api" }) + await app.register(geoipRoutes, { prefix: "/api" }) await app.register(serversApiPingRoutes, { prefix: "/api" }) await app.register(uptimeRoutes, { prefix: "/api" }) await app.register(networkRoutes, { prefix: "/api" }) @@ -135,6 +138,7 @@ export async function buildApp(opts?: { if (opts?.startScheduler !== false) { await refreshScheduler() + await initGeoip() await startTrafficFlowListener() app.addHook("onClose", async () => { stopScheduler() diff --git a/backend/src/routes/geoip.ts b/backend/src/routes/geoip.ts new file mode 100644 index 0000000..9cb93ea --- /dev/null +++ b/backend/src/routes/geoip.ts @@ -0,0 +1,57 @@ +import type { FastifyPluginAsyncZod } from "@fastify/type-provider-zod" +import { geoipSettingsPatchSchema } from "@mmapp/contracts/geoip" +import { refreshScheduler } from "../services/scheduler.js" +import { getGeoipSettings, updateGeoipSettings } from "../services/geoip-settings.js" +import { + GEOIP_ASN_FILE, + GEOIP_COUNTRY_FILE, + geoipReadersStatus, + initGeoip, +} from "../services/traffic-flow-geoip.js" +import { collectGeoipUpdateOnce, getGeoipUpdateState } from "../services/geoip-update-collector.js" + +async function buildGeoipStatus() { + await initGeoip() + const readers = geoipReadersStatus() + return { + ready: readers.countryLoaded && readers.asnLoaded, + countryLoaded: readers.countryLoaded, + asnLoaded: readers.asnLoaded, + countryFile: GEOIP_COUNTRY_FILE, + asnFile: GEOIP_ASN_FILE, + dir: readers.dir, + running: getGeoipUpdateState().running, + settings: await getGeoipSettings(), + } +} + +const geoipRoutes: FastifyPluginAsyncZod = async (app) => { + app.get("/geoip", async (_req, reply) => { + return reply.send(await buildGeoipStatus()) + }) + + app.put("/geoip", async (req, reply) => { + const parsed = geoipSettingsPatchSchema.safeParse(req.body ?? {}) + if (!parsed.success) { + return reply + .status(400) + .send({ error: "Некорректное тело запроса", details: parsed.error.flatten() }) + } + await updateGeoipSettings(parsed.data) + await refreshScheduler() + return reply.send({ ok: true, status: await buildGeoipStatus() }) + }) + + app.post("/geoip/update", async (_req, reply) => { + try { + const snapshot = await collectGeoipUpdateOnce({ force: true }) + return reply.send({ ok: !snapshot.fatalError && snapshot.errors.length === 0, snapshot }) + } catch (e) { + const status = (e as { statusCode?: number }).statusCode ?? 502 + const msg = e instanceof Error ? e.message : String(e) + return reply.status(status).send({ error: msg }) + } + }) +} + +export default geoipRoutes diff --git a/backend/src/services/geoip-settings.ts b/backend/src/services/geoip-settings.ts new file mode 100644 index 0000000..db0eb20 --- /dev/null +++ b/backend/src/services/geoip-settings.ts @@ -0,0 +1,100 @@ +import { eq } from "drizzle-orm" +import { db } from "../db/index.js" +import { geoipSettings } from "../db/schema.js" +import type { GeoipSettingsDto, GeoipSettingsPatch } from "@mmapp/contracts/geoip" + +const SETTINGS_ID = 1 +const DEFAULT_INTERVAL_SEC = 604800 + +let dbEnabled = true + +/** Тесты без PostgreSQL: геттеры отдают дефолты, touch/update — no-op. */ +export function disableGeoipDbForTests(): void { + dbEnabled = false +} + +export function resetGeoipSettingsForTests(): void { + dbEnabled = true +} + +type GeoipSettingsRow = typeof geoipSettings.$inferSelect + +async function getGeoipSettingsRow(): Promise { + if (!dbEnabled) return undefined + return ( + (await db.select().from(geoipSettings).where(eq(geoipSettings.id, SETTINGS_ID)).limit(1))[0] + ) +} + +function toDto(row: GeoipSettingsRow | undefined): GeoipSettingsDto { + return { + enabled: row?.enabled ?? true, + updateIntervalSec: row?.updateIntervalSec ?? DEFAULT_INTERVAL_SEC, + lastCheckAt: row?.lastCheckAt ?? null, + lastSuccessAt: row?.lastSuccessAt ?? null, + lastError: row?.lastError ?? null, + countryBuildAt: row?.countryBuildAt ?? null, + asnBuildAt: row?.asnBuildAt ?? null, + updatedAt: row?.updatedAt ?? new Date().toISOString(), + } +} + +export async function getGeoipSettings(): Promise { + return toDto(await getGeoipSettingsRow()) +} + +/** ETag'и зеркала для conditional GET (ключ — имя файла базы). */ +export async function getGeoipEtags(): Promise> { + const row = await getGeoipSettingsRow() + if (!row) return {} + const raw = row?.etagsJson + if (!raw || typeof raw !== "object") return {} + return Object.fromEntries( + Object.entries(raw as Record).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ) +} + +export async function updateGeoipSettings(patch: GeoipSettingsPatch): Promise { + const prev = await getGeoipSettingsRow() + const next = { + enabled: patch.enabled ?? prev?.enabled ?? true, + updateIntervalSec: patch.updateIntervalSec ?? prev?.updateIntervalSec ?? DEFAULT_INTERVAL_SEC, + updatedAt: new Date().toISOString(), + } + if (prev) { + await db.update(geoipSettings).set(next).where(eq(geoipSettings.id, SETTINGS_ID)) + } else { + await db.insert(geoipSettings).values({ id: SETTINGS_ID, ...next }) + } + return getGeoipSettings() +} + +export async function touchGeoipRunMeta(patch: { + lastCheckAt?: string + lastSuccessAt?: string | null + lastError?: string | null + countryBuildAt?: string | null + asnBuildAt?: string | null + etags?: Record +}): Promise { + const prev = await getGeoipSettingsRow() + const set: Partial = { + updatedAt: new Date().toISOString(), + } + if (patch.lastCheckAt !== undefined) set.lastCheckAt = patch.lastCheckAt + if (patch.lastSuccessAt !== undefined) set.lastSuccessAt = patch.lastSuccessAt + if (patch.lastError !== undefined) set.lastError = patch.lastError + if (patch.countryBuildAt !== undefined) set.countryBuildAt = patch.countryBuildAt + if (patch.asnBuildAt !== undefined) set.asnBuildAt = patch.asnBuildAt + if (patch.etags !== undefined) { + const prevEtags = (prev?.etagsJson as Record | null) ?? {} + set.etagsJson = { ...prevEtags, ...patch.etags } + } + if (prev) { + await db.update(geoipSettings).set(set).where(eq(geoipSettings.id, SETTINGS_ID)) + } else { + await db.insert(geoipSettings).values({ id: SETTINGS_ID, ...set }) + } +} diff --git a/backend/src/services/geoip-update-collector.ts b/backend/src/services/geoip-update-collector.ts new file mode 100644 index 0000000..aae66eb --- /dev/null +++ b/backend/src/services/geoip-update-collector.ts @@ -0,0 +1,198 @@ +import { rename, rm, mkdir, writeFile } from "node:fs/promises" +import path from "node:path" +import { open, type AsnResponse, type CountryResponse } from "maxmind" +import type { GeoipUpdateRunSnapshot } from "../types/scheduler-run-snapshot.js" +import { SCHEDULER_RUN_SNAPSHOT_VERSION } from "../types/scheduler-run-snapshot.js" +import { getGeoipEtags, getGeoipSettings, touchGeoipRunMeta } from "./geoip-settings.js" +import { + GEOIP_ASN_FILE, + GEOIP_COUNTRY_FILE, + geoipDir, + reloadGeoipReaders, +} from "./traffic-flow-geoip.js" + +/** Зеркало GeoLite2 без регистрации и ключей (см. README: GeoIP). */ +const MIRROR_BASE = "https://github.com/P3TERX/GeoLite.mmdb/raw/download" +const DOWNLOAD_TIMEOUT_MS = 120_000 +/** Пробный IP для валидации скачанной базы: Google DNS. */ +const PROBE_IP = "8.8.8.8" + +type GeoipDbKind = "country" | "asn" + +let updating = false +let fetchImpl: typeof fetch = globalThis.fetch.bind(globalThis) + +export function getGeoipUpdateState(): { running: boolean } { + return { running: updating } +} + +async function validateCountryFile(filePath: string): Promise { + const reader = await open(filePath) + const rec = reader.get(PROBE_IP) + const iso = rec?.country?.iso_code ?? rec?.registered_country?.iso_code ?? "" + if (iso !== "US") { + throw new Error(`база Country не распознала ${PROBE_IP} как US (${iso || "нет записи"})`) + } + return reader.metadata.buildEpoch.toISOString() +} + +async function validateAsnFile(filePath: string): Promise { + const reader = await open(filePath) + const rec = reader.get(PROBE_IP) + const asn = rec?.autonomous_system_number ?? 0 + if (asn !== 15169) { + throw new Error(`база ASN не распознала ${PROBE_IP} как AS15169 (${asn ? `AS${asn}` : "нет записи"})`) + } + return reader.metadata.buildEpoch.toISOString() +} + +let validateCountry = validateCountryFile +let validateAsn = validateAsnFile + +export function setGeoipFetchForTests(fn: typeof fetch): void { + fetchImpl = fn +} + +export function setGeoipValidateForTests(opts: { + country?: (filePath: string) => Promise + asn?: (filePath: string) => Promise +}): void { + validateCountry = opts.country ?? validateCountryFile + validateAsn = opts.asn ?? validateAsnFile +} + +export function resetGeoipUpdateForTests(): void { + updating = false + fetchImpl = globalThis.fetch.bind(globalThis) + validateCountry = validateCountryFile + validateAsn = validateAsnFile +} + +/** + * Разовая проверка/доставка баз с зеркала P3TERX. Conditional GET по ETag + * (304 = не меняем файл), валидация пробоем 8.8.8.8, атомарная подмена через rename. + */ +export async function collectGeoipUpdateOnce( + opts: { force?: boolean } = {}, +): Promise { + const sampledAt = new Date().toISOString() + if (updating) { + if (opts.force) { + throw Object.assign(new Error("Обновление GeoIP уже выполняется"), { statusCode: 409 }) + } + return emptySnapshot(sampledAt, true) + } + + const settings = await getGeoipSettings() + if (!settings.enabled && !opts.force) { + return emptySnapshot(sampledAt, true) + } + + updating = true + const snapshot: GeoipUpdateRunSnapshot = { + v: SCHEDULER_RUN_SNAPSHOT_VERSION, + job: "geoip_update", + sampledAt, + checked: 0, + downloaded: 0, + skippedUnchanged: 0, + bytes: 0, + errors: [], + } + + try { + const dir = geoipDir() + await mkdir(dir, { recursive: true }) + const storedEtags = await getGeoipEtags() + const etags: Record = {} + const buildAt: Partial> = {} + + for (const kind of ["country", "asn"] as const) { + snapshot.checked += 1 + const file = kind === "country" ? GEOIP_COUNTRY_FILE : GEOIP_ASN_FILE + const target = path.join(dir, file) + const tmp = `${target}.tmp` + const prevEtag = storedEtags[file] + try { + const ac = new AbortController() + const timer = setTimeout(() => ac.abort(), DOWNLOAD_TIMEOUT_MS) + let res: Response + try { + res = await fetchImpl(`${MIRROR_BASE}/${file}`, { + headers: prevEtag ? { "If-None-Match": prevEtag } : {}, + signal: ac.signal, + }) + } finally { + clearTimeout(timer) + } + if (res.status === 304) { + snapshot.skippedUnchanged += 1 + if (prevEtag) etags[file] = prevEtag + continue + } + if (!res.ok) throw new Error(`HTTP ${res.status}`) + const etag = res.headers.get("etag") ?? "" + const body = Buffer.from(await res.arrayBuffer()) + snapshot.bytes += body.byteLength + await writeFile(tmp, body) + buildAt[kind] = + kind === "country" ? await validateCountry(tmp) : await validateAsn(tmp) + + const prevFile = `${target}.prev` + await rm(prevFile, { force: true }) + await rename(target, prevFile).catch(() => { + /* текущего файла могло ещё не быть */ + }) + await rename(tmp, target) + snapshot.downloaded += 1 + if (etag) etags[file] = etag + } catch (e) { + await rm(tmp, { force: true }).catch(() => { + /* best-effort */ + }) + const message = e instanceof Error ? e.message : String(e) + snapshot.errors.push(`${file}: ${message}`) + } + } + + if (snapshot.downloaded > 0) { + await reloadGeoipReaders() + } + + await touchGeoipRunMeta({ + lastCheckAt: sampledAt, + lastSuccessAt: snapshot.errors.length ? null : sampledAt, + lastError: snapshot.errors.length ? snapshot.errors.join("; ") : null, + countryBuildAt: buildAt.country, + asnBuildAt: buildAt.asn, + etags, + }) + return snapshot + } catch (e) { + const message = e instanceof Error ? e.message : String(e) + snapshot.fatalError = message + await touchGeoipRunMeta({ + lastCheckAt: sampledAt, + lastError: message, + }).catch(() => { + /* best-effort */ + }) + return snapshot + } finally { + updating = false + } +} + +function emptySnapshot(sampledAt: string, skipped: boolean): GeoipUpdateRunSnapshot { + return { + v: SCHEDULER_RUN_SNAPSHOT_VERSION, + job: "geoip_update", + sampledAt, + skipped, + checked: 0, + downloaded: 0, + skippedUnchanged: 0, + bytes: 0, + errors: [], + } +} diff --git a/backend/src/services/scheduler.ts b/backend/src/services/scheduler.ts index 3ac3108..aa4c6d4 100644 --- a/backend/src/services/scheduler.ts +++ b/backend/src/services/scheduler.ts @@ -46,6 +46,8 @@ import { collectCertificatesRenewOnce } from "./certificate-renew-collector.js" import { getCertificateRenewSettings } from "./certificates-service.js" import { collectScheduledBackupsOnce } from "./backup-scheduler-collector.js" import { getBackupScheduleSettings } from "./backup-service.js" +import { collectGeoipUpdateOnce } from "./geoip-update-collector.js" +import { getGeoipSettings } from "./geoip-settings.js" import { endSchedulerJob, isSchedulerJobRunning, @@ -63,6 +65,7 @@ export const JOB_KEYS = [ "gre_bgp", "certificates_renew", "backups", + "geoip_update", "alert_engine", ] as const export type SchedulerJobKey = (typeof JOB_KEYS)[number] @@ -148,6 +151,9 @@ async function runSchedulerJobBody(jobKey: SchedulerJobKey): Promise { case "backups": snapshot = await collectScheduledBackupsOnce() break + case "geoip_update": + snapshot = await collectGeoipUpdateOnce() + break case "alert_engine": { const r = await runAlertEngineOnce() snapshot = { @@ -377,6 +383,18 @@ export async function refreshScheduler(): Promise { ) } + const geoip = await getGeoipSettings() + if (geoip.enabled) { + const geoipMs = Math.max(6 * 3600_000, geoip.updateIntervalSec * 1000) + void executeSchedulerJob("geoip_update").catch(() => {}) + timers.set( + "geoip_update", + setInterval(() => { + void executeSchedulerJob("geoip_update").catch(() => {}) + }, geoipMs), + ) + } + const alertMs = 20_000 void executeSchedulerJob("alert_engine").catch(() => {}) timers.set( @@ -409,6 +427,7 @@ export async function getSchedulerStatus() { const internetPath = await getInternetPathSettings() const certRenew = await getCertificateRenewSettings() const backupSchedule = await getBackupScheduleSettings() + const geoip = await getGeoipSettings() const resOn = uptime.resourcesEnabled ?? uptime.enabled const pingOn = uptime.pingEnabled ?? uptime.enabled @@ -424,6 +443,7 @@ export async function getSchedulerStatus() { gre_bgp: { enabled: true, intervalSec: 30 }, certificates_renew: { enabled: certRenew.enabled, intervalSec: certRenew.intervalSec }, backups: { enabled: backupSchedule.enabled, intervalSec: 60 }, + geoip_update: { enabled: geoip.enabled, intervalSec: geoip.updateIntervalSec }, alert_engine: { enabled: true, intervalSec: 20 }, } diff --git a/backend/src/services/traffic-flow-analytics.ts b/backend/src/services/traffic-flow-analytics.ts index 43030c6..d122c89 100644 --- a/backend/src/services/traffic-flow-analytics.ts +++ b/backend/src/services/traffic-flow-analytics.ts @@ -26,7 +26,8 @@ import { resolveIfaceName } from "./traffic-flow-ifaces.js" import { getTrafficFlowSettingsRow, listHostPeers } from "./traffic-flow-settings.js" import { applicationName, flowRowMatchesFilter } from "./traffic-flow-apps.js" import { dedupFlowRowsMaxBytes, flowTupleKey } from "./traffic-flow-dedup.js" -import { enqueueRipeMisses, lookupRipeCached } from "./traffic-flow-ripe.js" +import { enqueueRipeMisses } from "./traffic-flow-ripe.js" +import { resolveFlowIp } from "./traffic-flow-geoip.js" import { classifyFlowDst, refreshFlowCatalogInBackground } from "./traffic-flow-classify.js" import { isIsoCountry } from "./traffic-flow-brands.js" import { classifyFlowPlane, flowBps, shouldKeepPlane } from "./traffic-flow-planes.js" @@ -255,7 +256,7 @@ async function buildFlowAnalyticsUncached(q: FlowAnalyticsQuery): Promise> { + const out = new Map>() + for (const [k, acc] of minuteDims) { + // dimKey: serverId\0bucketAt\0dim\0key + const parts = k.split("\0") + const dim = parts[2] ?? "" + const key = parts.slice(3).join("\0") + let byKey = out.get(dim) + if (!byKey) { + byKey = new Map() + out.set(dim, byKey) + } + const prev = byKey.get(key) + byKey.set(key, { + bytes: (prev?.bytes ?? 0) + acc.bytes, + packets: (prev?.packets ?? 0) + acc.packets, + }) + } + return out +} diff --git a/backend/src/services/traffic-flow-geoip.test.ts b/backend/src/services/traffic-flow-geoip.test.ts new file mode 100644 index 0000000..6ad19c0 --- /dev/null +++ b/backend/src/services/traffic-flow-geoip.test.ts @@ -0,0 +1,186 @@ +import assert from "node:assert/strict" +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import path from "node:path" +import type { AsnResponse, CountryResponse, Reader } from "maxmind" +import { + disableRipeEnqueueForTests, + disableRipePersistForTests, + resetRipeCacheForTests, + seedRipeCacheForTests, +} from "./traffic-flow-ripe.js" +import { + lookupGeoip, + resetGeoipForTests, + resolveFlowIp, + setGeoipReadersForTests, +} from "./traffic-flow-geoip.js" +import { + resetEngineForTests, + ingestParsedFlowsForServerForTests, + minuteDimsSnapshotForTests, +} from "./traffic-flow-engine.js" +import { classifyFlowDst } from "./traffic-flow-classify.js" +import { disableGeoipDbForTests } from "./geoip-settings.js" +import { + collectGeoipUpdateOnce, + resetGeoipUpdateForTests, + setGeoipFetchForTests, + setGeoipValidateForTests, +} from "./geoip-update-collector.js" + +disableRipePersistForTests() +disableRipeEnqueueForTests() +resetRipeCacheForTests() +resetGeoipForTests() + +// ── lookupGeoip: приватные IP → negative без ридеров ────────────────────────── +assert.equal(lookupGeoip("10.1.1.8")?.ok, false) +assert.equal(lookupGeoip("192.168.0.1")?.prefix, "192.168.0.1/32") +assert.equal(lookupGeoip("100.64.1.2")?.ok, false) +assert.equal(lookupGeoip("fe80::1")?.prefix, "fe80::1/128") + +// ── без ридеров публичный IP → null, resolveFlowIp уходит в RIPE-кэш ───────── +assert.equal(lookupGeoip("1.2.3.10"), null) +seedRipeCacheForTests({ + prefix: "1.2.3.0/24", + asn: 64500, + country: "NL", + lat: null, + lng: null, + holder: "TEST", + ok: true, + fetchedAt: Date.now(), +}) +assert.equal(resolveFlowIp("1.2.3.10")?.country, "NL") +assert.equal(resolveFlowIp("1.2.3.10")?.asn, 64500) + +// ── fake-ридеры: geoip приоритетнее RIPE ────────────────────────────────────── +function fakeCountryReader(byIp: Record): Reader { + return { + get(ip: string) { + const iso = byIp[ip] + return iso ? ({ country: { iso_code: iso } } as CountryResponse) : null + }, + metadata: { buildEpoch: new Date("2026-09-02T00:00:00Z") }, + } as unknown as Reader +} + +function fakeAsnReader(byIp: Record): Reader { + return { + get(ip: string) { + const hit = byIp[ip] + return hit + ? ({ autonomous_system_number: hit.asn, autonomous_system_organization: hit.org } as AsnResponse) + : null + }, + metadata: { buildEpoch: new Date("2026-09-02T00:00:00Z") }, + } as unknown as Reader +} + +setGeoipReadersForTests({ + country: fakeCountryReader({ "8.8.8.8": "US", "6.6.6.6": "EU" }), + asn: fakeAsnReader({ + "8.8.8.8": { asn: 15169, org: "GOOGLE" }, + "6.6.6.6": { asn: 15169, org: "GOOGLE" }, + }), +}) + +const hit = resolveFlowIp("8.8.8.8") +assert.equal(hit?.country, "US") +assert.equal(hit?.asn, 15169) +assert.equal(hit?.holder, "GOOGLE") +assert.equal(hit?.ok, true) + +// 1.2.3.10 в fake-ридерах нет — по-прежнему из RIPE-кэша +assert.equal(resolveFlowIp("1.2.3.10")?.asn, 64500) + +// EU не ISO-страна: отфильтрована, страна выведена из ASN (HQ Google → US) +assert.equal(lookupGeoip("6.6.6.6")?.country, "US") + +// geoip-мета совместима с classifyFlowDst (бренд по ASN 15169) +const classified = classifyFlowDst("8.8.8.8", 6, 443, 51504, hit) +assert.equal(classified.service, "Google") + +// ── движок: dims country/asn наполняются из geoip-ридеров ──────────────────── +resetEngineForTests() +ingestParsedFlowsForServerForTests(1, [{ + src: "192.168.88.10", + dst: "8.8.8.8", + proto: 6, + srcPort: 51504, + dstPort: 443, + bytes: 1000, + packets: 10, + inIface: "wg-flow", + outIface: "", + nextHop: "", + flowStartMs: 0, + flowEndMs: 0, + natSrc: "", + natDst: "", +}]) +const dims = minuteDimsSnapshotForTests() +assert.equal(dims.get("country")?.get("US")?.bytes, 1000) +assert.equal(dims.get("asn")?.get("15169")?.bytes, 1000) + +// ── коллектор: 304 → обе базы без изменений ─────────────────────────────────── +disableGeoipDbForTests() +resetGeoipUpdateForTests() +const geoipDir = mkdtempSync(path.join(tmpdir(), "mm-geoip-test-")) +process.env.GEOIP_DIR = geoipDir + +setGeoipFetchForTests(async () => new Response(null, { status: 304 })) +let snap = await collectGeoipUpdateOnce({ force: true }) +assert.equal(snap.skippedUnchanged, 2) +assert.equal(snap.downloaded, 0) +assert.equal(existsSync(path.join(geoipDir, "GeoLite2-Country.mmdb")), false) + +// ── коллектор: 200 + валидация ok → подмена, старый файл в .prev ───────────── +const countryPath = path.join(geoipDir, "GeoLite2-Country.mmdb") +const asnPath = path.join(geoipDir, "GeoLite2-ASN.mmdb") +writeFileSync(countryPath, "old-country") + +setGeoipFetchForTests(async () => + new Response(new Uint8Array([1, 2, 3]), { status: 200, headers: { etag: '"v1"' } })) +setGeoipValidateForTests({ + country: async (p) => { + assert.ok(p.endsWith(".tmp"), "валидация должна идти по tmp-файлу") + return "2026-09-08T00:00:00.000Z" + }, + asn: async () => "2026-09-08T00:00:00.000Z", +}) +snap = await collectGeoipUpdateOnce({ force: true }) +assert.equal(snap.downloaded, 2) +assert.equal(snap.errors.length, 0) +assert.deepEqual(readFileSync(countryPath), Buffer.from([1, 2, 3])) +assert.equal(readFileSync(`${countryPath}.prev`, "utf8"), "old-country") +assert.equal(existsSync(`${asnPath}.prev`), false, "prev у asn не бывает при первой загрузке") +assert.equal(existsSync(`${countryPath}.tmp`), false) + +// ── коллектор: битая база → подмены нет, старый файл цел, tmp удалён ───────── +writeFileSync(asnPath, "good-asn") +setGeoipFetchForTests(async () => + new Response(new Uint8Array([9, 9]), { status: 200 })) +setGeoipValidateForTests({ + country: async () => { + throw new Error("битая база") + }, + asn: async () => { + throw new Error("битая база") + }, +}) +snap = await collectGeoipUpdateOnce({ force: true }) +assert.equal(snap.downloaded, 0) +assert.equal(snap.errors.length, 2) +assert.deepEqual(readFileSync(countryPath), Buffer.from([1, 2, 3]), "country не тронута") +assert.equal(readFileSync(asnPath, "utf8"), "good-asn", "asn не тронут") +assert.equal(existsSync(`${countryPath}.tmp`), false) +assert.equal(existsSync(`${asnPath}.tmp`), false) + +rmSync(geoipDir, { recursive: true, force: true }) +delete process.env.GEOIP_DIR +resetGeoipUpdateForTests() +resetGeoipForTests() + +console.log("traffic-flow-geoip.test.ts: ok") diff --git a/backend/src/services/traffic-flow-geoip.ts b/backend/src/services/traffic-flow-geoip.ts new file mode 100644 index 0000000..cd91365 --- /dev/null +++ b/backend/src/services/traffic-flow-geoip.ts @@ -0,0 +1,162 @@ +import { existsSync } from "node:fs" +import path from "node:path" +import { open, type AsnResponse, type CountryResponse, type Reader } from "maxmind" +import { isNonPublicIp } from "./traffic-flow-ip.js" +import { isIsoCountry, resolveRipeCountry } from "./traffic-flow-brands.js" +import { lookupRipeCached, type FlowIpMeta } from "./traffic-flow-ripe.js" + +export const GEOIP_COUNTRY_FILE = "GeoLite2-Country.mmdb" +export const GEOIP_ASN_FILE = "GeoLite2-ASN.mmdb" + +/** Каталог баз: `storage/geoip` рядом со storage/backups; переопределяется GEOIP_DIR. */ +export function geoipDir(): string { + return path.resolve(process.env.GEOIP_DIR ?? path.join(process.cwd(), "storage", "geoip")) +} + +export function geoipCountryPath(): string { + return path.join(geoipDir(), GEOIP_COUNTRY_FILE) +} + +export function geoipAsnPath(): string { + return path.join(geoipDir(), GEOIP_ASN_FILE) +} + +export interface GeoipReaders { + country: Reader | null + asn: Reader | null +} + +let readers: GeoipReaders = { country: null, asn: null } +let initPromise: Promise | null = null + +/** Открывает оба файла best-effort: отсутствующий/битый файл не мешает второму. */ +export async function openGeoipReaders(dir = geoipDir()): Promise { + const next: GeoipReaders = { country: null, asn: null } + if (existsSync(path.join(dir, GEOIP_COUNTRY_FILE))) { + try { + next.country = await open(path.join(dir, GEOIP_COUNTRY_FILE)) + } catch { + /* битый файл — работаем без country */ + } + } + if (existsSync(path.join(dir, GEOIP_ASN_FILE))) { + try { + next.asn = await open(path.join(dir, GEOIP_ASN_FILE)) + } catch { + /* битый файл — работаем без ASN */ + } + } + return next +} + +/** Открывает ридеры при старте; файлы есть — работают, нет — lookup уходит в RIPE-fallback. */ +export async function initGeoip(): Promise { + if (!initPromise) { + initPromise = openGeoipReaders().then((next) => { + readers = next + return next + }) + } + return initPromise +} + +/** Переоткрывает ридеры после обновления файлов (атомарная замена ссылок). */ +export async function reloadGeoipReaders(): Promise { + const next = await openGeoipReaders() + readers = next + initPromise = Promise.resolve(next) + return next +} + +export function setGeoipReadersForTests(next: Partial): void { + readers = { country: next.country ?? null, asn: next.asn ?? null } +} + +export function resetGeoipForTests(): void { + readers = { country: null, asn: null } + initPromise = null +} + +function negativeMeta(ip: string): FlowIpMeta { + const v6 = ip.includes(":") + return { + prefix: `${ip}/${v6 ? 128 : 32}`, + asn: 0, + country: "—", + lat: null, + lng: null, + holder: "", + ok: false, + fetchedAt: Date.now(), + } +} + +function safeCountryIso(reader: Reader, ip: string): string { + try { + const rec = reader.get(ip) + return rec?.country?.iso_code ?? rec?.registered_country?.iso_code ?? "" + } catch { + return "" + } +} + +function safeAsn(reader: Reader, ip: string): { asn: number; holder: string } { + try { + const rec = reader.get(ip) + return { + asn: rec?.autonomous_system_number ?? 0, + holder: rec?.autonomous_system_organization ?? "", + } + } catch { + return { asn: 0, holder: "" } + } +} + +/** + * Синхронный lookup по локальным GeoLite2. Возвращает FlowIpMeta в семантике RIPE-кэша + * (ok=true когда есть страна или ASN; null — данных нет, пусть пробует RIPE). + */ +export function lookupGeoip(ip: string): FlowIpMeta | null { + const trimmed = String(ip ?? "").trim() + if (!trimmed) return null + if (isNonPublicIp(trimmed)) return negativeMeta(trimmed) + const { country: countryReader, asn: asnReader } = readers + if (!countryReader && !asnReader) return null + const iso = countryReader ? safeCountryIso(countryReader, trimmed) : "" + const country = iso && isIsoCountry(iso) ? iso : "" + const { asn, holder } = asnReader ? safeAsn(asnReader, trimmed) : { asn: 0, holder: "" } + if (!asn && !country) return null + return { + prefix: `${trimmed}/${trimmed.includes(":") ? 128 : 32}`, + asn, + country: resolveRipeCountry(country, asn, holder) || "—", + lat: null, + lng: null, + holder, + ok: true, + fetchedAt: Date.now(), + } +} + +/** Главный вход для потребителей пайплайна: локальные базы первыми, RIPE-кэш fallback. */ +export function resolveFlowIp(ip: string): FlowIpMeta | null { + return lookupGeoip(ip) ?? lookupRipeCached(ip) +} + +export interface GeoipReadersStatus { + countryLoaded: boolean + asnLoaded: boolean + countryBuildAt: string | null + asnBuildAt: string | null + dir: string +} + +export function geoipReadersStatus(): GeoipReadersStatus { + return { + countryLoaded: Boolean(readers.country), + asnLoaded: Boolean(readers.asn), + countryBuildAt: readers.country?.metadata.buildEpoch.toISOString() ?? null, + asnBuildAt: readers.asn?.metadata.buildEpoch.toISOString() ?? null, + dir: geoipDir(), + } +} diff --git a/backend/src/services/traffic-flow-map-hops.ts b/backend/src/services/traffic-flow-map-hops.ts index 84be2e6..6e6679b 100644 --- a/backend/src/services/traffic-flow-map-hops.ts +++ b/backend/src/services/traffic-flow-map-hops.ts @@ -13,7 +13,8 @@ import { getFlowListenerState, listFlowRowsForWindow } from "./traffic-flow-inge import { resolveIfaceName } from "./traffic-flow-ifaces.js" import { classifyFlowPlane, shouldKeepPlane } from "./traffic-flow-planes.js" import { pickInternetPeer } from "./traffic-flow-ip.js" -import { lookupRipeCached, type FlowIpMeta } from "./traffic-flow-ripe.js" +import { type FlowIpMeta } from "./traffic-flow-ripe.js" +import { resolveFlowIp } from "./traffic-flow-geoip.js" import { getTrafficFlowSettingsRow } from "./traffic-flow-settings.js" import { loadFlowTopology, resolveClient, resolveEn, getServerCatalog } from "./traffic-flow-topology.js" import { flowDataEpoch } from "./traffic-flow-engine.js" @@ -394,7 +395,7 @@ async function buildFlowMapHopsUncached(q: FlowMapHopsQuery, minSharePct: number } for (const [dst, acc] of dstAcc) { - const ripe = lookupRipeCached(dst) + const ripe = resolveFlowIp(dst) const classified = classifyMapDstLite(dst, acc.proto, acc.dstPort, acc.srcPort, ripe) if (!classified) continue const toId = mapServiceNodeId(classified.service) diff --git a/backend/src/types/scheduler-run-snapshot.ts b/backend/src/types/scheduler-run-snapshot.ts index eecb45c..9ccb5d4 100644 --- a/backend/src/types/scheduler-run-snapshot.ts +++ b/backend/src/types/scheduler-run-snapshot.ts @@ -226,6 +226,19 @@ export interface BackupsRunSnapshot { fatalError?: string } +export interface GeoipUpdateRunSnapshot { + v: typeof SCHEDULER_RUN_SNAPSHOT_VERSION + job: "geoip_update" + sampledAt: string + skipped?: boolean + fatalError?: string + checked: number + downloaded: number + skippedUnchanged: number + bytes: number + errors: string[] +} + export type SchedulerRunSnapshot = | TrafficRunSnapshot | ResourcesRunSnapshot @@ -236,4 +249,5 @@ export type SchedulerRunSnapshot = | InternetPathRunSnapshot | CertificatesRenewRunSnapshot | BackupsRunSnapshot + | GeoipUpdateRunSnapshot | AlertEngineRunSnapshot diff --git a/components/traffic/netflow-settings-panel.tsx b/components/traffic/netflow-settings-panel.tsx index 80ddb33..ec6ebd8 100644 --- a/components/traffic/netflow-settings-panel.tsx +++ b/components/traffic/netflow-settings-panel.tsx @@ -10,6 +10,7 @@ import { Button } from "@/components/ui/button" import { Input } from "@/components/ui/input" import { CodeExportSheet, type CodeExportFormat } from "@/components/reui-kit/code-export-sheet" import type { TrafficFlowSettingsDto } from "@mmapp/contracts/traffic-flow" +import type { GeoipStatusDto } from "@mmapp/contracts/geoip" import { generateTrafficFlowKeys, getTrafficFlowHostFiles, @@ -17,6 +18,7 @@ import { purgeTrafficFlowData, putTrafficFlowSettings, } from "@/shared/api/traffic-flow" +import { getGeoipStatus, putGeoipSettings, runGeoipUpdateNow } from "@/shared/api/geoip" import { formatFlowPurgeResult, NetflowPurgeConfirm } from "@/components/traffic/netflow-purge-dialog" import { KeyRoundIcon, DownloadIcon, InfoIcon } from "lucide-react" @@ -29,6 +31,127 @@ const HOST_STEPS = [ "Проверка: wg show · ss -ulnp | grep 4739 · в этой панели — last datagram.", ] +function fmtDate(iso: string | null | undefined): string { + return iso ? new Date(iso).toLocaleString("ru-RU") : "—" +} + +function GeoipSettingsSection({ backendUrl }: { backendUrl: string }) { + const [status, setStatus] = useState(null) + const [busy, setBusy] = useState(false) + const [updating, setUpdating] = useState(false) + const [autoOn, setAutoOn] = useState(true) + const [intervalHours, setIntervalHours] = useState("168") + + const load = useCallback(async () => { + const s = await getGeoipStatus(backendUrl) + setStatus(s) + setAutoOn(s.settings.enabled) + setIntervalHours(String(Math.round(s.settings.updateIntervalSec / 3600))) + }, [backendUrl]) + + useEffect(() => { + void load().catch((e: unknown) => { + toast.error(e instanceof Error ? e.message : "Не удалось загрузить GeoIP") + }) + }, [load]) + + async function handleSave() { + setBusy(true) + try { + const hours = Math.min(720, Math.max(6, Number.parseInt(intervalHours, 10) || 168)) + const res = await putGeoipSettings(backendUrl, { + enabled: autoOn, + updateIntervalSec: hours * 3600, + }) + setStatus(res.status) + toast.success("Настройки GeoIP сохранены") + } catch (e) { + toast.error(e instanceof Error ? e.message : "Не удалось сохранить") + } finally { + setBusy(false) + } + } + + async function handleUpdateNow() { + setUpdating(true) + try { + const res = await runGeoipUpdateNow(backendUrl) + if (res.ok || res.snapshot.downloaded > 0 || res.snapshot.skippedUnchanged > 0) { + toast.success( + res.snapshot.downloaded > 0 + ? `Скачано баз: ${res.snapshot.downloaded} (${(res.snapshot.bytes / 1024 / 1024).toFixed(1)} МБ)` + : "Базы актуальны, скачивание не требуется", + ) + } else { + toast.error(res.snapshot.errors.join("; ") || "Обновление не выполнено") + } + await load() + } catch (e) { + toast.error(e instanceof Error ? e.message : "Не удалось обновить базы") + } finally { + setUpdating(false) + } + } + + return ( + + + country {status?.countryLoaded ? "ok" : "нет"} + + + asn {status?.asnLoaded ? "ok" : "нет"} + + + } + contentClassName="px-5 py-4 flex flex-col gap-4" + > +
+
+ + Автообновление +
+ + setIntervalHours(e.target.value)} + inputMode="numeric" + disabled={!autoOn} + /> + +
+ +

+ Сборка Country: {fmtDate(status?.settings.countryBuildAt)} · ASN: {fmtDate(status?.settings.asnBuildAt)} + {" · "}последняя проверка: {fmtDate(status?.settings.lastCheckAt)} + {status?.settings.lastError ? ` · ошибка: ${status.settings.lastError}` : ""} +

+

+ Каталог: {status?.dir || "storage/geoip"}. До загрузки баз + и при промахе lookup страна/ASN берутся из RIPEstat, как раньше. +

+ +
+ + +
+ +

+ Данные: MaxMind GeoLite2 (CC BY-SA 4.0), зеркало P3TERX/GeoLite.mmdb. +

+ + ) +} + function NetflowSettingsPanel({ backendUrl, enabled, @@ -276,6 +399,8 @@ function NetflowSettingsPanel({ + + setExportOpen(false)} diff --git a/lib/scheduler-run-snapshot.ts b/lib/scheduler-run-snapshot.ts index 1a84e49..5197d05 100644 --- a/lib/scheduler-run-snapshot.ts +++ b/lib/scheduler-run-snapshot.ts @@ -97,6 +97,19 @@ export interface BackupsRunSnapshot { fatalError?: string } +export interface GeoipUpdateRunSnapshot { + v: number + job: "geoip_update" + sampledAt: string + skipped?: boolean + fatalError?: string + checked: number + downloaded: number + skippedUnchanged: number + bytes: number + errors: string[] +} + export type SchedulerRunSnapshot = | TrafficRunSnapshot | ResourcesRunSnapshot @@ -107,6 +120,7 @@ export type SchedulerRunSnapshot = | InternetPathRunSnapshot | CertificatesRenewRunSnapshot | BackupsRunSnapshot + | GeoipUpdateRunSnapshot | AlertEngineRunSnapshot export interface TrafficServerSnapshot { diff --git a/lib/scheduler-settings.ts b/lib/scheduler-settings.ts index ece36ef..ed88a85 100644 --- a/lib/scheduler-settings.ts +++ b/lib/scheduler-settings.ts @@ -11,6 +11,7 @@ export const SCHEDULER_JOB_KEYS = [ "gre_bgp", "certificates_renew", "backups", + "geoip_update", "alert_engine", ] as const export type SchedulerJobKey = (typeof SCHEDULER_JOB_KEYS)[number] @@ -25,6 +26,7 @@ export const SCHEDULER_JOB_LABELS: Record = { gre_bgp: "GRE + BGP", certificates_renew: "Сертификаты: автообновление", backups: "Бэкапы", + geoip_update: "GeoIP: базы GeoLite2", alert_engine: "Оповещения", } @@ -43,6 +45,8 @@ export const SCHEDULER_JOB_DESCRIPTIONS: Record = { "Автообновление сертификатов, выпущенных через UI (ACME DNS-01 / Cloudflare). Отключается на странице «Сертификаты», если ACME ведёт RouterOS.", backups: "Плановые бэкапы RouterOS по расписанию со страницы «Бэкапы»; тик планировщика раз в минуту.", + geoip_update: + "Проверка и доставка GeoLite2 Country/ASN с зеркала P3TERX в backend/storage/geoip (ETag, атомарная подмена). Управление — в настройках NetFlow.", alert_engine: "Оценка правил по данным из PostgreSQL (сэмплы пишут джобы сбора, в т.ч. «GRE + BGP» и «Серверы: REST API»).", } diff --git a/package-lock.json b/package-lock.json index 4b3b970..1f604e0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -61,6 +61,7 @@ "drizzle-orm": "^0.45.2", "fastify": "^5.8.5", "fastify-plugin": "^5.1.0", + "maxmind": "^5.0.7", "pg": "^8.23.0", "undici": "^8.1.0", "zod": "^4.4.1" @@ -10386,6 +10387,20 @@ "node": ">= 0.4" } }, + "node_modules/maxmind": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/maxmind/-/maxmind-5.0.7.tgz", + "integrity": "sha512-+w637dwfv01MKjkrp4sKDBTEKHLPvWLYb647QTjiz3wG/teSemqudIKNShaS6eqZ7ffxC9oZlQQgIqY0rGojog==", + "license": "MIT", + "dependencies": { + "mmdb-lib": "3.0.3", + "tiny-lru": "13.0.0" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, "node_modules/media-typer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz", @@ -10531,6 +10546,16 @@ "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", "license": "MIT" }, + "node_modules/mmdb-lib": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/mmdb-lib/-/mmdb-lib-3.0.3.tgz", + "integrity": "sha512-xQPoBXcNjjHiOvOraFBKtA++uNWF6aCVHL9dRKFXEov8eI3QJwtgiw3qApsonFT5SpoqsEVISUTg3HIDs2DiXw==", + "license": "MIT", + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, "node_modules/mnemonist": { "version": "0.40.4", "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.40.4.tgz", @@ -13299,6 +13324,15 @@ "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", "license": "MIT" }, + "node_modules/tiny-lru": { + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-13.0.0.tgz", + "integrity": "sha512-xDHxKKS1FdF0Tv2P+QT7IeSEg74K/8cEDzbv3Tv6UyHHUgBOjOiQiBp818MGj66dhurQus/IBcoAbwIKtSGc6Q==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=14" + } + }, "node_modules/tinyglobby": { "version": "0.2.16", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.16.tgz", @@ -14797,21 +14831,6 @@ "dependencies": { "zod": "^4.4.1" } - }, - "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.2.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.2.4.tgz", - "integrity": "sha512-kMVGgsqhO5YTYODD9IPGGhA6iprWidQckK3LmPeW08PIFENRmgfb4MjXHO+p//d+ts2rpjvK5gXWzXSMrPl9cw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } } } } diff --git a/packages/contracts/package.json b/packages/contracts/package.json index fb156f4..d8e2dca 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -45,6 +45,10 @@ "./traffic-flow": { "types": "./dist/traffic-flow.d.ts", "default": "./dist/traffic-flow.js" + }, + "./geoip": { + "types": "./dist/geoip.d.ts", + "default": "./dist/geoip.js" } }, "dependencies": { diff --git a/packages/contracts/src/geoip.ts b/packages/contracts/src/geoip.ts new file mode 100644 index 0000000..c637c0d --- /dev/null +++ b/packages/contracts/src/geoip.ts @@ -0,0 +1,51 @@ +import { z } from "zod" + +export const geoipSettingsDtoSchema = z.object({ + enabled: z.boolean(), + updateIntervalSec: z.number().int().positive(), + lastCheckAt: z.string().nullable(), + lastSuccessAt: z.string().nullable(), + lastError: z.string().nullable(), + countryBuildAt: z.string().nullable(), + asnBuildAt: z.string().nullable(), + updatedAt: z.string(), +}) + +export const geoipSettingsPatchSchema = z.object({ + enabled: z.boolean().optional(), + updateIntervalSec: z + .number() + .int() + .min(6 * 3600) + .max(30 * 86400) + .optional(), +}) + +export const geoipStatusDtoSchema = z.object({ + ready: z.boolean(), + countryLoaded: z.boolean(), + asnLoaded: z.boolean(), + countryFile: z.string(), + asnFile: z.string(), + dir: z.string(), + running: z.boolean(), + settings: geoipSettingsDtoSchema, +}) + +export const geoipUpdateSnapshotDtoSchema = z.object({ + v: z.number(), + job: z.literal("geoip_update"), + sampledAt: z.string(), + skipped: z.boolean().optional(), + fatalError: z.string().optional(), + checked: z.number().int(), + downloaded: z.number().int(), + skippedUnchanged: z.number().int(), + bytes: z.number().int().nonnegative(), + errors: z.array(z.string()), +}) + +export type GeoipSettingsDto = z.infer +export type GeoipSettingsPatch = z.infer +export type GeoipStatusDto = z.infer +export type GeoipUpdateSnapshotDto = z.infer diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 60978bd..799b5e3 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -6,3 +6,4 @@ export * from "./backups.js" export * from "./wireguard.js" export * from "./users.js" export * from "./traffic-flow.js" +export * from "./geoip.js" diff --git a/shared/api/geoip.ts b/shared/api/geoip.ts new file mode 100644 index 0000000..913c0bc --- /dev/null +++ b/shared/api/geoip.ts @@ -0,0 +1,27 @@ +import type { + GeoipSettingsPatch, + GeoipStatusDto, + GeoipUpdateSnapshotDto, +} from "@mmapp/contracts/geoip" +import { requestJson } from "@/shared/api/http-client" + +export async function getGeoipStatus(baseUrl: string): Promise { + return requestJson(baseUrl, "/api/geoip") +} + +export async function putGeoipSettings( + baseUrl: string, + patch: GeoipSettingsPatch, +): Promise<{ ok: boolean; status: GeoipStatusDto }> { + return requestJson(baseUrl, "/api/geoip", { + method: "PUT", + body: JSON.stringify(patch), + }) +} + +export async function runGeoipUpdateNow(baseUrl: string): Promise<{ + ok: boolean + snapshot: GeoipUpdateSnapshotDto +}> { + return requestJson(baseUrl, "/api/geoip/update", { method: "POST" }) +}