MikrotikManager/backend/src/services/traffic-flow-ripe.ts

import { dbAll, dbQuery } from "../db/index.js"
import { canonicalIp, ipv4ToInt, isNonPublicIp, parseCidrV4 } from "./traffic-flow-ip.js"
import { resolveRipeCountry } from "./traffic-flow-brands.js"

export interface FlowIpMeta {
  prefix: string
  asn: number
  country: string
  lat: number | null
  lng: number | null
  holder: string
  ok: boolean
  fetchedAt: number
}

const HIT_TTL_MS = 24 * 60 * 60_000
const NEG_TTL_MS = 6 * 60 * 60_000
const MAX_NEW_PREFIX_PER_MIN = 30
const MAX_QUEUE = 90
const CONCURRENCY = 3
const RIPE_BASE = "https://stat.ripe.net/data"
const UA = "MikrotikManager-flow/1.0"

const mem = new Map<string, FlowIpMeta>()
const asnHolder = new Map<number, { holder: string; fetchedAt: number }>()
const inflight = new Map<string, Promise<FlowIpMeta | null>>()
const queue: string[] = []
const queued = new Set<string>()
const recentFetches: number[] = []

interface RipeIndexed {
  entry: FlowIpMeta
  net: number
  mask: number
  prefixLen: number
}

/** /24 → кандидаты с prefixLen ≥ 24. Более широкие префиксы — в `wideIndex`. */
const v24Index = new Map<number, RipeIndexed[]>()
const wideIndex: RipeIndexed[] = []
let lastCandidateCount = 0

let persistEnabled = true
let enqueueEnabled = true
let loaded = false
let workerRunning = false
let fetchImpl: typeof fetch = globalThis.fetch.bind(globalThis)
let fetchCount = 0

export function disableRipePersistForTests(): void {
  persistEnabled = false
}

export function disableRipeEnqueueForTests(): void {
  enqueueEnabled = false
}

export function resetRipeCacheForTests(): void {
  mem.clear()
  asnHolder.clear()
  inflight.clear()
  queue.length = 0
  queued.clear()
  recentFetches.length = 0
  v24Index.clear()
  wideIndex.length = 0
  lastCandidateCount = 0
  loaded = persistEnabled ? false : true
  workerRunning = false
  fetchCount = 0
  enqueueEnabled = true
  fetchImpl = globalThis.fetch.bind(globalThis)
}

export function seedRipeCacheForTests(entry: FlowIpMeta): void {
  remember(entry)
  loaded = true
}

/** Сколько CIDR смотрели в последнем lookup (для теста индекса /24). */
export function ripeLastCandidateCountForTests(): number {
  return lastCandidateCount
}

export function setRipeFetchForTests(fn: typeof fetch): void {
  fetchImpl = fn
  fetchCount = 0
}

export function ripeFetchCountForTests(): number {
  return fetchCount
}

export async function flushRipeQueueForTests(timeoutMs = 4000): Promise<void> {
  const start = Date.now()
  while (Date.now() - start < timeoutMs) {
    if (!queue.length && !inflight.size && !workerRunning) return
    await new Promise((r) => setTimeout(r, 20))
  }
}

function ttlMs(ok: boolean): number {
  return ok ? HIT_TTL_MS : NEG_TTL_MS
}

function isFresh(entry: FlowIpMeta): boolean {
  return Date.now() - entry.fetchedAt < ttlMs(entry.ok)
}

function unindexPrefix(prefix: string): void {
  const parsed = parseCidrV4(prefix)
  if (!parsed) return
  if (parsed.prefixLen >= 24) {
    const key = parsed.net >>> 8
    const list = v24Index.get(key)
    if (!list) return
    const next = list.filter((row) => row.entry.prefix !== prefix)
    if (next.length) v24Index.set(key, next)
    else v24Index.delete(key)
    return
  }
  const idx = wideIndex.findIndex((row) => row.entry.prefix === prefix)
  if (idx >= 0) wideIndex.splice(idx, 1)
}

function indexEntry(entry: FlowIpMeta): void {
  const parsed = parseCidrV4(entry.prefix)
  if (!parsed) return
  const row: RipeIndexed = {
    entry,
    net: parsed.net,
    mask: parsed.mask,
    prefixLen: parsed.prefixLen,
  }
  if (parsed.prefixLen >= 24) {
    const key = parsed.net >>> 8
    const list = v24Index.get(key)
    if (list) list.push(row)
    else v24Index.set(key, [row])
    return
  }
  wideIndex.push(row)
}

function remember(entry: FlowIpMeta): void {
  const prev = mem.get(entry.prefix)
  if (prev) unindexPrefix(prev.prefix)
  mem.set(entry.prefix, entry)
  indexEntry(entry)
}

function loadSqlite(): void {
  if (loaded || !persistEnabled) {
    loaded = true
    return
  }
  loaded = true
  void loadPg()
}

async function loadPg(): Promise<void> {
  try {
    const rows = await dbAll<{
      prefix: string
      asn: number | null
      country: string
      lat: number | null
      lng: number | null
      holder: string
      ok: number
      fetched_at: string
    }>(`
      SELECT prefix, asn, country, lat, lng, holder, ok, fetched_at
      FROM flow_ip_meta
    `)
    for (const r of rows) {
      const fetchedAt = Date.parse(r.fetched_at)
      const asn = Number(r.asn ?? 0) || 0
      const holder = r.holder || ""
      remember({
        prefix: r.prefix,
        asn,
        country: resolveRipeCountry(r.country || "", asn, holder) || "—",
        lat: r.lat == null ? null : Number(r.lat),
        lng: r.lng == null ? null : Number(r.lng),
        holder,
        ok: r.ok !== 0,
        fetchedAt: Number.isFinite(fetchedAt) ? fetchedAt : 0,
      })
    }
    const asns = await dbAll<{
      asn: number
      holder: string
      fetched_at: string
    }>(`SELECT asn, holder, fetched_at FROM flow_asn_meta`)
    for (const a of asns) {
      const fetchedAt = Date.parse(a.fetched_at)
      asnHolder.set(a.asn, { holder: a.holder || "", fetchedAt: Number.isFinite(fetchedAt) ? fetchedAt : 0 })
    }
  } catch {
    /* table may not exist in isolated tests */
  }
}

function persist(entry: FlowIpMeta): void {
  if (!persistEnabled) return
  void dbQuery(`
      INSERT INTO flow_ip_meta (prefix, asn, country, lat, lng, holder, ok, fetched_at)
      VALUES (@prefix, @asn, @country, @lat, @lng, @holder, @ok, @fetchedAt)
      ON CONFLICT(prefix) DO UPDATE SET
        asn=excluded.asn, country=excluded.country, lat=excluded.lat, lng=excluded.lng,
        holder=excluded.holder, ok=excluded.ok, fetched_at=excluded.fetched_at
    `, {
    prefix: entry.prefix,
    asn: entry.asn,
    country: entry.country,
    lat: entry.lat,
    lng: entry.lng,
    holder: entry.holder,
    ok: entry.ok ? 1 : 0,
    fetchedAt: new Date(entry.fetchedAt).toISOString(),
  }).catch(() => { /* ignore persist errors */ })
}

function persistAsn(asn: number, holder: string): void {
  if (!persistEnabled || !asn) return
  void dbQuery(`
      INSERT INTO flow_asn_meta (asn, holder, fetched_at)
      VALUES (@asn, @holder, @fetchedAt)
      ON CONFLICT(asn) DO UPDATE SET holder=excluded.holder, fetched_at=excluded.fetched_at
    `, {
    asn,
    holder,
    fetchedAt: new Date().toISOString(),
  }).catch(() => { /* ignore */ })
}

/** Удаляет просроченный RIPE-кэш с диска (hit 24h / negative 6h). */
export async function pruneRipeSqlite(nowMs = Date.now()): Promise<void> {
  if (!persistEnabled) return
  try {
    const hitCutoff = new Date(nowMs - HIT_TTL_MS).toISOString()
    const negCutoff = new Date(nowMs - NEG_TTL_MS).toISOString()
    await dbQuery(`DELETE FROM flow_ip_meta WHERE ok != 0 AND fetched_at < ?`, [hitCutoff])
    await dbQuery(`DELETE FROM flow_ip_meta WHERE ok = 0 AND fetched_at < ?`, [negCutoff])
    await dbQuery(`DELETE FROM flow_asn_meta WHERE fetched_at < ?`, [hitCutoff])
  } catch {
    /* table may not exist in isolated tests */
  }
}

function negative(prefix: string): FlowIpMeta {
  return {
    prefix,
    asn: 0,
    country: "—",
    lat: null,
    lng: null,
    holder: "",
    ok: false,
    fetchedAt: Date.now(),
  }
}

export function lookupRipeCached(ip: string): FlowIpMeta | null {
  loadSqlite()
  const trimmed = canonicalIp(ip)
  lastCandidateCount = 0
  if (!trimmed) return null
  if (isNonPublicIp(trimmed)) {
    return negative(`${trimmed.includes(":") ? trimmed : trimmed}/32`)
  }
  const addr = ipv4ToInt(trimmed)
  if (addr == null) return null
  const bucket = v24Index.get(addr >>> 8)
  const candidates = bucket ? bucket.concat(wideIndex) : wideIndex
  lastCandidateCount = candidates.length
  let best: FlowIpMeta | null = null
  let bestLen = -1
  for (const row of candidates) {
    if (!isFresh(row.entry)) continue
    if (((addr & row.mask) >>> 0) !== row.net) continue
    if (row.prefixLen > bestLen) {
      best = row.entry
      bestLen = row.prefixLen
    }
  }
  return best
    ? { ...best, country: resolveRipeCountry(best.country, best.asn, best.holder) || "—" }
    : null
}

async function ripeJson(path: string, resource: string): Promise<unknown> {
  fetchCount += 1
  const url = `${RIPE_BASE}/${path}/data.json?resource=${encodeURIComponent(resource)}`
  const ac = new AbortController()
  const t = setTimeout(() => ac.abort(), 12_000)
  try {
    const res = await fetchImpl(url, {
      headers: { Accept: "application/json", "User-Agent": UA },
      signal: ac.signal,
    })
    if (!res.ok) throw new Error(`HTTP ${res.status}`)
    return await res.json()
  } finally {
    clearTimeout(t)
  }
}

function pickPrefix(data: unknown): string {
  const d = data as { data?: { prefix?: string } }
  return String(d?.data?.prefix ?? "").trim()
}

function pickAsns(data: unknown): number {
  const d = data as { data?: { asns?: unknown } }
  const raw = d?.data?.asns
  const first = Array.isArray(raw) ? raw[0] : raw
  const n = Number.parseInt(String(first ?? "").replace(/^AS/i, ""), 10)
  return Number.isFinite(n) ? n : 0
}

function pickGeo(data: unknown): { country: string; lat: number | null; lng: number | null } {
  const d = data as {
    data?: {
      located_resources?: Array<{
        locations?: Array<{ country?: string; latitude?: number; longitude?: number }>
      }>
    }
  }
  const loc = d?.data?.located_resources?.[0]?.locations?.[0]
  const country = resolveRipeCountry(String(loc?.country ?? ""), 0, "")
  const lat = loc?.latitude == null ? null : Number(loc.latitude)
  const lng = loc?.longitude == null ? null : Number(loc.longitude)
  return {
    country: country || "—",
    lat: Number.isFinite(lat) ? lat : null,
    lng: Number.isFinite(lng) ? lng : null,
  }
}

function pickHolder(data: unknown): string {
  const d = data as { data?: { holder?: string } }
  return String(d?.data?.holder ?? "").trim()
}

function allowNewPrefix(): boolean {
  const now = Date.now()
  while (recentFetches.length && now - recentFetches[0]! > 60_000) recentFetches.shift()
  return recentFetches.length < MAX_NEW_PREFIX_PER_MIN
}

async function resolveIp(ip: string): Promise<FlowIpMeta | null> {
  const cached = lookupRipeCached(ip)
  if (cached) return cached
  const pending = inflight.get(ip)
  if (pending) return pending

  const job = (async () => {
    if (!allowNewPrefix()) return null
    recentFetches.push(Date.now())
    try {
      const net = await ripeJson("network-info", ip)
      const prefix = pickPrefix(net) || `${ip}/32`
      const existing = mem.get(prefix)
      if (existing && isFresh(existing)) return existing
      const asn = pickAsns(net)
      let geo = { country: "—", lat: null as number | null, lng: null as number | null }
      try {
        geo = pickGeo(await ripeJson("maxmind-geo-lite", prefix))
      } catch {
        /* best-effort */
      }
      let holder = asnHolder.get(asn)?.holder ?? ""
      if (asn && (!holder || Date.now() - (asnHolder.get(asn)?.fetchedAt ?? 0) > HIT_TTL_MS)) {
        try {
          holder = pickHolder(await ripeJson("as-overview", `AS${asn}`))
          asnHolder.set(asn, { holder, fetchedAt: Date.now() })
          persistAsn(asn, holder)
        } catch {
          /* best-effort */
        }
      }
      const country = resolveRipeCountry(geo.country, asn, holder)
      const entry: FlowIpMeta = {
        prefix,
        asn,
        country: country || "—",
        lat: geo.lat,
        lng: geo.lng,
        holder,
        ok: Boolean(asn || country),
        fetchedAt: Date.now(),
      }
      remember(entry)
      persist(entry)
      return entry
    } catch {
      const prefix = `${ip}/32`
      const entry = negative(prefix)
      remember(entry)
      persist(entry)
      return entry
    } finally {
      inflight.delete(ip)
    }
  })()

  inflight.set(ip, job)
  return job
}

async function runWorker(): Promise<void> {
  if (workerRunning) return
  workerRunning = true
  try {
    while (queue.length) {
      const batch: string[] = []
      while (batch.length < CONCURRENCY && queue.length) {
        const ip = queue.shift()
        if (!ip) break
        queued.delete(ip)
        if (lookupRipeCached(ip)) continue
        if (ipv4ToInt(ip) == null && !ip.includes(":")) continue
        batch.push(ip)
      }
      if (!batch.length) {
        if (!allowNewPrefix()) {
          await new Promise((r) => setTimeout(r, 1000))
        }
        continue
      }
      await Promise.all(batch.map((ip) => resolveIp(ip)))
    }
  } finally {
    workerRunning = false
    if (queue.length) void runWorker()
  }
}

/** HTTP / SSE never await this — cache miss is filled on a later tick. */
export function enqueueRipeMisses(ips: Iterable<string>): void {
  if (!enqueueEnabled) return
  loadSqlite()
  for (const raw of ips) {
    if (queue.length >= MAX_QUEUE) break
    const ip = String(raw ?? "").trim()
    if (!ip || isNonPublicIp(ip)) continue
    if (lookupRipeCached(ip)) continue
    if (queued.has(ip) || inflight.has(ip)) continue
    queued.add(ip)
    queue.push(ip)
  }
  if (queue.length) void runWorker()
}