feat: harden firewall policies telemetry and operations

Implement reliable agent apply, telemetry v2, analytics retention, guarded rollouts, enrollment, labels and webhooks. Improve ReUI frontend data safety and bounded list loading, and add Docker smoke, recovery and resource checks. Document local verification and pending native acceptance.

apps/api/src/services/lists/refresh.ts

@@ -1,51 +1,47 @@
-import { createHash } from 'node:crypto'
 import type { Db } from '@evofw/db'
 import { repos } from '@evofw/db'
 import { refreshAllHostnameRules } from '../policy/resolve-hostname.js'
-import {
-  findParentListIds,
-  rebuildListCascade,
-  rebuildManualListEntries,
-} from './entries.js'
+import { rebuildListCascade, publishListCascade } from './entries.js'
 import { uniqCidrs } from '../uniq.js'
 import { decryptSecret } from '../secret-cipher.js'
+import { fetchEvobgpPages } from '../evobgp.js'
+import { safeFetchJson, SOURCE_LIMITS } from '../safe-fetch.js'
+
 
-function hashCidrs(cidrs: string[]): string {
-  return `sha256:${createHash('sha256').update(cidrs.join('\n')).digest('hex')}`
-}
 
 async function fetchJsonUrl(url: string): Promise<string[]> {
-  const res = await fetch(url, {
-    headers: { Accept: 'application/json' },
-    signal: AbortSignal.timeout(30_000),
-  })
-  if (!res.ok) throw new Error(`JSON URL HTTP ${res.status}`)
-  const data = (await res.json()) as unknown
+  const { data } = await safeFetchJson(url)
   const out: string[] = []
+  let recognized = false
   const push = (v: unknown) => {
-    if (typeof v === 'string' && v.trim()) out.push(v.trim())
+    if (out.length >= SOURCE_LIMITS.elements) throw new Error('JSON prefix element limit exceeded')
+    if (typeof v !== 'string' || !v.trim()) throw new Error('Invalid JSON prefix item')
+    out.push(v.trim())
   }
   if (Array.isArray(data)) {
+    recognized = true
     for (const item of data) {
       if (typeof item === 'string') push(item)
       else if (item && typeof item === 'object') {
         const o = item as Record<string, unknown>
         push(o.cidr ?? o.prefix ?? o.ip ?? o.network)
-      }
+      } else throw new Error('Invalid JSON prefix item')
     }
   } else if (data && typeof data === 'object') {
     const o = data as Record<string, unknown>
     const arr = (o.prefixes ?? o.cidrs ?? o.ips ?? o.items) as unknown
     if (Array.isArray(arr)) {
+      recognized = true
       for (const item of arr) {
         if (typeof item === 'string') push(item)
         else if (item && typeof item === 'object') {
           const x = item as Record<string, unknown>
           push(x.cidr ?? x.prefix ?? x.ip)
-        }
+        } else throw new Error('Invalid JSON prefix item')
       }
     }
   }
+  if (!recognized) throw new Error('Invalid JSON prefix response')
   return uniqCidrs(out)
 }
 
@@ -62,20 +58,12 @@ async function resolveEvobgpCommunityId(
   if (UUID_RE.test(key)) return key
 
   // Heal lists created when Autocomplete stored label instead of UUID.
-  const res = await fetch(`${base}/v1/communities?limit=200`, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-      Accept: 'application/json',
-    },
-    signal: AbortSignal.timeout(20_000),
+  const pages = await fetchEvobgpPages(base, token, '/v1/communities', 200)
+  const items = pages.flatMap((page) => {
+    if (!Array.isArray(page.items)) throw new Error('Invalid EvoBGP communities')
+    return page.items as { id?: string; community?: string; title?: string | null }[]
   })
-  if (!res.ok) {
-    throw new Error(`EvoBGP communities HTTP ${res.status}`)
-  }
-  const data = (await res.json()) as {
-    items?: { id?: string; community?: string; title?: string | null }[]
-  }
-  const hit = (data.items ?? []).find((c) => {
+  const hit = items.find((c) => {
     if (!c.id || !c.community) return false
     if (c.id === key || c.community === key) return true
     if (c.title && `${c.community} · ${c.title}` === key) return true
@@ -87,40 +75,50 @@ async function resolveEvobgpCommunityId(
   return hit.id
 }
 
-async function fetchEvobgpCommunity(
-  apiUrl: string,
-  token: string,
-  communityId: string,
-): Promise<{ cidrs: string[]; resolvedId: string }> {
+async function fetchEvobgpCommunity(apiUrl: string, token: string, communityId: string): Promise<{ cidrs: string[]; resolvedId: string }> {
   const base = apiUrl.replace(/\/$/, '')
   const resolvedId = await resolveEvobgpCommunityId(base, token, communityId)
-  const url = `${base}/v1/communities/${encodeURIComponent(resolvedId)}/prefixes?limit=5000`
-  const res = await fetch(url, {
-    headers: {
-      Authorization: `Bearer ${token}`,
-      Accept: 'application/json',
-    },
-    signal: AbortSignal.timeout(45_000),
+  const pages = await fetchEvobgpPages(base, token, `/v1/communities/${encodeURIComponent(resolvedId)}/prefixes`, 5000)
+  const cidrs = pages.flatMap((page) => {
+    if (Array.isArray(page.prefixes)) return page.prefixes as string[]
+    if (Array.isArray(page.items)) return page.items.map((item) => {
+      if (!item || typeof item !== 'object' || typeof item.prefix !== 'string') throw new Error('Invalid EvoBGP prefix item')
+      return item.prefix as string
+    })
+    throw new Error('Invalid EvoBGP prefix page')
   })
-  if (!res.ok) {
-    throw new Error(`EvoBGP community prefixes HTTP ${res.status}`)
-  }
-  const data = (await res.json()) as {
-    items?: { prefix?: string }[]
-    prefixes?: string[]
-  }
-  let cidrs: string[] = []
-  if (Array.isArray(data.prefixes) && data.prefixes.length > 0) {
-    cidrs = uniqCidrs(data.prefixes)
-  } else if (Array.isArray(data.items)) {
-    cidrs = uniqCidrs(data.items.map((i) => i.prefix ?? '').filter(Boolean))
-  }
-  return { cidrs, resolvedId }
+  return { cidrs: uniqCidrs(cidrs), resolvedId }
 }
 
-export async function refreshIpList(db: Db, listId: string): Promise<void> {
+let activeRefreshes = 0
+const waiting: (() => void)[] = []
+const refreshing = new WeakMap<Db, Map<string, Promise<void>>>()
+
+async function withRefreshSlot(work: () => Promise<void>): Promise<void> {
+  if (activeRefreshes >= 2) await new Promise<void>(resolve => waiting.push(resolve))
+  else activeRefreshes++
+  try { await work() } finally {
+    const next = waiting.shift()
+    if (next) next()
+    else activeRefreshes--
+  }
+}
+
+/** Coalesce duplicate triggers per DB/list; at most two refreshes process-wide. */
+export function refreshIpList(db: Db, listId: string): Promise<void> {
+  let running = refreshing.get(db)
+  if (!running) { running = new Map(); refreshing.set(db, running) }
+  const existing = running.get(listId)
+  if (existing) return existing
+  const task = withRefreshSlot(() => performRefresh(db, listId)).finally(() => running!.delete(listId))
+  running.set(listId, task)
+  return task
+}
+
+async function performRefresh(db: Db, listId: string): Promise<void> {
   const list = repos.getIpList(db, listId)
   if (!list) return
+  const sourceSettings = [repos.getSetting(db, 'evobgp_api_url'), repos.getSetting(db, 'evobgp_api_token')]
 
   let config: Record<string, unknown> = {}
   try {
@@ -132,12 +130,14 @@ export async function refreshIpList(db: Db, listId: string): Promise<void> {
   try {
     let cidrs: string[] = []
     if (list.type === 'static' || list.type === 'domains') {
-      cidrs = await rebuildManualListEntries(db, listId)
+      await rebuildListCascade(db, listId)
+      repos.updateIpList(db, listId, { refreshedAt: new Date().toISOString(), lastError: null })
+      return
     } else if (list.type === 'json_url') {
       const url = String(config.url ?? '')
       if (!url) throw new Error('config.url required')
       cidrs = await fetchJsonUrl(url)
-      repos.replaceIpListEntries(db, listId, cidrs)
+
     } else if (list.type === 'evobgp_community') {
       const apiUrl =
         String(config.api_url ?? '') ||
@@ -152,39 +152,23 @@ export async function refreshIpList(db: Db, listId: string): Promise<void> {
       }
       const fetched = await fetchEvobgpCommunity(apiUrl, token, communityId)
       cidrs = fetched.cidrs
-      repos.replaceIpListEntries(db, listId, cidrs)
+
       // Persist resolved UUID if list was saved with autocomplete label.
       if (fetched.resolvedId !== communityId) {
-        repos.updateIpList(db, listId, {
-          configJson: JSON.stringify({
-            ...config,
-            community_id: fetched.resolvedId,
-          }),
-        })
+        config.community_id = fetched.resolvedId
       }
-    }
+    } else throw new Error('Unsupported list source type')
 
-    const contentHash = hashCidrs(cidrs)
-    repos.updateIpList(db, listId, {
-      contentHash,
-      refreshedAt: new Date().toISOString(),
-      lastError: null,
-    })
-
-    // Cascade to parents that nest this list (skip self rebuild for non-manual —
-    // CIDRs already replaced above).
-    if (list.type === 'static' || list.type === 'domains') {
-      await rebuildListCascade(db, listId)
-    } else {
-      repos.bumpAgentsForList(db, listId)
-      for (const parentId of findParentListIds(db, listId)) {
-        await rebuildListCascade(db, parentId)
-      }
+    if (!cidrs.length && config.allow_empty !== true) throw new Error('Empty source snapshot quarantined; set config.allow_empty=true to publish it')
+    if (list.type === 'evobgp_community' && sourceSettings.some((value, index) => value !== repos.getSetting(db, index === 0 ? 'evobgp_api_url' : 'evobgp_api_token'))) {
+      throw new Error('EvoBGP settings changed during refresh; retry refresh')
     }
+    await publishListCascade(db, listId, { cidrs, configJson: JSON.stringify(config), expected: list })
   } catch (err) {
+    const current = repos.getIpList(db, listId)
+    if (!current || current.configJson !== list.configJson || current.type !== list.type || current.contentHash !== list.contentHash || current.updatedAt !== list.updatedAt) return
     repos.updateIpList(db, listId, {
-      lastError: err instanceof Error ? err.message : String(err),
-      refreshedAt: new Date().toISOString(),
+      lastError: (err instanceof Error ? err.message : String(err)).slice(0, 512),
     })
   }
 }