feat(firewall): improve blocklist parsing and nft element addition

Enhanced the blocklist parsing function to log when the blocklist file is empty. Introduced new helper functions `nft_join_elements` and `nft_add_v4_chunk` to streamline the addition of elements to the nftables, allowing for batch processing and improved error handling. Adjusted the chunk size for element addition to optimize performance. Updated logging to provide better visibility into the blocklist processing and applied prefixes.

apps/web/src/routes/_auth/firewall.tsx

@@ -34,6 +34,16 @@ import {
 } from '@/queries/firewall'
 import type { BgpCommunity, FirewallClient } from '@/types/api'
 
+function httpsOrigin(origin: string): string {
+  try {
+    const u = new URL(origin)
+    u.protocol = 'https:'
+    return u.origin
+  } catch {
+    return origin.replace(/^http:/i, 'https:')
+  }
+}
+
 export const Route = createFileRoute('/_auth/firewall')({
   component: FirewallPage,
 })
@@ -51,13 +61,13 @@ function FirewallPage() {
 
   const [clientName, setClientName] = useState('web-01')
   const [cpUrl, setCpUrl] = useState(() =>
-    typeof window !== 'undefined' ? window.location.origin : 'https://api.example.com',
+    typeof window !== 'undefined' ? httpsOrigin(window.location.origin) : 'https://api.example.com',
   )
   const [seed, setSeed] = useState('')
 
   useEffect(() => {
     if (installCtx?.suggested_cp_url) {
-      setCpUrl(installCtx.suggested_cp_url)
+      setCpUrl(httpsOrigin(installCtx.suggested_cp_url))
     }
     if (installCtx?.bundle_seed) {
       setSeed(installCtx.bundle_seed)

deploy/compose/docker-compose.production.example.yaml

@@ -159,10 +159,18 @@ services:
         condition: service_started
     labels:
       - traefik.enable=true
+      # Публичные firewall-эндпоинты — без WEBUI_IP_WHITELIST (установка с произвольных серверов).
+      - traefik.http.routers.evobgp-firewall-public.rule=Host(`${WEBUI_DOMAIN}`) && (Path(`/v1/firewall/install.sh`) || Path(`/v1/firewall/sync-script`) || PathPrefix(`/v1/firewall/enroll`))
+      - traefik.http.routers.evobgp-firewall-public.entrypoints=websecure
+      - traefik.http.routers.evobgp-firewall-public.tls=true
+      - traefik.http.routers.evobgp-firewall-public.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-firewall-public.priority=100
+      - traefik.http.routers.evobgp-firewall-public.service=evobgp-web
       - traefik.http.routers.evobgp-web.rule=Host(`${WEBUI_DOMAIN}`)
       - traefik.http.routers.evobgp-web.entrypoints=websecure
       - traefik.http.routers.evobgp-web.tls=true
       - traefik.http.routers.evobgp-web.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-web.priority=10
       - traefik.http.routers.evobgp-web.middlewares=webui-ipwhitelist@docker
       - traefik.http.middlewares.webui-ipwhitelist.ipallowlist.sourcerange=${WEBUI_IP_WHITELIST}
       - traefik.http.services.evobgp-web.loadbalancer.server.port=80

deploy/compose/docker-compose.yaml

@@ -247,10 +247,18 @@ services:
       - evobgp-all
     labels:
       - traefik.enable=true
+      # Публичные firewall-эндпоинты — без WEBUI_IP_WHITELIST (установка с произвольных серверов).
+      - traefik.http.routers.evobgp-firewall-public.rule=Host(`${WEBUI_DOMAIN}`) && (Path(`/v1/firewall/install.sh`) || Path(`/v1/firewall/sync-script`) || PathPrefix(`/v1/firewall/enroll`))
+      - traefik.http.routers.evobgp-firewall-public.entrypoints=websecure
+      - traefik.http.routers.evobgp-firewall-public.tls=true
+      - traefik.http.routers.evobgp-firewall-public.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-firewall-public.priority=100
+      - traefik.http.routers.evobgp-firewall-public.service=evobgp-web
       - traefik.http.routers.evobgp-web.rule=Host(`${WEBUI_DOMAIN}`)
       - traefik.http.routers.evobgp-web.entrypoints=websecure
       - traefik.http.routers.evobgp-web.tls=true
       - traefik.http.routers.evobgp-web.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-web.priority=10
       - traefik.http.routers.evobgp-web.middlewares=webui-ipwhitelist@docker
       - traefik.http.middlewares.webui-ipwhitelist.ipallowlist.sourcerange=${WEBUI_IP_WHITELIST}
       - traefik.http.services.evobgp-web.loadbalancer.server.port=80

deploy/compose/stack.microvps-full.yaml

@@ -162,10 +162,18 @@ services:
         condition: service_started
     labels:
       - traefik.enable=true
+      # Публичные firewall-эндпоинты — без WEBUI_IP_WHITELIST (установка с произвольных серверов).
+      - traefik.http.routers.evobgp-firewall-public.rule=Host(`${WEBUI_DOMAIN}`) && (Path(`/v1/firewall/install.sh`) || Path(`/v1/firewall/sync-script`) || PathPrefix(`/v1/firewall/enroll`))
+      - traefik.http.routers.evobgp-firewall-public.entrypoints=websecure
+      - traefik.http.routers.evobgp-firewall-public.tls=true
+      - traefik.http.routers.evobgp-firewall-public.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-firewall-public.priority=100
+      - traefik.http.routers.evobgp-firewall-public.service=evobgp-web
       - traefik.http.routers.evobgp-web.rule=Host(`${WEBUI_DOMAIN}`)
       - traefik.http.routers.evobgp-web.entrypoints=websecure
       - traefik.http.routers.evobgp-web.tls=true
       - traefik.http.routers.evobgp-web.tls.certresolver=letsencrypt
+      - traefik.http.routers.evobgp-web.priority=10
       - traefik.http.routers.evobgp-web.middlewares=webui-ipwhitelist@docker
       - traefik.http.middlewares.webui-ipwhitelist.ipallowlist.sourcerange=${WEBUI_IP_WHITELIST}
       - traefik.http.services.evobgp-web.loadbalancer.server.port=80

deploy/docker/evobgp-web/nginx.conf

@@ -17,7 +17,7 @@ server {
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
     }
 
     location = /metrics {

docs/firewall.md

@@ -21,6 +21,10 @@
 
 ## Установка на сервер
 
+Публичные URL (без API-ключа, вне `WEBUI_IP_WHITELIST` Traefik): `GET /v1/firewall/install.sh`, `GET /v1/firewall/sync-script`, `POST /v1/firewall/enroll`. Всегда **HTTPS**.
+
+Требуется миграция **`000027_firewall`** в PostgreSQL (применяется при старте API с актуальным бинарём). Если enroll отвечает `503` / `database schema outdated` — перезапустите `evobgp-api` / `evobgp-all` после деплоя новой версии.
+
 ```bash
 curl -fsSL https://<api>/v1/firewall/install.sh | \
   EVOBGP_CP_URL=https://<api> \
@@ -31,6 +35,16 @@ curl -fsSL https://<api>/v1/firewall/install.sh | \
 
 Файлы: `/etc/evobgp/firewall.conf`, `/usr/local/sbin/evobgp-firewall.sh`, systemd timer `evobgp-firewall.timer`.
 
+После **approve** в UI выполните на сервере (или дождитесь timer):
+
+```bash
+sudo rm -f /var/lib/evobgp-firewall/last_hash
+sudo /usr/local/sbin/evobgp-firewall.sh
+sudo nft list table inet evobgp_blocklist
+```
+
+Для парсинга JSON нужен `jq` или `python3` (install.sh ставит `jq` на Debian/Ubuntu при отсутствии).
+
 ## Failover через speaker
 
 При `EVOBGP_FIREWALL_FAILOVER_ENABLED=1` на speaker-agent CP реплицирует состояние через `POST /v1/agent/firewall-replicate`. Клиенты используют тот же DNS-домен.

docs/quickstart.md

@@ -153,6 +153,7 @@ docker compose --env-file .env --env-file .env.web-sec --profile microvps-full u
 - `http://<WEBUI_DOMAIN>` должен редиректить на `https://<WEBUI_DOMAIN>`;
 - с IP из `WEBUI_IP_WHITELIST` UI доступен по HTTPS;
 - с неразрешенного IP Traefik вернет `403`.
+- исключение: `GET /v1/firewall/install.sh`, `GET /v1/firewall/sync-script`, `POST /v1/firewall/enroll` — публичные, без whitelist (см. [firewall.md](firewall.md)).
 
 Health API: `http://<IP>:8080/v1/health`.
 

internal/firewallscripts/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,87 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +155,66 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=64
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +226,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +240,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

internal/firewallscripts/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -38,10 +48,18 @@ CP_URL="${EVOBGP_CP_URL%/}"
 ENROLL_BODY=$(printf '{"name":"%s","hostname":"%s","client_token":"%s","client_version":"install.sh/1"}' \
   "$EVOBGP_CLIENT_NAME" "$HOSTNAME" "$CLIENT_TOKEN")
 
-RESP=$(curl -fsS -X POST "${CP_URL}/v1/firewall/enroll" \
+ENROLL_TMP=$(mktemp)
+trap 'rm -f "$ENROLL_TMP"' EXIT
+ENROLL_CODE=$(curl -sS -o "$ENROLL_TMP" -w "%{http_code}" -X POST "${CP_URL}/v1/firewall/enroll" \
   -H "Content-Type: application/json" \
   -H "X-EvoBGP-Seed: ${EVOBGP_SEED}" \
   -d "$ENROLL_BODY")
+if [[ "$ENROLL_CODE" != "201" ]]; then
+  echo "evobgp-firewall enroll failed: HTTP ${ENROLL_CODE} from ${CP_URL}/v1/firewall/enroll" >&2
+  cat "$ENROLL_TMP" >&2
+  exit 1
+fi
+RESP=$(cat "$ENROLL_TMP")
 
 CLIENT_ID=""
 if command -v jq >/dev/null 2>&1; then
@@ -102,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi

internal/httpapi/routes_crud.go

@@ -16,6 +16,8 @@ import (
 	"evobgp/internal/pipeline"
 	"evobgp/internal/runtimelogs"
 	"evobgp/internal/store"
+
+	"github.com/jackc/pgx/v5/pgconn"
 )
 
 func (s *Server) registerCRUDRoutes(m *http.ServeMux) {
@@ -188,9 +190,29 @@ func writeStoreErr(w http.ResponseWriter, err error) {
 		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
 		return
 	}
+	if writePostgresStoreErr(w, err) {
+		return
+	}
 	writeInternalError(w, "store", err)
 }
 
+func writePostgresStoreErr(w http.ResponseWriter, err error) bool {
+	var pgErr *pgconn.PgError
+	if !errors.As(err, &pgErr) {
+		return false
+	}
+	switch pgErr.Code {
+	case "42P01":
+		writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable",
+			"database schema outdated; restart API after deploy or apply migration 000027_firewall")
+		return true
+	case "23505":
+		writeProblem(w, http.StatusConflict, "Conflict", "resource already exists")
+		return true
+	}
+	return false
+}
+
 func (s *Server) handleListCDNSources(w http.ResponseWriter, r *http.Request) {
 	a, ok := authFromContext(r.Context())
 	if !ok || !s.requireAtLeast(w, a, "viewer") {

internal/httpapi/routes_firewall.go

@@ -50,20 +50,13 @@ func (s *Server) handleFirewallInstallContext(w http.ResponseWriter, r *http.Req
 	writeJSON(w, http.StatusOK, map[string]any{
 		"bundle_seed":            seed,
 		"bundle_seed_configured": seed != "",
-		"suggested_cp_url":       requestBaseURL(r),
-		"install_sh_url":         requestBaseURL(r) + "/v1/firewall/install.sh",
+		"suggested_cp_url":       publicHTTPSBaseURL(r),
+		"install_sh_url":         publicHTTPSBaseURL(r) + "/v1/firewall/install.sh",
 	})
 }
 
-func requestBaseURL(r *http.Request) string {
-	scheme := "https"
-	if r.TLS == nil {
-		if xf := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto")); xf != "" {
-			scheme = strings.ToLower(strings.Split(xf, ",")[0])
-		} else if strings.EqualFold(r.URL.Scheme, "http") {
-			scheme = "http"
-		}
-	}
+// publicHTTPSBaseURL is the external HTTPS origin for firewall install/enroll links.
+func publicHTTPSBaseURL(r *http.Request) string {
 	host := strings.TrimSpace(r.Host)
 	if xf := strings.TrimSpace(r.Header.Get("X-Forwarded-Host")); xf != "" {
 		host = strings.TrimSpace(strings.Split(xf, ",")[0])
@@ -71,7 +64,11 @@ func requestBaseURL(r *http.Request) string {
 	if host == "" {
 		return ""
 	}
-	return scheme + "://" + host
+	return "https://" + host
+}
+
+func requestBaseURL(r *http.Request) string {
+	return publicHTTPSBaseURL(r)
 }
 
 func (s *Server) handleFirewallEnrollPublic(w http.ResponseWriter, r *http.Request) {
@@ -126,7 +123,7 @@ func (s *Server) handleFirewallEnrollPublic(w http.ResponseWriter, r *http.Reque
 			writeProblem(w, http.StatusConflict, "Conflict", "client token already enrolled")
 			return
 		}
-		writeInternalError(w, "internal", err)
+		writeStoreErr(w, err)
 		return
 	}
 	writeJSON(w, http.StatusCreated, map[string]any{

internal/httpapi/routes_firewall_test.go

@@ -186,7 +186,10 @@ func TestFirewallInstallContext(t *testing.T) {
 	if configured, _ := ctx["bundle_seed_configured"].(bool); !configured {
 		t.Fatal("bundle_seed_configured want true")
 	}
-	if url, _ := ctx["install_sh_url"].(string); !strings.HasSuffix(url, "/v1/firewall/install.sh") {
+	if url, _ := ctx["suggested_cp_url"].(string); !strings.HasPrefix(url, "https://") {
+		t.Fatalf("suggested_cp_url=%q want https", url)
+	}
+	if url, _ := ctx["install_sh_url"].(string); !strings.HasPrefix(url, "https://") || !strings.HasSuffix(url, "/v1/firewall/install.sh") {
 		t.Fatalf("install_sh_url=%q", url)
 	}
 

internal/repository/postgres_firewall.go

@@ -13,14 +13,17 @@ import (
 	"github.com/jackc/pgx/v5"
 )
 
+const firewallClientSelectCols = `
+	id, name, COALESCE(hostname, ''), token_prefix, status,
+	last_seen_at, COALESCE(last_seen_at_source, ''), COALESCE(last_seen_ip, ''),
+	last_apply_at, COALESCE(last_apply_status, ''), COALESCE(last_apply_error, ''),
+	COALESCE(last_apply_prefix_count, 0), COALESCE(last_apply_ip_count, 0), COALESCE(last_apply_source, ''),
+	COALESCE(client_version, ''), created_at, approved_at, approved_by_api_key_id, revoked_at`
+
 func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient, error) {
 	ctx := context.Background()
 	rows, err := p.pool.Query(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE tenant_id=$1 ORDER BY created_at DESC`, tenantID)
 	if err != nil {
 		return nil, err
@@ -40,11 +43,7 @@ func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient
 func (p *Postgres) GetFirewallClient(tenantID, id string) (*store.FirewallClient, error) {
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE id=$1 AND tenant_id=$2`, id, tenantID)
 	c, err := scanFirewallClientRow(row.Scan, tenantID)
 	if err != nil {
@@ -147,11 +146,7 @@ func (p *Postgres) LookupFirewallClientByTokenHash(hash []byte) (*store.Firewall
 	}
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT tenant_id, id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT tenant_id, `+firewallClientSelectCols+`
 		FROM firewall_client WHERE token_hash=$1`, hash)
 	c, err := scanFirewallClientLookupRow(row.Scan)
 	if err != nil {

internal/repository/postgres_firewall_test.go

@@ -0,0 +1,52 @@
+package repository
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"evobgp/internal/authkey"
+	"evobgp/internal/db"
+	"evobgp/internal/store"
+)
+
+func TestPostgresFirewallClientCreateAndGetIntegration(t *testing.T) {
+	dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("EVOBGP_TEST_DATABASE_URL not set")
+	}
+	ctx := context.Background()
+	pool, err := db.OpenPostgresPool(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	pg, err := NewPostgres(ctx, pool, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tenant, _, _, _, _ := pg.DemoIDs()
+	if tenant == "" {
+		t.Fatal("demo tenant required")
+	}
+	tok := "evobgp_fw_pgtest_" + t.Name()
+	hash := authkey.HashToken(tok)
+	client, err := pg.CreateFirewallClient(tenant, &store.FirewallClientCreate{
+		Name:          "pg-firewall-test",
+		Hostname:      "test.local",
+		TokenPrefix:   tok[:12],
+		TokenHash:     hash,
+		ClientVersion: "test/1",
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	got, err := pg.GetFirewallClient(tenant, client.ID)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if got.Name != "pg-firewall-test" || got.Status != "pending" {
+		t.Fatalf("got %+v", got)
+	}
+	_ = pg.DeleteFirewallClient(tenant, client.ID)
+}

scripts/firewall/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,87 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +155,66 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=64
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +226,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +240,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

scripts/firewall/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -38,10 +48,18 @@ CP_URL="${EVOBGP_CP_URL%/}"
 ENROLL_BODY=$(printf '{"name":"%s","hostname":"%s","client_token":"%s","client_version":"install.sh/1"}' \
   "$EVOBGP_CLIENT_NAME" "$HOSTNAME" "$CLIENT_TOKEN")
 
-RESP=$(curl -fsS -X POST "${CP_URL}/v1/firewall/enroll" \
+ENROLL_TMP=$(mktemp)
+trap 'rm -f "$ENROLL_TMP"' EXIT
+ENROLL_CODE=$(curl -sS -o "$ENROLL_TMP" -w "%{http_code}" -X POST "${CP_URL}/v1/firewall/enroll" \
   -H "Content-Type: application/json" \
   -H "X-EvoBGP-Seed: ${EVOBGP_SEED}" \
   -d "$ENROLL_BODY")
+if [[ "$ENROLL_CODE" != "201" ]]; then
+  echo "evobgp-firewall enroll failed: HTTP ${ENROLL_CODE} from ${CP_URL}/v1/firewall/enroll" >&2
+  cat "$ENROLL_TMP" >&2
+  exit 1
+fi
+RESP=$(cat "$ENROLL_TMP")
 
 CLIENT_ID=""
 if command -v jq >/dev/null 2>&1; then
@@ -102,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi