feat(firewall): enhance blocklist handling and installation script

Updated the firewall scripts to improve blocklist handling by introducing a new method for fetching and parsing blocklist data using either `jq` or `python3`. Enhanced the installation script to ensure the presence of required dependencies and provided user guidance for post-approval actions. Additionally, improved logging for applied prefixes and total counts, ensuring better visibility into the firewall's operational status.

docs/firewall.md

@@ -23,6 +23,8 @@
 
 Публичные URL (без API-ключа, вне `WEBUI_IP_WHITELIST` Traefik): `GET /v1/firewall/install.sh`, `GET /v1/firewall/sync-script`, `POST /v1/firewall/enroll`. Всегда **HTTPS**.
 
+Требуется миграция **`000027_firewall`** в PostgreSQL (применяется при старте API с актуальным бинарём). Если enroll отвечает `503` / `database schema outdated` — перезапустите `evobgp-api` / `evobgp-all` после деплоя новой версии.
+
 ```bash
 curl -fsSL https://<api>/v1/firewall/install.sh | \
   EVOBGP_CP_URL=https://<api> \
@@ -33,6 +35,16 @@ curl -fsSL https://<api>/v1/firewall/install.sh | \
 
 Файлы: `/etc/evobgp/firewall.conf`, `/usr/local/sbin/evobgp-firewall.sh`, systemd timer `evobgp-firewall.timer`.
 
+После **approve** в UI выполните на сервере (или дождитесь timer):
+
+```bash
+sudo rm -f /var/lib/evobgp-firewall/last_hash
+sudo /usr/local/sbin/evobgp-firewall.sh
+sudo nft list table inet evobgp_blocklist
+```
+
+Для парсинга JSON нужен `jq` или `python3` (install.sh ставит `jq` на Debian/Ubuntu при отсутствии).
+
 ## Failover через speaker
 
 При `EVOBGP_FIREWALL_FAILOVER_ENABLED=1` на speaker-agent CP реплицирует состояние через `POST /v1/agent/firewall-replicate`. Клиенты используют тот же DNS-домен.

internal/firewallscripts/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,53 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +121,67 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=128
+    local n
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +193,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +207,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

internal/firewallscripts/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -38,10 +48,18 @@ CP_URL="${EVOBGP_CP_URL%/}"
 ENROLL_BODY=$(printf '{"name":"%s","hostname":"%s","client_token":"%s","client_version":"install.sh/1"}' \
   "$EVOBGP_CLIENT_NAME" "$HOSTNAME" "$CLIENT_TOKEN")
 
-RESP=$(curl -fsS -X POST "${CP_URL}/v1/firewall/enroll" \
+ENROLL_TMP=$(mktemp)
+trap 'rm -f "$ENROLL_TMP"' EXIT
+ENROLL_CODE=$(curl -sS -o "$ENROLL_TMP" -w "%{http_code}" -X POST "${CP_URL}/v1/firewall/enroll" \
   -H "Content-Type: application/json" \
   -H "X-EvoBGP-Seed: ${EVOBGP_SEED}" \
   -d "$ENROLL_BODY")
+if [[ "$ENROLL_CODE" != "201" ]]; then
+  echo "evobgp-firewall enroll failed: HTTP ${ENROLL_CODE} from ${CP_URL}/v1/firewall/enroll" >&2
+  cat "$ENROLL_TMP" >&2
+  exit 1
+fi
+RESP=$(cat "$ENROLL_TMP")
 
 CLIENT_ID=""
 if command -v jq >/dev/null 2>&1; then
@@ -102,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi

internal/httpapi/routes_crud.go

@@ -16,6 +16,8 @@ import (
 	"evobgp/internal/pipeline"
 	"evobgp/internal/runtimelogs"
 	"evobgp/internal/store"
+
+	"github.com/jackc/pgx/v5/pgconn"
 )
 
 func (s *Server) registerCRUDRoutes(m *http.ServeMux) {
@@ -188,9 +190,29 @@ func writeStoreErr(w http.ResponseWriter, err error) {
 		writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
 		return
 	}
+	if writePostgresStoreErr(w, err) {
+		return
+	}
 	writeInternalError(w, "store", err)
 }
 
+func writePostgresStoreErr(w http.ResponseWriter, err error) bool {
+	var pgErr *pgconn.PgError
+	if !errors.As(err, &pgErr) {
+		return false
+	}
+	switch pgErr.Code {
+	case "42P01":
+		writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable",
+			"database schema outdated; restart API after deploy or apply migration 000027_firewall")
+		return true
+	case "23505":
+		writeProblem(w, http.StatusConflict, "Conflict", "resource already exists")
+		return true
+	}
+	return false
+}
+
 func (s *Server) handleListCDNSources(w http.ResponseWriter, r *http.Request) {
 	a, ok := authFromContext(r.Context())
 	if !ok || !s.requireAtLeast(w, a, "viewer") {

internal/httpapi/routes_firewall.go

@@ -123,7 +123,7 @@ func (s *Server) handleFirewallEnrollPublic(w http.ResponseWriter, r *http.Reque
 			writeProblem(w, http.StatusConflict, "Conflict", "client token already enrolled")
 			return
 		}
-		writeInternalError(w, "internal", err)
+		writeStoreErr(w, err)
 		return
 	}
 	writeJSON(w, http.StatusCreated, map[string]any{

internal/repository/postgres_firewall.go

@@ -13,14 +13,17 @@ import (
 	"github.com/jackc/pgx/v5"
 )
 
+const firewallClientSelectCols = `
+	id, name, COALESCE(hostname, ''), token_prefix, status,
+	last_seen_at, COALESCE(last_seen_at_source, ''), COALESCE(last_seen_ip, ''),
+	last_apply_at, COALESCE(last_apply_status, ''), COALESCE(last_apply_error, ''),
+	COALESCE(last_apply_prefix_count, 0), COALESCE(last_apply_ip_count, 0), COALESCE(last_apply_source, ''),
+	COALESCE(client_version, ''), created_at, approved_at, approved_by_api_key_id, revoked_at`
+
 func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient, error) {
 	ctx := context.Background()
 	rows, err := p.pool.Query(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE tenant_id=$1 ORDER BY created_at DESC`, tenantID)
 	if err != nil {
 		return nil, err
@@ -40,11 +43,7 @@ func (p *Postgres) ListFirewallClients(tenantID string) ([]*store.FirewallClient
 func (p *Postgres) GetFirewallClient(tenantID, id string) (*store.FirewallClient, error) {
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT `+firewallClientSelectCols+`
 		FROM firewall_client WHERE id=$1 AND tenant_id=$2`, id, tenantID)
 	c, err := scanFirewallClientRow(row.Scan, tenantID)
 	if err != nil {
@@ -147,11 +146,7 @@ func (p *Postgres) LookupFirewallClientByTokenHash(hash []byte) (*store.Firewall
 	}
 	ctx := context.Background()
 	row := p.pool.QueryRow(ctx, `
-		SELECT tenant_id, id, name, hostname, token_prefix, status,
-			last_seen_at, last_seen_at_source, last_seen_ip,
-			last_apply_at, last_apply_status, last_apply_error,
-			last_apply_prefix_count, last_apply_ip_count, last_apply_source,
-			client_version, created_at, approved_at, approved_by_api_key_id, revoked_at
+		SELECT tenant_id, `+firewallClientSelectCols+`
 		FROM firewall_client WHERE token_hash=$1`, hash)
 	c, err := scanFirewallClientLookupRow(row.Scan)
 	if err != nil {

internal/repository/postgres_firewall_test.go

@@ -0,0 +1,52 @@
+package repository
+
+import (
+	"context"
+	"os"
+	"testing"
+
+	"evobgp/internal/authkey"
+	"evobgp/internal/db"
+	"evobgp/internal/store"
+)
+
+func TestPostgresFirewallClientCreateAndGetIntegration(t *testing.T) {
+	dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
+	if dsn == "" {
+		t.Skip("EVOBGP_TEST_DATABASE_URL not set")
+	}
+	ctx := context.Background()
+	pool, err := db.OpenPostgresPool(ctx, dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer pool.Close()
+	pg, err := NewPostgres(ctx, pool, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tenant, _, _, _, _ := pg.DemoIDs()
+	if tenant == "" {
+		t.Fatal("demo tenant required")
+	}
+	tok := "evobgp_fw_pgtest_" + t.Name()
+	hash := authkey.HashToken(tok)
+	client, err := pg.CreateFirewallClient(tenant, &store.FirewallClientCreate{
+		Name:          "pg-firewall-test",
+		Hostname:      "test.local",
+		TokenPrefix:   tok[:12],
+		TokenHash:     hash,
+		ClientVersion: "test/1",
+	})
+	if err != nil {
+		t.Fatalf("create: %v", err)
+	}
+	got, err := pg.GetFirewallClient(tenant, client.ID)
+	if err != nil {
+		t.Fatalf("get: %v", err)
+	}
+	if got.Name != "pg-firewall-test" || got.Status != "pending" {
+		t.Fatalf("got %+v", got)
+	}
+	_ = pg.DeleteFirewallClient(tenant, client.ID)
+}

scripts/firewall/evobgp-firewall.sh

@@ -5,6 +5,7 @@ CONF_FILE=/etc/evobgp/firewall.conf
 LOG_FILE=/var/log/evobgp-firewall.log
 STATE_DIR=/var/lib/evobgp-firewall
 HASH_FILE="${STATE_DIR}/last_hash"
+PREFIX_FILE="${STATE_DIR}/last_prefixes.txt"
 
 log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) $*" | tee -a "$LOG_FILE"; }
 
@@ -17,36 +18,32 @@ source "$CONF_FILE"
 
 : "${EVOBGP_CP_URL:?}"
 : "${CLIENT_TOKEN:?}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\r'/}"
+CLIENT_TOKEN="${CLIENT_TOKEN//$'\n'/}"
 
 mkdir -p "$STATE_DIR"
 BACKEND="${KERNEL_BACKEND:-auto}"
 
-curl_get_blocklist() {
+curl_get_blocklist_file() {
   local url="$1"
-  local host
-  host=$(echo "$url" | sed -E 's#https?://([^/]+)/?.*#\1#')
-  local tmp
-  tmp=$(mktemp)
+  local dest="$2"
   local code
-  code=$(curl -sS -o "$tmp" -w "%{http_code}" \
+  code=$(curl -sS -o "$dest" -w "%{http_code}" \
     -H "Authorization: Bearer ${CLIENT_TOKEN}" \
     -H "Accept: application/json" \
     "${url}/v1/firewall/blocklist") || return 1
   if [[ "$code" == "403" ]]; then
     log "pending approval"
-    rm -f "$tmp"
-    exit 0
+    return 2
   fi
   if [[ "$code" != "200" ]]; then
     log "blocklist HTTP $code from $url"
-    rm -f "$tmp"
     return 1
   fi
-  cat "$tmp"
-  rm -f "$tmp"
+  return 0
 }
 
-try_urls() {
+try_fetch_blocklist() {
   local urls=()
   if [[ -n "${EVOBGP_FAILOVER_URLS:-}" ]]; then
     IFS=',' read -r -a urls <<<"$EVOBGP_FAILOVER_URLS"
@@ -57,7 +54,12 @@ try_urls() {
   for u in "${urls[@]}"; do
     u="${u// /}"
     u="${u%/}"
-    if OUT=$(curl_get_blocklist "$u"); then
+    local rc=0
+    curl_get_blocklist_file "$u" "$PREFIX_FILE" || rc=$?
+    if [[ "$rc" == 2 ]]; then
+      exit 0
+    fi
+    if [[ "$rc" == 0 ]]; then
       CP_HIT="$u"
       return 0
     fi
@@ -65,22 +67,53 @@ try_urls() {
   return 1
 }
 
-if ! OUT=$(try_urls); then
+parse_blocklist_file() {
+  local f="$1"
+  if command -v jq >/dev/null 2>&1; then
+    HASH=$(jq -r '.hash // empty' "$f")
+    TOTAL=$(jq -r '.total // 0' "$f")
+    mapfile -t PREFIXES < <(jq -r '.prefixes[]? // empty' "$f")
+    return 0
+  fi
+  if command -v python3 >/dev/null 2>&1; then
+    local parsed
+    parsed=$(python3 - "$f" <<'PY'
+import json, sys
+with open(sys.argv[1], encoding="utf-8") as fh:
+    data = json.load(fh)
+print(data.get("hash") or "")
+print(data.get("total") or 0)
+for p in data.get("prefixes") or []:
+    if p:
+        print(p)
+PY
+    )
+    HASH=$(echo "$parsed" | sed -n '1p')
+    TOTAL=$(echo "$parsed" | sed -n '2p')
+    mapfile -t PREFIXES < <(echo "$parsed" | sed -n '3,$p')
+    return 0
+  fi
+  HASH=$(grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' "$f" | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
+  TOTAL=$(grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' "$f" | head -1 | grep -o '[0-9]*$' || true)
+  mapfile -t PREFIXES < <(grep -oE '"[0-9]+(\.[0-9]+){3}/[0-9]+"' "$f" | tr -d '"' || true)
+  return 0
+}
+
+if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
 fi
 
-if command -v jq >/dev/null 2>&1; then
-  HASH=$(echo "$OUT" | jq -r '.hash // empty')
-  TOTAL=$(echo "$OUT" | jq -r '.total // 0')
-  mapfile -t PREFIXES < <(echo "$OUT" | jq -r '.prefixes[]?')
-else
-  HASH=$(echo "$OUT" | grep -o '"hash"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"\(sha256:[^"]*\)".*/\1/')
-  TOTAL=$(echo "$OUT" | grep -o '"total"[[:space:]]*:[[:space:]]*[0-9]*' | head -1 | grep -o '[0-9]*$')
-  mapfile -t PREFIXES < <(echo "$OUT" | grep -o '"[0-9a-fA-F:.]*/[0-9]*"' | tr -d '"')
+HASH=""
+TOTAL=0
+PREFIXES=()
+parse_blocklist_file "$PREFIX_FILE"
+
+if [[ -z "${TOTAL// }" ]]; then
+  TOTAL=${#PREFIXES[@]}
 fi
 
-if [[ -f "$HASH_FILE" && "$(cat "$HASH_FILE")" == "$HASH" ]]; then
+if [[ -f "$HASH_FILE" && "$(tr -d '\r\n' <"$HASH_FILE")" == "$HASH" && -n "$HASH" ]]; then
   log "unchanged hash $HASH — skip kernel apply"
   exit 0
 fi
@@ -88,48 +121,67 @@ fi
 apply_nft() {
   local table=inet
   local name=evobgp_blocklist
+  local v4=()
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    v4+=("$p")
+  done
+
   nft list table "$table" "$name" >/dev/null 2>&1 || nft add table "$table" "$name"
-  nft list set "$table" "$name" v4 >/dev/null 2>&1 || nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
+  nft list set "$table" "$name" v4 >/dev/null 2>&1 || \
+    nft add set "$table" "$name" v4 '{ type ipv4_addr; flags interval; }'
   nft flush set "$table" "$name" v4
-  if ((${#PREFIXES[@]})); then
-    local v4=()
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      v4+=("$p")
+
+  if ((${#v4[@]})); then
+    local batch=()
+    local chunk=128
+    local n
+    for p in "${v4[@]}"; do
+      batch+=("$p")
+      if ((${#batch[@]} >= chunk)); then
+        nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+        batch=()
+      fi
     done
-    if ((${#v4[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${v4[*]}") }"
+    if ((${#batch[@]})); then
+      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
     fi
   fi
+
   nft list chain "$table" "$name" input >/dev/null 2>&1 || {
-    nft add chain "$table" "$name" input '{ type filter hook input priority 0; }'
+    nft add chain "$table" "$name" input '{ type filter hook input priority 0; policy accept; }'
     nft add rule "$table" "$name" input ip saddr @v4 drop
   }
+  APPLIED_V4=${#v4[@]}
 }
 
 apply_ipset() {
   local set=evobgp_blocklist_v4
+  local n=0
   ipset list "$set" >/dev/null 2>&1 || ipset create "$set" hash:net family inet hashsize 4096 maxelem 1048576
   ipset flush "$set"
   local p
   for p in "${PREFIXES[@]}"; do
     [[ "$p" == *:* ]] && continue
     ipset add "$set" "$p" -exist
+    n=$((n + 1))
   done
   iptables -C INPUT -m set --match-set "$set" src -j DROP 2>/dev/null || \
     iptables -I INPUT -m set --match-set "$set" src -j DROP
+  APPLIED_V4=$n
 }
 
 apply_iptables_only() {
   iptables -D INPUT -m comment --comment evobgp-block -j DROP 2>/dev/null || true
-  if ((${#PREFIXES[@]})); then
-    local p
-    for p in "${PREFIXES[@]}"; do
-      [[ "$p" == *:* ]] && continue
-      iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
-    done
-  fi
+  local n=0
+  local p
+  for p in "${PREFIXES[@]}"; do
+    [[ "$p" == *:* ]] && continue
+    iptables -C INPUT -s "$p" -j DROP 2>/dev/null || iptables -A INPUT -s "$p" -j DROP
+    n=$((n + 1))
+  done
+  APPLIED_V4=$n
 }
 
 clear_block() {
@@ -141,10 +193,13 @@ clear_block() {
       ;;
     iptables) iptables -S INPUT | grep -i evobgp | sed 's/^-A /-D /' | while read -r line; do iptables $line 2>/dev/null || true; done ;;
   esac
+  APPLIED_V4=0
 }
 
-if [[ "$TOTAL" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
+APPLIED_V4=0
+if [[ "${TOTAL:-0}" == "0" || ${#PREFIXES[@]} -eq 0 ]]; then
   clear_block
+  log "cleared blocklist (api total=${TOTAL:-0}) backend=$BACKEND"
 else
   case "$BACKEND" in
     nft|auto) if command -v nft >/dev/null 2>&1; then apply_nft; else apply_ipset; fi ;;
@@ -152,12 +207,12 @@ else
     iptables) apply_iptables_only ;;
     *) apply_ipset ;;
   esac
+  log "applied api_total=${TOTAL} ipv4_in_kernel=${APPLIED_V4} from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND hash=${HASH:-empty}"
 fi
 
 echo "$HASH" >"$HASH_FILE"
-log "applied $TOTAL prefixes from ${CP_HIT:-$EVOBGP_CP_URL} backend=$BACKEND"
 
-REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":0,"source":"cp"}' "${TOTAL:-0}")
+REPORT=$(printf '{"status":"ok","prefix_count":%s,"ip_count":%s,"source":"cp"}' "${TOTAL:-0}" "${APPLIED_V4:-0}")
 curl -fsS -X POST "${EVOBGP_CP_URL%/}/v1/firewall/apply-report" \
   -H "Authorization: Bearer ${CLIENT_TOKEN}" \
   -H "Content-Type: application/json" \

scripts/firewall/install.sh

@@ -10,6 +10,16 @@ for cmd in curl bash; do
   command -v "$cmd" >/dev/null 2>&1 || { echo "missing $cmd" >&2; exit 1; }
 done
 
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  if command -v apt-get >/dev/null 2>&1; then
+    apt-get update -qq && apt-get install -y -qq jq
+  fi
+fi
+if ! command -v jq >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
+  echo "evobgp-firewall install: install jq or python3 for blocklist JSON parsing" >&2
+  exit 1
+fi
+
 : "${EVOBGP_CP_URL:?EVOBGP_CP_URL required}"
 : "${EVOBGP_SEED:?EVOBGP_SEED required}"
 : "${EVOBGP_CLIENT_NAME:?EVOBGP_CLIENT_NAME required}"
@@ -38,10 +48,18 @@ CP_URL="${EVOBGP_CP_URL%/}"
 ENROLL_BODY=$(printf '{"name":"%s","hostname":"%s","client_token":"%s","client_version":"install.sh/1"}' \
   "$EVOBGP_CLIENT_NAME" "$HOSTNAME" "$CLIENT_TOKEN")
 
-RESP=$(curl -fsS -X POST "${CP_URL}/v1/firewall/enroll" \
+ENROLL_TMP=$(mktemp)
+trap 'rm -f "$ENROLL_TMP"' EXIT
+ENROLL_CODE=$(curl -sS -o "$ENROLL_TMP" -w "%{http_code}" -X POST "${CP_URL}/v1/firewall/enroll" \
   -H "Content-Type: application/json" \
   -H "X-EvoBGP-Seed: ${EVOBGP_SEED}" \
   -d "$ENROLL_BODY")
+if [[ "$ENROLL_CODE" != "201" ]]; then
+  echo "evobgp-firewall enroll failed: HTTP ${ENROLL_CODE} from ${CP_URL}/v1/firewall/enroll" >&2
+  cat "$ENROLL_TMP" >&2
+  exit 1
+fi
+RESP=$(cat "$ENROLL_TMP")
 
 CLIENT_ID=""
 if command -v jq >/dev/null 2>&1; then
@@ -102,6 +120,7 @@ WantedBy=timers.target
 UNIT
   systemctl daemon-reload
   systemctl enable --now evobgp-firewall.timer
+  echo "Tip: after UI approve, run: rm -f /var/lib/evobgp-firewall/last_hash && ${SYNC_SCRIPT}"
 else
   echo "*/5 * * * * root ${SYNC_SCRIPT}" >/etc/cron.d/evobgp-firewall
 fi