feat(firewall): improve blocklist parsing and nft element addition

Enhanced the blocklist parsing function to log when the blocklist file is empty. Introduced new helper functions `nft_join_elements` and `nft_add_v4_chunk` to streamline the addition of elements to the nftables, allowing for batch processing and improved error handling. Adjusted the chunk size for element addition to optimize performance. Updated logging to provide better visibility into the blocklist processing and applied prefixes.

internal/firewallscripts/evobgp-firewall.sh

@@ -69,6 +69,10 @@ try_fetch_blocklist() {
 
 parse_blocklist_file() {
   local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
   if command -v jq >/dev/null 2>&1; then
     HASH=$(jq -r '.hash // empty' "$f")
     TOTAL=$(jq -r '.total // 0' "$f")
@@ -99,6 +103,35 @@ PY
   return 0
 }
 
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
 if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
@@ -108,6 +141,7 @@ HASH=""
 TOTAL=0
 PREFIXES=()
 parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
 
 if [[ -z "${TOTAL// }" ]]; then
   TOTAL=${#PREFIXES[@]}
@@ -135,17 +169,16 @@ apply_nft() {
 
   if ((${#v4[@]})); then
     local batch=()
-    local chunk=128
-    local n
+    local chunk=64
     for p in "${v4[@]}"; do
       batch+=("$p")
       if ((${#batch[@]} >= chunk)); then
-        nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
         batch=()
       fi
     done
     if ((${#batch[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi
 

scripts/firewall/evobgp-firewall.sh

@@ -69,6 +69,10 @@ try_fetch_blocklist() {
 
 parse_blocklist_file() {
   local f="$1"
+  if [[ ! -s "$f" ]]; then
+    log "blocklist file empty: $f"
+    return 1
+  fi
   if command -v jq >/dev/null 2>&1; then
     HASH=$(jq -r '.hash // empty' "$f")
     TOTAL=$(jq -r '.total // 0' "$f")
@@ -99,6 +103,35 @@ PY
   return 0
 }
 
+nft_join_elements() {
+  local out="" p
+  for p in "$@"; do
+    if [[ -n "$out" ]]; then
+      out+=", "
+    fi
+    out+="$p"
+  done
+  printf '%s' "$out"
+}
+
+nft_add_v4_chunk() {
+  local table=$1 name=$2
+  shift 2
+  local joined
+  joined=$(nft_join_elements "$@")
+  if nft add element "$table" "$name" v4 "{ ${joined} }" 2>>"$LOG_FILE"; then
+    return 0
+  fi
+  log "nft batch add failed (chunk=$#), retrying one-by-one"
+  local p ok=0
+  for p in "$@"; do
+    if nft add element "$table" "$name" v4 "{ $p }" 2>>"$LOG_FILE"; then
+      ok=$((ok + 1))
+    fi
+  done
+  [[ "$ok" -gt 0 ]]
+}
+
 if ! try_fetch_blocklist; then
   log "all endpoints failed"
   exit 1
@@ -108,6 +141,7 @@ HASH=""
 TOTAL=0
 PREFIXES=()
 parse_blocklist_file "$PREFIX_FILE"
+log "blocklist bytes=$(wc -c <"$PREFIX_FILE" | tr -d ' ') parsed=${#PREFIXES[@]} api_total=${TOTAL:-0}"
 
 if [[ -z "${TOTAL// }" ]]; then
   TOTAL=${#PREFIXES[@]}
@@ -135,17 +169,16 @@ apply_nft() {
 
   if ((${#v4[@]})); then
     local batch=()
-    local chunk=128
-    local n
+    local chunk=64
     for p in "${v4[@]}"; do
       batch+=("$p")
       if ((${#batch[@]} >= chunk)); then
-        nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+        nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft chunk add partial failure"
         batch=()
       fi
     done
     if ((${#batch[@]})); then
-      nft add element "$table" "$name" v4 "{ $(IFS=,; echo "${batch[*]}") }"
+      nft_add_v4_chunk "$table" "$name" "${batch[@]}" || log "nft tail chunk add partial failure"
     fi
   fi