feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <[email protected]>
This commit is contained in:
DenozordecandCursor committed 2026-11-02 17:23:59 +07:00
1 parent 924f111e48
commit f8e0520817
197 files changed
+18498 -1259

No files matched your search

+71
View File
@@ -322,6 +322,66 @@ jobs:
go build -o "$out/$name" "./$d" go build -o "$out/$name" "./$d"
done done
postgres-integration:
needs: [changes]
if: needs.changes.outputs.go == 'true'
runs-on: ubuntu-latest
services:
postgres:
image: postgres:16
env:
POSTGRES_USER: evobgp
POSTGRES_PASSWORD: evobgp_ci
POSTGRES_DB: evobgp_test
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U evobgp -d evobgp_test"
--health-interval 5s
--health-timeout 5s
--health-retries 12
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version: "1.24"
cache: false
- name: Export cache paths
run: sh scripts/ci/export-cache-env.sh
- id: go-hash
run: echo "key=$(sha256sum go.sum | awk '{print $1}')" >> "$GITHUB_OUTPUT"
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
with:
path: |
${{ env.GOMODCACHE }}
${{ env.GOCACHE }}
key: go-${{ runner.os }}-1.24-postgres-${{ steps.go-hash.outputs.key }}
restore-keys: |
go-${{ runner.os }}-1.24-postgres-
- name: Run migrations and verify PostgreSQL integration tests executed
env:
GOFLAGS: -mod=readonly
EVOBGP_TEST_DATABASE_URL: postgres://evobgp:[email protected]:5432/evobgp_test?sslmode=disable
run: bash scripts/ci/postgres-integration.sh
legacy-compatibility:
needs: [changes]
if: needs.changes.outputs.go == 'true'
runs-on: ubuntu-latest
strategy:
matrix:
legacy_version: ["2.56.2", "1.0.0"]
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version: "1.24"
cache: false
- name: Build pinned legacy executables and test current control plane
env:
EVOBGP_LEGACY_VERSION: ${{ matrix.legacy_version }}
run: bash scripts/ci/legacy-node-compatibility.sh
bird2: bird2:
runs-on: ubuntu-latest runs-on: ubuntu-latest
needs: [changes, go] needs: [changes, go]
@@ -335,6 +395,10 @@ jobs:
needs.changes.outputs.docker_go == 'true') needs.changes.outputs.docker_go == 'true')
steps: steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
with:
go-version: "1.24"
cache: false
- name: Install bird2 (репозиторий Ubuntu runner, как в образе evobgp-bird2) - name: Install bird2 (репозиторий Ubuntu runner, как в образе evobgp-bird2)
run: | run: |
set -euxo pipefail set -euxo pipefail
@@ -358,6 +422,11 @@ jobs:
echo "==> $conf" echo "==> $conf"
bird -c "$WS/$conf" -p bird -c "$WS/$conf" -p
done done
- name: Parse staged birddeploy generation with real BIRD
env:
GOFLAGS: -mod=readonly
EVOBGP_TEST_BIRD_BIN: bird
run: go test ./internal/birddeploy -run '^TestRealBirdParseGeneration$' -count=1 -v
commitlint: commitlint:
if: inputs.is_pull_request if: inputs.is_pull_request
@@ -405,6 +474,8 @@ jobs:
runs-on: ubuntu-latest runs-on: ubuntu-latest
steps: steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Offline mirror behavior tests
run: bash scripts/tests/mirror-base-images.test.sh
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
with: with:
name: evobgp name: evobgp
+3
View File
@@ -32,3 +32,6 @@ deploy/compose/runtime-logs/
.cursor/mcp.json .cursor/mcp.json
.mcp.json .mcp.json
.codegraph/daemon.pid .codegraph/daemon.pid
# Local backend resource snapshots (host metadata, not source artifacts)
artifacts/backend-baseline/
+79 -18
View File
@@ -6,10 +6,14 @@ import (
"fmt" "fmt"
"log" "log"
"os" "os"
"os/signal"
"strings"
"sync" "sync"
"syscall"
"time" "time"
"evobgp/internal/agentserver" "evobgp/internal/agentserver"
"evobgp/internal/agentwatch"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
) )
@@ -18,19 +22,22 @@ func main() {
birdc := flag.String("birdc", "", "path to birdc binary (default: birdc from PATH)") birdc := flag.String("birdc", "", "path to birdc binary (default: birdc from PATH)")
socket := flag.String("socket", "", "optional birdc control socket (-s)") socket := flag.String("socket", "", "optional birdc control socket (-s)")
timeout := flag.Duration("timeout", 30*time.Second, "timeout for bird/birdc") timeout := flag.Duration("timeout", 30*time.Second, "timeout for bird/birdc")
watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between birdc configure") watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between configuration checks")
listen := flag.String("listen", "", "for serve: listen address (default :8443 or EVOBGP_AGENT_LISTEN)") listen := flag.String("listen", "", "for serve: listen address (default :8443 or EVOBGP_AGENT_LISTEN)")
flag.Usage = func() { flag.Usage = func() {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <command>\n", os.Args[0]) fmt.Fprintf(os.Stderr, "Usage: %s [flags] <command>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "Commands:\n") fmt.Fprintf(os.Stderr, "Commands:\n")
fmt.Fprintf(os.Stderr, " parse-check <path/to/bird.conf> run bird -c <path> -p (syntax check)\n") fmt.Fprintf(os.Stderr, " parse-check <path/to/bird.conf> run bird -c <path> -p (syntax check)\n")
fmt.Fprintf(os.Stderr, " configure run birdc configure (reload running BIRD)\n") fmt.Fprintf(os.Stderr, " configure run birdc configure (reload running BIRD)\n")
fmt.Fprintf(os.Stderr, " watch periodically run birdc configure (compose sidecar)\n") fmt.Fprintf(os.Stderr, " watch reload changed BIRD configuration and periodically reconcile\n")
fmt.Fprintf(os.Stderr, " serve Panel→Node HTTP API (POST /v1/agent/sync)\n") fmt.Fprintf(os.Stderr, " serve Panel→Node HTTP API (POST /v1/agent/sync)\n")
flag.PrintDefaults() flag.PrintDefaults()
} }
flag.Parse() args, err := parseAgentArgs(flag.CommandLine, os.Args[1:])
args := flag.Args() if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
if len(args) < 1 { if len(args) < 1 {
flag.Usage() flag.Usage()
os.Exit(2) os.Exit(2)
@@ -47,10 +54,12 @@ func main() {
switch args[0] { switch args[0] {
case "serve": case "serve":
runServe(*listen, *timeout) runServe(*listen, *timeout)
case "parse-check", "configure", "watch": case "watch":
runWatch(ctl, *watchEvery, *timeout)
case "parse-check", "configure":
ctx, cancel := context.WithTimeout(context.Background(), *timeout) ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel() defer cancel()
runBirdCommand(ctx, args, ctl, *watchEvery, *timeout) runBirdCommand(ctx, args, ctl)
default: default:
fmt.Fprintf(os.Stderr, "unknown command: %s\n", args[0]) fmt.Fprintf(os.Stderr, "unknown command: %s\n", args[0])
flag.Usage() flag.Usage()
@@ -58,7 +67,47 @@ func main() {
} }
} }
func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, watchEvery, timeout time.Duration) { // parseAgentArgs accepts flags before or immediately after the subcommand.
// A second parse preserves previously supplied flag values and rejects unknown
// flags instead of silently passing them to watch or serve as positional args.
func parseAgentArgs(fs *flag.FlagSet, argv []string) ([]string, error) {
if err := fs.Parse(argv); err != nil {
return nil, err
}
args := fs.Args()
if len(args) == 0 {
return args, nil
}
command := args[0]
globalStop := false
for i, consumed := 0, len(argv)-len(args); i < consumed; i++ {
if argv[i] == "--" {
globalStop = true
break
}
name, _, hasValue := strings.Cut(strings.TrimLeft(argv[i], "-"), "=")
if !hasValue {
if f := fs.Lookup(name); f != nil {
boolean, ok := f.Value.(interface{ IsBoolFlag() bool })
if !ok || !boolean.IsBoolFlag() {
i++
}
}
}
}
if !globalStop {
if err := fs.Parse(args[1:]); err != nil {
return nil, err
}
args = append([]string{command}, fs.Args()...)
}
if (command == "watch" || command == "serve" || command == "configure") && len(args) != 1 {
return nil, fmt.Errorf("%s takes no positional arguments", command)
}
return args, nil
}
func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl) {
switch args[0] { switch args[0] {
case "parse-check": case "parse-check":
if len(args) != 2 { if len(args) != 2 {
@@ -78,21 +127,33 @@ func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, wa
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
os.Exit(1) os.Exit(1)
} }
case "watch": }
if watchEvery <= 0 { }
fmt.Fprintln(os.Stderr, "watch-interval must be > 0")
func runWatch(ctl *birdfmt.BirdCtl, every, timeout time.Duration) {
if every <= 0 || timeout <= 0 {
fmt.Fprintln(os.Stderr, "watch-interval and timeout must be > 0")
os.Exit(2) os.Exit(2)
} }
log.Printf("evobgp-agent watch: birdc configure every %s (socket=%q)", watchEvery, ctl.Socket) reconcile := 5 * time.Minute
for { if raw := os.Getenv("EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL"); raw != "" {
cctx, cancel := context.WithTimeout(context.Background(), timeout) var err error
err := ctl.Configure(cctx) reconcile, err = time.ParseDuration(raw)
cancel() if err != nil || reconcile <= 0 {
if err != nil { fmt.Fprintln(os.Stderr, "EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL must be a positive duration")
log.Printf("evobgp-agent watch: configure: %v", err) os.Exit(2)
} }
time.Sleep(watchEvery)
} }
active := os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")
if active == "" {
active = "/etc/bird"
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
log.Printf("evobgp-agent watch: checking %s every %s, forced reconciliation every %s (socket=%q)", active, every, reconcile, ctl.Socket)
if err := agentwatch.Run(ctx, ctl, agentwatch.Config{ActiveDir: active, Every: every, ReconcileEvery: reconcile, CommandTimeout: timeout}); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
} }
} }
+49
View File
@@ -0,0 +1,49 @@
package main
import (
"flag"
"io"
"reflect"
"testing"
"time"
)
func TestAgentFlagsBeforeAndAfterCommand(t *testing.T) {
for _, tc := range []struct {
name string
args []string
want []string
socket, listen string
every time.Duration
}{
{"legacy flags first", []string{"-socket=/custom/bird.ctl", "-watch-interval=45s", "watch"}, []string{"watch"}, "/custom/bird.ctl", "", 45 * time.Second},
{"compose command first", []string{"watch", "-socket=/run/bird/bird.ctl", "-watch-interval=30s"}, []string{"watch"}, "/run/bird/bird.ctl", "", 30 * time.Second},
{"serve", []string{"serve", "-listen=127.0.0.1:9999"}, []string{"serve"}, "", "127.0.0.1:9999", time.Minute},
{"mixed", []string{"-socket=/first", "watch", "-watch-interval=5s"}, []string{"watch"}, "/first", "", 5 * time.Second},
{"parse check", []string{"parse-check", "bird.conf"}, []string{"parse-check", "bird.conf"}, "", "", time.Minute},
{"global delimiter", []string{"--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "", "", time.Minute},
{"flags then delimiter", []string{"-socket=/first", "--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "/first", "", time.Minute},
{"delimiter as value", []string{"-socket", "--", "watch", "-watch-interval=5s"}, []string{"watch"}, "--", "", 5 * time.Second},
} {
t.Run(tc.name, func(t *testing.T) {
fs := flag.NewFlagSet("agent", flag.ContinueOnError)
fs.SetOutput(io.Discard)
socket := fs.String("socket", "", "")
listen := fs.String("listen", "", "")
every := fs.Duration("watch-interval", time.Minute, "")
args, err := parseAgentArgs(fs, tc.args)
if err != nil || !reflect.DeepEqual(args, tc.want) || *socket != tc.socket || *listen != tc.listen || *every != tc.every {
t.Fatalf("args=%v socket=%s listen=%s every=%s err=%v", args, *socket, *listen, *every, err)
}
})
}
}
func TestAgentRejectsIgnoredFlagsAndUnexpectedArguments(t *testing.T) {
for _, argv := range [][]string{{"watch", "-unknown=1"}, {"serve", "-unknown=1"}, {"watch", "unused"}, {"serve", "unused"}, {"configure", "unused"}} {
fs := flag.NewFlagSet("agent", flag.ContinueOnError)
fs.SetOutput(io.Discard)
if _, err := parseAgentArgs(fs, argv); err == nil {
t.Fatalf("ignored argv: %v", argv)
}
}
}
+31 -13
View File
@@ -41,23 +41,41 @@ func main() {
apiBase := strings.TrimSpace(os.Getenv("EVOBGP_CONTROL_PLANE_URL")) apiBase := strings.TrimSpace(os.Getenv("EVOBGP_CONTROL_PLANE_URL"))
apiTok := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_BEARER")) apiTok := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_BEARER"))
if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" && apiBase == "" { mode := strings.ToLower(strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_MODE")))
log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_STANDALONE=1 requires EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER (split deploy must not use in-process jobs.Registry)") if mode == "" {
switch {
case apiBase != "" || apiTok != "":
mode = "http"
case pool != nil:
mode = "db"
default:
mode = "local"
} }
deps := &scheduler.Deps{
Store: st,
HTTP: &http.Client{Timeout: 45 * time.Second},
} }
if (apiBase != "") != (apiTok != "") { deps := &scheduler.Deps{Store: st, HTTP: &http.Client{Timeout: 45 * time.Second}}
log.Fatal("evobgp-scheduler: set both EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER for HTTP mode, or leave both empty for in-process job registry") switch mode {
case "db":
if pool == nil {
log.Fatal("evobgp-scheduler: db mode requires EVOBGP_DATABASE_URL")
} }
if apiBase != "" {
deps.APIBase = apiBase
deps.APIToken = apiTok
log.Printf("evobgp-scheduler: control plane HTTP mode (%s)", apiBase)
} else {
deps.Jobs = reg deps.Jobs = reg
log.Printf("evobgp-scheduler: in-process job registry mode (use EVOBGP_CONTROL_PLANE_URL + EVOBGP_SCHEDULER_BEARER for split containers)") log.Print("evobgp-scheduler: trusted DB enqueue mode; API workers consume the shared durable queue")
case "http":
tenant := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_TENANT_ID"))
if apiBase == "" || apiTok == "" || tenant == "" {
log.Fatal("evobgp-scheduler: http mode requires EVOBGP_CONTROL_PLANE_URL, EVOBGP_SCHEDULER_BEARER and EVOBGP_SCHEDULER_TENANT_ID")
}
deps.APIBase, deps.APIToken, deps.APITenantID = apiBase, apiTok, tenant
log.Print("evobgp-scheduler: single-tenant HTTP enqueue mode")
case "local":
if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" || pool != nil {
log.Fatal("evobgp-scheduler: local mode is for in-process development only; use db for a shared PostgreSQL queue")
}
deps.Jobs = reg
defer func() { reg.RequestCancelAll(); _ = reg.Drain(context.Background()) }()
log.Print("evobgp-scheduler: local development mode")
default:
log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_MODE must be db, http or local")
} }
scheduler.Run(ctx, deps) scheduler.Run(ctx, deps)
} }
+213 -9
View File
@@ -10,6 +10,20 @@ info:
**Соглашения:** префикс `/v1`; идентификаторы - UUID v7 или ULID (строки); время - ISO 8601 UTC. **Соглашения:** префикс `/v1`; идентификаторы - UUID v7 или ULID (строки); время - ISO 8601 UTC.
Ошибки - `application/problem+json` ([RFC 9457](https://www.rfc-editor.org/rfc/rfc9457)). Ошибки - `application/problem+json` ([RFC 9457](https://www.rfc-editor.org/rfc/rfc9457)).
Пагинация списков - `cursor` + `limit`; ответ содержит `items`, `next_cursor`, `has_more`. Пагинация списков - `cursor` + `limit`; ответ содержит `items`, `next_cursor`, `has_more`.
**Общий предел request body:** после аутентификации каждый непубличный `/v1/...`
запрос проверяется целиком до выполнения handler: максимум 8 MiB (8388608 bytes),
независимо от HTTP-метода, Content-Type и наличия Content-Length. Превышение,
включая whitespace после JSON, возвращает `413 application/problem+json`
до мутаций. Ошибка чтения тела возвращает `400`, отмена/таймаут чтения — `408`.
Пустые node GET/pull и streaming SSE responses не меняются. CSV дополнительно
сохраняет ограничения числа строк и полей своей операции.
Запросы с телом занимают ограниченный слот API instance до окончания handler:
по умолчанию 4 одновременно, `EVOBGP_API_BODY_MAX_CONCURRENT` задаёт 1..64.
Свободных слотов нет — немедленный `429` с `Retry-After: 1`, без очереди ожидания.
Пустые запросы (`http.NoBody`) слота не занимают.
Если CRUD-изменение сохранено, но автоматическая refresh/reconcile задача не поставлена,
успешный ответ содержит `X-EvoBGP-Job-Status: enqueue-failed`; ошибка также попадает в аудит.
Изменение уже сохранено: следует явно запустить refresh/reconcile, а не повторять создание объекта.
**Аутентификация (dual):** **Аутентификация (dual):**
- **API key** — `Authorization: Bearer <token>` из `EVOBGP_API_KEYS` / таблицы `api_key` (роли `viewer`/`editor`/`operator`/`node`/`firewall`). - **API key** — `Authorization: Bearer <token>` из `EVOBGP_API_KEYS` / таблицы `api_key` (роли `viewer`/`editor`/`operator`/`node`/`firewall`).
@@ -226,6 +240,16 @@ components:
Основные значения: `module_refresh`, `peer_reconcile`, `deploy_apply`, `revision_rollback`, `bird_reload`. Основные значения: `module_refresh`, `peer_reconcile`, `deploy_apply`, `revision_rollback`, `bird_reload`.
responses: responses:
ExpensiveOperationLimited:
description: Локальная квота процесса API — по умолчанию 4 активных preview/lookup, 2 на tenant; 120 lookup и по 30 CDN/revision preview в минуту на tenant. Квоты умножаются на число реплик. Повторить после Retry-After, без очереди ожидания.
headers:
Retry-After:
description: Минимальная задержка повторной попытки в секундах; не гарантия свободного места.
schema: {type: integer, minimum: 1}
content:
application/problem+json:
schema:
$ref: "#/components/schemas/Problem"
Unauthorized: Unauthorized:
description: Нет или невалидный Bearer-токен. description: Нет или невалидный Bearer-токен.
content: content:
@@ -263,7 +287,10 @@ components:
schema: schema:
$ref: "#/components/schemas/Problem" $ref: "#/components/schemas/Problem"
DefaultProblem: DefaultProblem:
description: Ошибка (см. тело Problem). description: >
Ошибка (см. тело Problem). Общие для authenticated `/v1` ответы middleware:
413 при request body свыше 8 MiB, 400 при ошибке чтения тела, 408 при отмене/таймауте чтения.
429 с Retry-After: 1 при занятых слотах обработки request body.
content: content:
application/problem+json: application/problem+json:
schema: schema:
@@ -903,6 +930,21 @@ components:
connected_speaker_label: connected_speaker_label:
type: string type: string
description: Человекочитаемая метка ноды из live-опроса. description: Человекочитаемая метка ноды из live-опроса.
last_known_session_state:
type: ["string", "null"]
description: Последнее сохранённое состояние; не является текущим наблюдением.
session_state:
type: ["string", "null"]
description: >
При live-ответе текущее наблюдение: BGP state, absent после успешного
пустого опроса или null, если актуальное состояние неизвестно.
live_observation:
$ref: "#/components/schemas/LiveObservation"
description: >
Качество BGP-наблюдений при GET /v1/peers?live=1. Для назначенного
пира учитывается только bgp_speaker_id; для общего — все speakers.
Смешанный успех и сбой не дают quality=ok или session_state=absent.
Timestamps относятся только к ожидаемым speakers.
session_on_speakers: session_on_speakers:
type: array type: array
description: Состояние протокола пира на каждой опрошенной ноде. description: Состояние протокола пира на каждой опрошенной ноде.
@@ -915,6 +957,10 @@ components:
type: string type: string
state: state:
type: string type: string
poll_error:
type: string
observation:
$ref: "#/components/schemas/LiveObservation"
established_on_speakers: established_on_speakers:
type: array type: array
description: Ноды, где сессия в состоянии Established (один пир может быть на нескольких). description: Ноды, где сессия в состоянии Established (один пир может быть на нескольких).
@@ -1001,7 +1047,11 @@ components:
description: Человекочитаемая метка ноды (agent domain или CP master). description: Человекочитаемая метка ноды (agent domain или CP master).
agent_ok: agent_ok:
type: boolean type: boolean
description: true если agent /v1/agent/health успешен (master — local birdc poll). description: true только после успешного agent /v1/agent/health (master — local birdc poll).
agent_observation:
$ref: "#/components/schemas/LiveObservation"
bgp_observation:
$ref: "#/components/schemas/LiveObservation"
agent_error: agent_error:
type: string type: string
agent_last_sync_at: agent_last_sync_at:
@@ -1015,15 +1065,49 @@ components:
bgp_poll_error: bgp_poll_error:
type: string type: string
bgp_sessions_total: bgp_sessions_total:
type: integer type: ["integer", "null"]
description: null при отсутствии успешного опроса.
bgp_established: bgp_established:
type: integer type: ["integer", "null"]
description: null при отсутствии успешного опроса.
sessions: sessions:
type: array type: array
items: items:
$ref: "#/components/schemas/BgpSessionLive" $ref: "#/components/schemas/BgpSessionLive"
additionalProperties: true additionalProperties: true
LiveObservation:
type: object
description: >
Метаданные конкретного опроса. observed_at — время завершения опроса;
received_at — время получения результата CP; last_success_at — время
последнего успешного опроса. Старые агенты не обязаны передавать эти поля:
CP формирует их по собственным часам. При отсутствии опроса timestamps null.
required: [observed_at, received_at, last_success_at, source, quality]
properties:
observed_at:
type: ["string", "null"]
format: date-time
received_at:
type: ["string", "null"]
format: date-time
last_success_at:
type: ["string", "null"]
format: date-time
expires_at:
type: ["string", "null"]
format: date-time
description: Срок актуальности snapshot по часам CP и настроенному интервалу collector; клиент не должен считать cached quality=ok актуальным после этого времени. Для агрегата — самый ранний срок источников.
source:
type: string
enum: [none, cp_birdc, agent_bird_protocols, agent_health, mixed]
quality:
type: string
enum: [ok, stale, unreachable, unknown, disabled]
error_code:
type: string
enum: [not_configured, not_polled, poll_failed, reported_unhealthy, snapshot_expired, configuration_changed, ambiguous_authority]
BgpSessionLive: BgpSessionLive:
type: object type: object
properties: properties:
@@ -1052,9 +1136,12 @@ components:
ok: ok:
type: boolean type: boolean
session_count: session_count:
type: integer type: ["integer", "null"]
description: null, если успешного опроса не было.
poll_error: poll_error:
type: string type: string
observation:
$ref: "#/components/schemas/LiveObservation"
additionalProperties: true additionalProperties: true
PostgresOverview: PostgresOverview:
@@ -2108,6 +2195,8 @@ paths:
application/json: application/json:
schema: schema:
$ref: "#/components/schemas/LookupResponse" $ref: "#/components/schemas/LookupResponse"
"429":
$ref: "#/components/responses/ExpensiveOperationLimited"
"400": "400":
$ref: "#/components/responses/BadRequest" $ref: "#/components/responses/BadRequest"
"401": "401":
@@ -2115,6 +2204,29 @@ paths:
default: default:
$ref: "#/components/responses/DefaultProblem" $ref: "#/components/responses/DefaultProblem"
/v1/telemetry/refresh:
parameters:
- $ref: "#/components/parameters/TenantId"
post:
tags: [Speakers]
summary: Запросить фоновое обновление телеметрии tenant
description: Требует bgp:network:read. Объединяет повторные запросы в пределах 10 секунд; HTTP handler не опрашивает агентов или BIRD. GET speakers/peers/discoveries читает сохранённые snapshots, включая live=1. Запрос во время выполняющегося poll сохраняется для следующего цикла. Не изменяет конфигурацию speaker и не требует нового протокола агента.
operationId: requestTelemetryRefresh
responses:
"202":
description: Запрос принят либо объединён с уже принятым.
content:
application/json:
schema:
type: object
required: [requested]
properties:
requested:
type: boolean
description: true при постановке нового запроса; false при объединении или отсутствии целей.
default:
$ref: "#/components/responses/DefaultProblem"
/v1/router-lists/catalog: /v1/router-lists/catalog:
get: get:
tags: [Modules] tags: [Modules]
@@ -2789,6 +2901,9 @@ paths:
- `ipRange,community` - `ipRange,community`
В поле `community` можно передавать либо ID community, либо её значение. В поле `community` можно передавать либо ID community, либо её значение.
Предел запроса — 8 MiB (8388608 байт), 10000 строк данных без заголовка и 64 поля в записи.
Превышение возвращает 413 до создания записей; большой импорт следует разбить на части.
Ошибка в данных или БД после начала создания записей может оставить частичный импорт.
operationId: importModuleEntriesCsv operationId: importModuleEntriesCsv
parameters: parameters:
- $ref: "#/components/parameters/IdempotencyKey" - $ref: "#/components/parameters/IdempotencyKey"
@@ -2812,6 +2927,12 @@ paths:
minimum: 0 minimum: 0
module_type: module_type:
type: string type: string
"413":
description: Превышен лимит байт, строк или полей CSV; записи не созданы.
content:
application/problem+json:
schema:
$ref: "#/components/schemas/Problem"
"404": "404":
$ref: "#/components/responses/NotFound" $ref: "#/components/responses/NotFound"
"422": "422":
@@ -2898,6 +3019,8 @@ paths:
application/json: application/json:
schema: schema:
$ref: "#/components/schemas/CdnPreviewResponse" $ref: "#/components/schemas/CdnPreviewResponse"
"429":
$ref: "#/components/responses/ExpensiveOperationLimited"
"401": "401":
$ref: "#/components/responses/Unauthorized" $ref: "#/components/responses/Unauthorized"
"403": "403":
@@ -3146,7 +3269,7 @@ paths:
type: string type: string
enum: ["1"] enum: ["1"]
description: > description: >
Опрос birdc на CP и GET /v1/agent/bird/protocols на репликах; обогащает session_state и connected_speaker_*. Сохранённые результаты фонового опроса birdc на CP и agent protocols на репликах; обогащает session_state и connected_speaker_*. GET не запускает опрос; live=1 сохранён для совместимости.
responses: responses:
"200": "200":
description: Успешно. description: Успешно.
@@ -3167,7 +3290,7 @@ paths:
live_speaker_poll: live_speaker_poll:
type: array type: array
description: > description: >
При live=1 — результат опроса каждой ноды (CP birdc + agent protocols). Сохранённый результат фонового опроса каждой ноды (CP birdc + agent protocols); live=1 не запускает новый опрос.
items: items:
$ref: "#/components/schemas/LiveSpeakerPoll" $ref: "#/components/schemas/LiveSpeakerPoll"
default: default:
@@ -3204,7 +3327,7 @@ paths:
summary: Список обнаруженных (неодобренных) пиров summary: Список обнаруженных (неодобренных) пиров
description: > description: >
Dynamic BGP-сессии (`evobgp_dyn_*`), которых ещё нет в `bgp_peer`. Dynamic BGP-сессии (`evobgp_dyn_*`), которых ещё нет в `bgp_peer`.
По умолчанию возвращает `status=pending`. При листинге выполняет live-опрос birdc/agent и upsert pending. По умолчанию возвращает `status=pending`. Читает сохранённые discoveries. Фоновый collector выполняет опрос и fenced upsert; GET не опрашивает ноды и не изменяет данные.
operationId: listDiscoveredPeers operationId: listDiscoveredPeers
parameters: parameters:
- $ref: "#/components/parameters/TenantId" - $ref: "#/components/parameters/TenantId"
@@ -3380,7 +3503,7 @@ paths:
type: string type: string
enum: ["1"] enum: ["1"]
description: > description: >
Live-опрос agent /v1/agent/health и BGP protocols на репликах; CP — local birdc. Сохранённые результаты фоновых опросов agent health/BGP и local birdc. GET не запускает опрос.
Обогащает каждый item полем `live`. Обогащает каждый item полем `live`.
responses: responses:
"200": "200":
@@ -3640,6 +3763,8 @@ paths:
description: Только чтение, без apply. description: Только чтение, без apply.
operationId: getRevisionPreview operationId: getRevisionPreview
responses: responses:
"429":
$ref: "#/components/responses/ExpensiveOperationLimited"
"200": "200":
description: Текст или структурированное представление - формат задаётся реализацией. description: Текст или структурированное представление - формат задаётся реализацией.
content: content:
@@ -3881,6 +4006,58 @@ paths:
default: default:
$ref: "#/components/responses/DefaultProblem" $ref: "#/components/responses/DefaultProblem"
/v1/jobs/{job_id}/report:
parameters:
- $ref: "#/components/parameters/TenantId"
- $ref: "#/components/parameters/JobId"
get:
tags: [Jobs]
summary: Отчёт задачи и сохранённый прогресс доставки
description: Требует bgp:operations:read. Прогресс хранится независимо от meta задачи. Для reconciliation без собственного прогресса возвращается прогресс исходной задачи в том же tenant; после начала восстановления приоритет имеет собственный прогресс. agent_reported не означает проверку состояния BIRD.
operationId: getJobReport
responses:
"200":
description: Успешно.
content:
application/json:
schema:
type: object
required: [job_id, kind, status, deployment_progress]
properties:
job_id: {type: string}
kind: {type: string}
status: {type: string}
meta: {type: [object, "null"], additionalProperties: true}
error: {type: [string, "null"]}
created_at: {type: string, format: date-time}
revision_id: {type: string}
log_entries: {type: array, items: {type: object, additionalProperties: true}}
log_total: {type: integer}
source_deployment_job_id: {type: string}
deployment_progress:
type: array
items:
type: object
required: [job_id, speaker_id, revision_id, generation, state, confirmation, dispatch_attempts, retry_round]
properties:
job_id: {type: string}
speaker_id: {type: string}
revision_id: {type: string}
generation: {type: integer, format: int64}
state: {type: string}
confirmation: {type: string}
error: {type: string}
dispatch_attempts: {type: integer}
retry_round: {type: integer}
health_received_at: {type: [string, "null"], format: date-time}
health_revision_id: {type: string}
health_source_at: {type: string}
health_error: {type: string}
"404":
$ref: "#/components/responses/NotFound"
default:
$ref: "#/components/responses/DefaultProblem"
/v1/jobs/{job_id}/cancel: /v1/jobs/{job_id}/cancel:
parameters: parameters:
- $ref: "#/components/parameters/TenantId" - $ref: "#/components/parameters/TenantId"
@@ -3904,6 +4081,33 @@ paths:
default: default:
$ref: "#/components/responses/DefaultProblem" $ref: "#/components/responses/DefaultProblem"
/v1/jobs/{job_id}/retry:
parameters:
- $ref: "#/components/parameters/TenantId"
- $ref: "#/components/parameters/JobId"
post:
tags: [Jobs]
summary: Повторить завершившуюся ошибкой задачу
description: Операторский повтор сохраняет ID и исходный запрос, сбрасывая бюджет попыток.
operationId: postJobRetry
responses:
"202":
description: Повтор поставлен в очередь.
content:
application/json:
schema:
$ref: "#/components/schemas/Job"
"409":
$ref: "#/components/responses/Conflict"
"429":
description: Очередь задач заполнена.
content:
application/problem+json:
schema:
$ref: "#/components/schemas/Problem"
default:
$ref: "#/components/responses/DefaultProblem"
/v1/speakers/{speaker_id}/revisions/latest: /v1/speakers/{speaker_id}/revisions/latest:
parameters: parameters:
- $ref: "#/components/parameters/SpeakerId" - $ref: "#/components/parameters/SpeakerId"
+312
View File
@@ -0,0 +1,312 @@
package agentwatch
import (
"context"
"crypto/sha256"
"encoding/binary"
"errors"
"fmt"
"hash"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
)
const (
maxConfigFiles = 512
maxConfigBytes = 64 << 20
maxFileBytes = 16 << 20
maxIncludeDepth = 16
maxDirectoryEntries = 4096
)
var errMainNotReady = errors.New("main bird.conf is not ready")
// Only standalone literal include statements are hashable. Other valid BIRD
// grammar falls back to the legacy configure-on-every-tick path.
var includeLine = regexp.MustCompile(`(?m)^[\t ]*include[\t ]+"([^"\r\n]+)"[\t ]*;[\t ]*(?:#[^\r\n]*)?\r?$`)
type configScanner struct {
ctx context.Context
root string
h hash.Hash
seen map[string]bool
files int
bytes int64
}
func hashConfigTree(ctx context.Context, activeDir string) ([sha256.Size]byte, error) {
var zero [sha256.Size]byte
root, err := filepath.Abs(activeDir)
if err != nil {
return zero, fmt.Errorf("agentwatch: active directory: %w", err)
}
root, err = filepath.EvalSymlinks(root)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return zero, fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
}
return zero, fmt.Errorf("agentwatch: resolve active directory: %w", err)
}
s := &configScanner{ctx: ctx, root: filepath.Clean(root), h: sha256.New(), seen: make(map[string]bool)}
if err := s.visit(filepath.Join(s.root, "bird.conf"), 0); err != nil {
return zero, err
}
var digest [sha256.Size]byte
copy(digest[:], s.h.Sum(nil))
return digest, nil
}
func (s *configScanner) visit(path string, depth int) error {
if err := s.ctx.Err(); err != nil {
return err
}
if depth > maxIncludeDepth {
return fmt.Errorf("agentwatch: include depth exceeds %d", maxIncludeDepth)
}
path = filepath.Clean(path)
rel, err := filepath.Rel(s.root, path)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return fmt.Errorf("agentwatch: include %q escapes active directory", path)
}
if s.seen[path] {
return nil
}
info, err := os.Lstat(path)
if err != nil {
if depth == 0 && errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
}
return fmt.Errorf("agentwatch: inspect %q: %w", path, err)
}
if !info.Mode().IsRegular() {
return fmt.Errorf("agentwatch: include %q is not a regular file", path)
}
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return fmt.Errorf("agentwatch: resolve %q: %w", path, err)
}
if _, err := s.withinRoot(resolved); err != nil {
return err
}
if info.Size() > maxFileBytes {
return fmt.Errorf("agentwatch: include %q exceeds %d bytes", path, maxFileBytes)
}
if s.files >= maxConfigFiles || s.bytes+info.Size() > maxConfigBytes {
return fmt.Errorf("agentwatch: configuration tree exceeds %d files or %d bytes", maxConfigFiles, maxConfigBytes)
}
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("agentwatch: open %q: %w", path, err)
}
defer func() { _ = f.Close() }()
limit := int64(maxFileBytes)
if remaining := int64(maxConfigBytes) - s.bytes; remaining < limit {
limit = remaining
}
content, err := readBounded(f, limit)
if err != nil {
return fmt.Errorf("agentwatch: read %q: %w", path, err)
}
s.seen[path] = true
s.files++
s.bytes += int64(len(content))
writeHash(s.h, filepath.ToSlash(rel), content)
patterns, err := includePatterns(string(content))
if err != nil {
return fmt.Errorf("agentwatch: %s: %w", rel, err)
}
for _, pattern := range patterns {
if strings.ContainsAny(pattern, `[\`) {
return fmt.Errorf("agentwatch: unsupported escaped include %q", pattern)
}
candidate := pattern
if !filepath.IsAbs(candidate) {
candidate = filepath.Join(filepath.Dir(path), filepath.FromSlash(candidate))
}
candidate = filepath.Clean(candidate)
// The pattern itself changes the digest even when its glob currently matches nothing.
writeHash(s.h, "include", []byte(filepath.ToSlash(candidate)))
if _, err := s.withinRoot(candidate); err != nil {
return err
}
if strings.ContainsAny(candidate, "*?") {
if strings.ContainsAny(filepath.Dir(candidate), "*?") {
return fmt.Errorf("agentwatch: unsupported include directory glob %q", pattern)
}
resolvedDir, err := filepath.EvalSymlinks(filepath.Dir(candidate))
if err != nil {
return fmt.Errorf("agentwatch: resolve include directory %q: %w", pattern, err)
}
if _, err := s.withinRoot(resolvedDir); err != nil {
return err
}
matches, err := boundedGlob(candidate)
if err != nil {
return fmt.Errorf("agentwatch: include glob %q: %w", pattern, err)
}
for _, match := range matches {
if err := s.visit(match, depth+1); err != nil {
return err
}
}
continue
}
if err := s.visit(candidate, depth+1); err != nil {
return err
}
}
return nil
}
func readBounded(r io.Reader, limit int64) ([]byte, error) {
content, err := io.ReadAll(io.LimitReader(r, limit+1))
if err != nil {
return nil, err
}
if int64(len(content)) > limit {
return nil, fmt.Errorf("configuration tree changed or exceeded size limit")
}
return content, nil
}
// boundedGlob only handles * and ? in a filename. Broader BIRD glob grammar
// falls back to every-tick configure instead of risking a false unchanged hash.
func boundedGlob(pattern string) ([]string, error) {
dir, base := filepath.Dir(pattern), filepath.Base(pattern)
f, err := os.Open(dir)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
names, err := f.Readdirnames(maxDirectoryEntries + 1)
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
if len(names) > maxDirectoryEntries {
return nil, fmt.Errorf("directory exceeds %d entries", maxDirectoryEntries)
}
matches := make([]string, 0)
for _, name := range names {
match, err := filepath.Match(base, name)
if err != nil {
return nil, err
}
if match {
matches = append(matches, filepath.Join(dir, name))
if len(matches) > maxConfigFiles {
return nil, fmt.Errorf("glob exceeds %d matches", maxConfigFiles)
}
}
}
sort.Strings(matches)
return matches, nil
}
func (s *configScanner) withinRoot(path string) (string, error) {
rel, err := filepath.Rel(s.root, path)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return "", fmt.Errorf("agentwatch: include %q escapes active directory", path)
}
return rel, nil
}
func writeHash(h hash.Hash, name string, content []byte) {
var size [8]byte
binary.LittleEndian.PutUint64(size[:], uint64(len(name)))
_, _ = h.Write(size[:])
_, _ = h.Write([]byte(name))
binary.LittleEndian.PutUint64(size[:], uint64(len(content)))
_, _ = h.Write(size[:])
_, _ = h.Write(content)
}
func includePatterns(content string) ([]string, error) {
tokens, unsupportedQuotedText := includeTokenOffsets(content)
if unsupportedQuotedText {
return nil, fmt.Errorf("unsupported quoted text or escape")
}
tokenSet := make(map[int]bool, len(tokens))
for _, token := range tokens {
tokenSet[token] = true
}
var patterns []string
for _, match := range includeLine.FindAllStringSubmatchIndex(content, -1) {
keyword := match[0] + strings.Index(content[match[0]:match[1]], "include")
if !tokenSet[keyword] {
continue
} // text inside a block comment
delete(tokenSet, keyword)
patterns = append(patterns, content[match[2]:match[3]])
}
if len(tokenSet) != 0 {
return nil, fmt.Errorf("unsupported include syntax")
}
return patterns, nil
}
// Recognize include keywords outside BIRD comments and quoted strings.
func includeTokenOffsets(config string) ([]int, bool) {
var offsets []int
quote := byte(0)
lineComment, blockComment := false, false
for i := 0; i < len(config); {
c := config[i]
if lineComment {
if c == '\n' {
lineComment = false
}
i++
continue
}
if blockComment {
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
blockComment = false
i += 2
} else {
i++
}
continue
}
if quote != 0 {
if c == '\\' {
return offsets, true
}
if c == quote {
quote = 0
}
i++
continue
}
if c == '#' || (c == '/' && i+1 < len(config) && config[i+1] == '/') {
lineComment = true
i++
continue
}
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
blockComment = true
i += 2
continue
}
if c == '"' || c == '\'' {
quote = c
i++
continue
}
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
start := i
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
i++
}
if config[start:i] == "include" {
offsets = append(offsets, start)
}
continue
}
i++
}
return offsets, quote != 0 || blockComment
}
+109
View File
@@ -0,0 +1,109 @@
// Package agentwatch reloads BIRD when its active configuration tree changes.
package agentwatch
import (
"context"
"crypto/sha256"
"errors"
"fmt"
"log"
"path/filepath"
"time"
)
// Control is the subset of BIRD operations required by the watch loop.
type Control interface {
ParseCheck(context.Context, string) error
Configure(context.Context) error
}
// Config sets bounded polling and reconciliation behavior.
type Config struct {
ActiveDir string
Every time.Duration
ReconcileEvery time.Duration
CommandTimeout time.Duration
Logf func(string, ...any)
}
type watcher struct {
ctl Control
cfg Config
hash [sha256.Size]byte
haveHash bool
lastConfigured time.Time
}
// Run starts with one check, then polls until ctx is cancelled. A successful
// birdc exit advances the remembered hash; it is not proof of BIRD ACK.
func Run(ctx context.Context, ctl Control, cfg Config) error {
if ctl == nil || cfg.Every <= 0 || cfg.ReconcileEvery <= 0 || cfg.CommandTimeout <= 0 {
return fmt.Errorf("agentwatch: control and positive intervals/timeouts are required")
}
if cfg.ActiveDir == "" {
cfg.ActiveDir = "/etc/bird"
}
if cfg.Logf == nil {
cfg.Logf = log.Printf
}
w := &watcher{ctl: ctl, cfg: cfg}
if err := w.step(ctx, time.Now()); err != nil && ctx.Err() == nil {
cfg.Logf("evobgp-agent watch: %v", err)
}
ticker := time.NewTicker(cfg.Every)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case now := <-ticker.C:
if err := w.step(ctx, now); err != nil && ctx.Err() == nil {
cfg.Logf("evobgp-agent watch: %v", err)
}
}
}
}
func (w *watcher) step(ctx context.Context, now time.Time) error {
if err := ctx.Err(); err != nil {
return err
}
digest, scanErr := hashConfigTree(ctx, w.cfg.ActiveDir)
if scanErr != nil {
if errors.Is(scanErr, errMainNotReady) {
return fmt.Errorf("configuration not ready: %w", scanErr)
}
// Unsupported/unhashable trees retain the old every-tick configure cadence.
w.cfg.Logf("evobgp-agent watch: cannot hash complete config tree (%v); using every-tick parse/configure", scanErr)
} else if w.haveHash && digest == w.hash && now.Sub(w.lastConfigured) < w.cfg.ReconcileEvery {
return nil
}
main := filepath.Join(w.cfg.ActiveDir, "bird.conf")
if err := w.command(ctx, func(cctx context.Context) error { return w.ctl.ParseCheck(cctx, main) }); err != nil {
return fmt.Errorf("parse check: %w", err)
}
if scanErr == nil {
stable, err := hashConfigTree(ctx, w.cfg.ActiveDir)
if err != nil {
return fmt.Errorf("configuration changed during parse check: %w", err)
}
if stable != digest {
return fmt.Errorf("configuration changed during parse check; retrying")
}
}
if err := w.command(ctx, w.ctl.Configure); err != nil {
return fmt.Errorf("configure: %w", err)
}
w.haveHash = scanErr == nil
if w.haveHash {
w.hash = digest
}
w.lastConfigured = now
return nil
}
func (w *watcher) command(parent context.Context, run func(context.Context) error) error {
ctx, cancel := context.WithTimeout(parent, w.cfg.CommandTimeout)
defer cancel()
return run(ctx)
}
+311
View File
@@ -0,0 +1,311 @@
package agentwatch
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
type fakeControl struct {
parsed int
configured int
failConfigure int
onParse func()
}
func (f *fakeControl) ParseCheck(ctx context.Context, _ string) error {
if err := ctx.Err(); err != nil {
return err
}
f.parsed++
if f.onParse != nil {
f.onParse()
}
return nil
}
func (f *fakeControl) Configure(ctx context.Context) error {
if err := ctx.Err(); err != nil {
return err
}
f.configured++
if f.failConfigure > 0 {
f.failConfigure--
return errors.New("birdc unavailable")
}
return nil
}
func testConfig(t *testing.T) (string, string) {
t.Helper()
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "bird.d"), 0o755); err != nil {
t.Fatal(err)
}
main := filepath.Join(root, "bird.conf")
if err := os.WriteFile(main, []byte("router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
t.Fatal(err)
}
fragment := filepath.Join(root, "bird.d", "peer.conf")
if err := os.WriteFile(fragment, []byte("# peer v1\n"), 0o644); err != nil {
t.Fatal(err)
}
return root, fragment
}
func testWatcher(root string, ctl Control) *watcher {
return &watcher{ctl: ctl, cfg: Config{ActiveDir: root, Every: 30 * time.Second, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}}}
}
func TestHashConfigTreeTracksMainIncludesAndGlobMembership(t *testing.T) {
root, fragment := testConfig(t)
first, err := hashConfigTree(context.Background(), root)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fragment, []byte("# peer v2\n"), 0o644); err != nil {
t.Fatal(err)
}
second, err := hashConfigTree(context.Background(), root)
if err != nil || second == first {
t.Fatalf("included edit not detected: digest=%x err=%v", second, err)
}
if err := os.WriteFile(filepath.Join(root, "bird.d", "manual.conf"), []byte("# manual\n"), 0o644); err != nil {
t.Fatal(err)
}
third, err := hashConfigTree(context.Background(), root)
if err != nil || third == second {
t.Fatalf("new glob member not detected: digest=%x err=%v", third, err)
}
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
t.Fatal(err)
}
fourth, err := hashConfigTree(context.Background(), root)
if err != nil || fourth == third {
t.Fatalf("main edit not detected: digest=%x err=%v", fourth, err)
}
}
func TestWatchSkipsUnchangedRetriesErrorsAndForcesReconciliation(t *testing.T) {
root, fragment := testConfig(t)
ctl := &fakeControl{}
w := testWatcher(root, ctl)
now := time.Now()
if err := w.step(context.Background(), now); err != nil {
t.Fatal(err)
}
if err := w.step(context.Background(), now.Add(30*time.Second)); err != nil {
t.Fatal(err)
}
if ctl.configured != 1 || ctl.parsed != 1 {
t.Fatalf("unchanged tree configured again: %+v", ctl)
}
if err := os.WriteFile(fragment, []byte("# changed\n"), 0o644); err != nil {
t.Fatal(err)
}
ctl.failConfigure = 1
if err := w.step(context.Background(), now.Add(time.Minute)); err == nil {
t.Fatal("expected configure failure")
}
if !w.haveHash || ctl.configured != 2 {
t.Fatalf("failure changed remembered state: %+v", w)
}
if err := w.step(context.Background(), now.Add(90*time.Second)); err != nil {
t.Fatal(err)
}
if ctl.configured != 3 {
t.Fatalf("failed change was not retried: %+v", ctl)
}
if err := w.step(context.Background(), now.Add(7*time.Minute)); err != nil {
t.Fatal(err)
}
if ctl.configured != 4 {
t.Fatalf("forced reconciliation missing: %+v", ctl)
}
// A fresh watcher must reconcile once even when another process already loaded the same bytes.
restarted := testWatcher(root, ctl)
if err := restarted.step(context.Background(), now.Add(8*time.Minute)); err != nil {
t.Fatal(err)
}
if ctl.configured != 5 {
t.Fatalf("restart did not reconcile: %+v", ctl)
}
}
func TestWatchRejectsChangingTreeBeforeConfigure(t *testing.T) {
root, fragment := testConfig(t)
ctl := &fakeControl{onParse: func() { _ = os.WriteFile(fragment, []byte("# raced\n"), 0o644) }}
w := testWatcher(root, ctl)
if err := w.step(context.Background(), time.Now()); err == nil || !strings.Contains(err.Error(), "changed during parse check") {
t.Fatalf("want race retry, got %v", err)
}
if ctl.configured != 0 {
t.Fatal("configured a tree that changed after parse check")
}
ctl.onParse = nil
if err := w.step(context.Background(), time.Now()); err != nil {
t.Fatal(err)
}
if ctl.configured != 1 {
t.Fatal("stable tree was not configured")
}
}
func TestUnsupportedIncludeUsesLegacyEveryTickCadence(t *testing.T) {
root, _ := testConfig(t)
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include variable;\n"), 0o644); err != nil {
t.Fatal(err)
}
ctl := &fakeControl{}
w := testWatcher(root, ctl)
now := time.Now()
for i := 0; i < 3; i++ {
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
t.Fatal(err)
}
}
if ctl.parsed != 3 || ctl.configured != 3 || w.haveHash {
t.Fatalf("unsupported tree was skipped: %+v", ctl)
}
}
func TestMissingIncludedFileFallsBackButMissingMainWaits(t *testing.T) {
root, fragment := testConfig(t)
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/peer.conf\";\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Remove(fragment); err != nil {
t.Fatal(err)
}
ctl := &fakeControl{}
w := testWatcher(root, ctl)
now := time.Now()
for i := 0; i < 2; i++ {
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
t.Fatal(err)
}
}
if ctl.configured != 2 {
t.Fatalf("missing include skipped legacy cadence: %+v", ctl)
}
if err := os.Remove(filepath.Join(root, "bird.conf")); err != nil {
t.Fatal(err)
}
if err := w.step(context.Background(), now.Add(time.Minute)); !errors.Is(err, errMainNotReady) {
t.Fatalf("missing main should wait, got %v", err)
}
if ctl.configured != 2 {
t.Fatal("configured without main bird.conf")
}
}
func TestHashConfigTreeBoundsAndEscapes(t *testing.T) {
root, fragment := testConfig(t)
if err := os.WriteFile(fragment, make([]byte, maxFileBytes+1), 0o644); err != nil {
t.Fatal(err)
}
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("want size limit, got %v", err)
}
if err := os.WriteFile(fragment, []byte("# small\n"), 0o644); err != nil {
t.Fatal(err)
}
outside := filepath.Join(t.TempDir(), "outside.conf")
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \""+filepath.ToSlash(outside)+"\";\n"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "escapes active directory") {
t.Fatalf("want outside include rejection, got %v", err)
}
}
func TestHashConfigTreeRejectsSymlinkInclude(t *testing.T) {
root, fragment := testConfig(t)
if err := os.Remove(fragment); err != nil {
t.Fatal(err)
}
outside := filepath.Join(t.TempDir(), "outside.conf")
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, fragment); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not a regular file") {
t.Fatalf("want symlink rejection, got %v", err)
}
}
func TestIncludeCommentsAndUnsupportedForms(t *testing.T) {
patterns, err := includePatterns("# include \"ignored.conf\";\n/*\ninclude \"also-ignored.conf\";\n*/\ninclude \"bird.d/*.conf\";\n")
if err != nil || len(patterns) != 1 || patterns[0] != "bird.d/*.conf" {
t.Fatalf("comment handling failed: patterns=%v err=%v", patterns, err)
}
if _, err := includePatterns("if true then include \"dynamic.conf\";\n"); err == nil {
t.Fatal("inline include should fall back to legacy cadence")
}
if _, err := includePatterns("include \"bird.d/[!a].conf\";\n"); err != nil {
// Bracket glob is rejected by the scanner after lexical parsing.
t.Fatalf("literal include statement should parse before glob validation: %v", err)
}
root, _ := testConfig(t)
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/[!a].conf\";\n"), 0o644); err != nil {
t.Fatal(err)
}
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "unsupported") {
t.Fatalf("bracket glob should fall back, got %v", err)
}
if _, err := includePatterns("define note = \"single\\\";\ninclude \"bird.d/real.conf\";\n"); err == nil {
t.Fatal("backslash in quoted text must force conservative fallback")
}
}
func TestBoundedGlobLimitsMatches(t *testing.T) {
root := t.TempDir()
for i := 0; i <= maxConfigFiles; i++ {
name := filepath.Join(root, fmt.Sprintf("peer-%03d.conf", i))
if err := os.WriteFile(name, nil, 0o644); err != nil {
t.Fatal(err)
}
}
if _, err := boundedGlob(filepath.Join(root, "*.conf")); err == nil || !strings.Contains(err.Error(), "exceeds") {
t.Fatalf("want bounded glob rejection, got %v", err)
}
}
func TestReadBoundedStopsAfterLimit(t *testing.T) {
r := strings.NewReader("abcdefghij")
if _, err := readBounded(r, 3); err == nil {
t.Fatal("growing file exceeded limit")
}
if got := r.Len(); got != 6 {
t.Fatalf("reader consumed beyond limit+1: %d bytes remain", got)
}
}
func TestRunStopsOnCancellation(t *testing.T) {
root, _ := testConfig(t)
ctx, cancel := context.WithCancel(context.Background())
ctl := &fakeControl{}
done := make(chan error, 1)
go func() {
done <- Run(ctx, ctl, Config{ActiveDir: root, Every: time.Hour, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}})
}()
cancel()
select {
case err := <-done:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("watch did not stop after cancellation")
}
}
+21 -8
View File
@@ -23,6 +23,8 @@ const DefaultASOverviewURL = "https://stat.ripe.net/data/as-overview/data.json"
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat). // AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) { func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
ctx, cancel := ripeRequestContext(ctx)
defer cancel()
if hc == nil { if hc == nil {
hc = httpclient.New(httpclient.DefaultTimeout) hc = httpclient.New(httpclient.DefaultTimeout)
} }
@@ -43,13 +45,14 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err) return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200)) return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
} }
body, err := httpclient.ReadResponseBody(resp, 32<<20)
if err != nil {
return nil, fmt.Errorf("ripestat AS%d response: %w", asn, err)
}
var wrap struct { var wrap struct {
Status string `json:"status"` Status string `json:"status"`
@@ -85,6 +88,8 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort). // ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) { func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
ctx, cancel := ripeRequestContext(ctx)
defer cancel()
if hc == nil { if hc == nil {
hc = httpclient.New(httpclient.DefaultTimeout) hc = httpclient.New(httpclient.DefaultTimeout)
} }
@@ -105,13 +110,14 @@ func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, erro
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err) return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200)) return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
} }
body, err := httpclient.ReadResponseBody(resp, 4<<20)
if err != nil {
return "", fmt.Errorf("ripestat as-overview AS%d response: %w", asn, err)
}
var wrap struct { var wrap struct {
Status string `json:"status"` Status string `json:"status"`
@@ -135,3 +141,10 @@ func truncateForErr(b []byte, n int) string {
} }
return s return s
} }
func ripeRequestContext(ctx context.Context) (context.Context, context.CancelFunc) {
if _, ok := ctx.Deadline(); ok {
return context.WithCancel(ctx)
}
return context.WithTimeout(ctx, httpclient.DefaultTimeout)
}
+103
View File
@@ -0,0 +1,103 @@
package asnresolve
import (
"compress/gzip"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"evobgp/internal/httpclient"
)
type ripeRoundTrip func(*http.Request) (*http.Response, error)
func (f ripeRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
type ripeCountingBody struct {
io.Reader
read int
closed bool
}
func (b *ripeCountingBody) Read(p []byte) (int, error) {
n, e := b.Reader.Read(p)
b.read += n
return n, e
}
func (b *ripeCountingBody) Close() error { b.closed = true; return nil }
func TestRIPECompleteResponseLimits(t *testing.T) {
cases := []struct {
name string
limit int
prefix string
call func(context.Context, *http.Client) error
}{
{"announced", 32 << 20, `{"status":"ok","data":{"prefixes":[]}}`, func(ctx context.Context, hc *http.Client) error { _, e := AnnouncedPrefixes(ctx, hc, 64496); return e }},
{"holder", 4 << 20, `{"status":"ok","data":{"holder":"Example"}}`, func(ctx context.Context, hc *http.Client) error { _, e := ASHolderName(ctx, hc, 64496); return e }},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
for _, known := range []bool{true, false} {
for _, extra := range []int{0, 1} {
body := &ripeCountingBody{Reader: strings.NewReader(tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra))}
hc := &http.Client{Transport: ripeRoundTrip(func(r *http.Request) (*http.Response, error) {
if _, ok := r.Context().Deadline(); !ok {
t.Error("missing request deadline")
}
length := int64(-1)
if known {
length = int64(tc.limit + extra)
}
return &http.Response{StatusCode: 200, Body: body, ContentLength: length, Header: make(http.Header)}, nil
})}
err := tc.call(context.Background(), hc)
if extra == 0 && err != nil {
t.Fatalf("exact limit rejected known=%v: %v", known, err)
}
if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) {
t.Fatalf("oversized valid prefix accepted known=%v: %v", known, err)
}
if body.read > tc.limit+1 || !body.closed {
t.Fatalf("read=%d closed=%v", body.read, body.closed)
}
}
}
})
}
}
func TestRIPEErrorBodySmallAndContextPreserved(t *testing.T) {
body := &ripeCountingBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))}
hc := &http.Client{Transport: ripeRoundTrip(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil
})}
if _, err := AnnouncedPrefixes(context.Background(), hc, 1); err == nil || body.read > 4<<10 || !body.closed {
t.Fatalf("non200 unbounded read=%d close=%v err=%v", body.read, body.closed, err)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
defer cancel()
hc.Transport = ripeRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() })
if _, err := ASHolderName(ctx, hc, 1); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("caller deadline lost: %v", err)
}
}
func TestRIPEGzipExpandedLimit(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Encoding", "gzip")
gz := gzip.NewWriter(w)
_, _ = io.WriteString(gz, `{"status":"ok","data":{"holder":"Example"}}`+strings.Repeat(" ", 4<<20))
_ = gz.Close()
}))
defer server.Close()
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", server.URL)
if _, err := ASHolderName(context.Background(), server.Client(), 1); !errors.Is(err, httpclient.ErrResponseTooLarge) {
t.Fatalf("gzip bypassed decoded limit: %v", err)
}
}
+370 -89
View File
@@ -1,152 +1,433 @@
// Package birddeploy applies revision preview fragments to a shared BIRD volume with parse-check and LKG (plan §13).
package birddeploy package birddeploy
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io/fs" "io/fs"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"sort"
"strings" "strings"
"time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/store" "evobgp/internal/store"
) )
// Config drives two-phase deploy from a revision in the store. const recoveryTimeout = 30 * time.Second
var includeLine = regexp.MustCompile(`(?m)^([ \t]*include[ \t]+")([^"]+)("[ \t]*;[ \t]*(?:#[^\n]*)?$)`)
// Config identifies the BIRD instance. StagingDir remains for caller compatibility;
// generations live inside ActiveDir so referenced files survive process restart.
type Config struct { type Config struct {
ActiveDir string // e.g. /etc/bird (live config tree) ActiveDir string
StagingDir string // e.g. /tmp/evobgp-staging (same host/volume as ActiveDir) StagingDir string
BirdBin string BirdBin string
BirdcBin string BirdcBin string
Socket string // birdc -s Socket string
writeFile func(string, []byte) error // test-only fault injection; nil uses synced writes
} }
// ApplyRevision writes preview fragments to staging, runs bird -p, swaps into active, runs birdc configure; restores LKG on failure. // ApplyRevision installs a checked generation through one atomic bird.conf rename.
func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error { func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error {
if rev == nil || len(rev.PreviewFragments) == 0 { if rev == nil || len(rev.PreviewFragments) == 0 {
return fmt.Errorf("birddeploy: no preview fragments") return fmt.Errorf("birddeploy: no preview fragments")
} }
active := strings.TrimSpace(cfg.ActiveDir) if strings.TrimSpace(cfg.ActiveDir) == "" {
staging := strings.TrimSpace(cfg.StagingDir) return fmt.Errorf("birddeploy: ActiveDir required")
if active == "" || staging == "" {
return fmt.Errorf("birddeploy: ActiveDir and StagingDir required")
} }
if err := os.MkdirAll(staging, 0o755); err != nil { active, err := filepath.Abs(cfg.ActiveDir)
return err if err != nil {
return fmt.Errorf("birddeploy: resolve active directory: %w", err)
} }
if err := os.MkdirAll(filepath.Join(active, "bird.d"), 0o755); err != nil { if err := os.MkdirAll(active, 0o755); err != nil {
return err return fmt.Errorf("birddeploy: create active directory: %w", err)
} }
lkg := filepath.Join(active, ".evobgp_lkg") lock, err := lockTarget(ctx, filepath.Join(active, ".evobgp-apply.lock"))
if err != nil {
// Phase 1: write staging tree return fmt.Errorf("birddeploy: acquire apply lock: %w", err)
for rel, content := range rev.PreviewFragments {
rel = strings.TrimPrefix(rel, "/")
if !isDeployableBirdFragment(rel) {
continue
} }
dst := filepath.Join(staging, rel) defer func() {
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { if err := lock.Close(); err != nil {
return err fmt.Fprintf(os.Stderr, "birddeploy: release apply lock: %v\n", err)
} }
if err := os.WriteFile(dst, []byte(content), 0o644); err != nil { }()
return err if err := ctx.Err(); err != nil {
} return fmt.Errorf("birddeploy: apply cancelled: %w", err)
}
mainConf := filepath.Join(staging, "bird.conf")
if _, err := os.Stat(mainConf); err != nil {
return fmt.Errorf("birddeploy: staging missing bird.conf: %w", err)
} }
bc := ctl bc := ctl
if bc == nil { if bc == nil {
bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket} bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
} }
if err := bc.ParseCheck(ctx, mainConf); err != nil { writeFile := cfg.writeFile
return fmt.Errorf("birddeploy: bird -p failed: %w", err) if writeFile == nil {
writeFile = writeSynced
} }
mainPath := filepath.Join(active, "bird.conf")
// Snapshot LKG (best-effort copy of current active bird.conf + bird.d) mainInfo, err := os.Lstat(mainPath)
_ = os.RemoveAll(lkg) if err != nil {
_ = snapshotBirdTree(active, lkg) return fmt.Errorf("birddeploy: inspect current bird.conf: %w", err)
// Phase 2: atomic swap into active
for rel, content := range rev.PreviewFragments {
rel = strings.TrimPrefix(rel, "/")
if !isDeployableBirdFragment(rel) {
continue
} }
dst := filepath.Join(active, rel) if !mainInfo.Mode().IsRegular() {
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { return fmt.Errorf("birddeploy: current bird.conf must be a regular file; refusing to replace %s", mainInfo.Mode())
_ = restoreLKG(lkg, active) }
oldMain, err := os.ReadFile(mainPath)
if err != nil {
return fmt.Errorf("birddeploy: read current bird.conf: %w", err)
}
if err := bc.ParseCheck(ctx, mainPath); err != nil {
return fmt.Errorf("birddeploy: current configuration is not a usable LKG: %w", err)
}
gen, err := os.MkdirTemp(active, ".evobgp-generation-")
if err != nil {
return fmt.Errorf("birddeploy: create generation: %w", err)
}
activated := false
defer func() {
if !activated {
if err := os.RemoveAll(gen); err != nil {
fmt.Fprintf(os.Stderr, "birddeploy: remove unactivated generation %s: %v\n", gen, err)
}
}
}()
if err := writeGenerationWith(active, gen, rev.PreviewFragments, writeFile); err != nil {
return err return err
} }
tmp := dst + ".tmp" newMain, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil { if err != nil {
_ = restoreLKG(lkg, active) return fmt.Errorf("birddeploy: read generated bird.conf: %w", err)
return err
} }
if err := os.Rename(tmp, dst); err != nil { if err := bc.ParseCheck(ctx, filepath.Join(gen, "bird.conf")); err != nil {
_ = restoreLKG(lkg, active) return fmt.Errorf("birddeploy: staged bird -p failed: %w", err)
return err
} }
if err := writeFile(filepath.Join(gen, "previous-bird.conf"), oldMain); err != nil {
return fmt.Errorf("birddeploy: save LKG: %w", err)
}
if err := syncDir(gen); err != nil {
return fmt.Errorf("birddeploy: sync LKG generation: %w", err)
}
if err := ctx.Err(); err != nil {
return fmt.Errorf("birddeploy: apply cancelled: %w", err)
}
swapped, err := replaceMain(active, newMain)
activated = swapped
if err != nil {
if swapped {
return recoverPrevious(bc, active, gen, oldMain, fmt.Errorf("activate generation: %w", err))
}
return fmt.Errorf("birddeploy: activate generation: %w", err)
} }
if err := bc.Configure(ctx); err != nil { if err := bc.Configure(ctx); err != nil {
_ = restoreLKG(lkg, active) return recoverPrevious(bc, active, gen, oldMain, err)
_ = bc.Configure(ctx) }
return fmt.Errorf("birddeploy: birdc configure failed, restored LKG: %w", err) if err := pruneGenerations(active, gen, oldMain); err != nil {
fmt.Fprintf(os.Stderr, "birddeploy: configuration applied; generation cleanup failed: %v\n", err)
} }
return nil return nil
} }
// isDeployableBirdFragment skips UI-only preview keys (e.g. flattened full config) not referenced from bird.conf. func recoverPrevious(bc *birdfmt.BirdCtl, active, gen string, oldMain []byte, cause error) error {
func isDeployableBirdFragment(rel string) bool { ctx, cancel := context.WithTimeout(context.Background(), recoveryTimeout)
return rel != "" && !strings.HasPrefix(rel, "_") defer cancel()
if _, err := replaceMain(active, oldMain); err != nil {
return fmt.Errorf("birddeploy: configure failed: %w; restore failed: %v", cause, err)
}
if err := bc.Configure(ctx); err != nil {
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored on disk but recovery configure failed: %v", cause, err)
}
if err := os.RemoveAll(gen); err != nil {
fmt.Fprintf(os.Stderr, "birddeploy: LKG restored; remove failed generation %s: %v\n", gen, err)
}
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored and recovery configure succeeded", cause)
} }
func snapshotBirdTree(activeRoot, dstRoot string) error { func writeGeneration(active, gen string, fragments map[string]string) error {
if err := os.MkdirAll(dstRoot, 0o755); err != nil { return writeGenerationWith(active, gen, fragments, writeSynced)
}
func writeGenerationWith(active, gen string, fragments map[string]string, writeFile func(string, []byte) error) error {
main, ok := fragments["bird.conf"]
if !ok {
return fmt.Errorf("birddeploy: preview missing bird.conf")
}
oldBirdD := filepath.Join(active, birdfmt.DirBirdD)
newBirdD := filepath.Join(gen, birdfmt.DirBirdD)
if err := os.MkdirAll(newBirdD, 0o755); err != nil {
return fmt.Errorf("birddeploy: create generation bird.d: %w", err)
}
if err := filepath.WalkDir(oldBirdD, func(path string, d fs.DirEntry, walkErr error) error {
if errors.Is(walkErr, fs.ErrNotExist) && path == oldBirdD {
return nil
}
if walkErr != nil {
return walkErr
}
rel, err := filepath.Rel(oldBirdD, path)
if err != nil {
return err return err
} }
mc := filepath.Join(activeRoot, "bird.conf") if d.Type()&os.ModeSymlink != 0 {
if b, err := os.ReadFile(mc); err == nil { return fmt.Errorf("operator snippet %q is a symlink", rel)
_ = os.WriteFile(filepath.Join(dstRoot, "bird.conf"), b, 0o644)
} }
bd := filepath.Join(activeRoot, "bird.d") if rel == "." {
if st, err := os.Stat(bd); err == nil && st.IsDir() { if !d.IsDir() {
_ = filepath.WalkDir(bd, func(path string, d fs.DirEntry, err error) error { return fmt.Errorf("birddeploy: bird.d is not a directory")
if err != nil || d.IsDir() { }
return err return nil
}
if d.IsDir() {
return os.MkdirAll(filepath.Join(newBirdD, rel), 0o755)
}
if !d.Type().IsRegular() {
return fmt.Errorf("operator snippet %q is not a regular file", rel)
}
_, replaced := fragments[filepath.ToSlash(filepath.Join(birdfmt.DirBirdD, rel))]
if strings.HasPrefix(filepath.Base(rel), "evobgp_") || replaced {
return nil
} }
suffix, _ := filepath.Rel(activeRoot, path)
out := filepath.Join(dstRoot, suffix)
_ = os.MkdirAll(filepath.Dir(out), 0o755)
b, err := os.ReadFile(path) b, err := os.ReadFile(path)
if err != nil { if err != nil {
return nil return err
} }
return os.WriteFile(out, b, 0o644) if countIncludeTokens(string(b)) > 0 {
return fmt.Errorf("operator snippet %q has a nested include; cannot pin the complete tree", rel)
}
return writeFile(filepath.Join(newBirdD, rel), b)
}); err != nil {
return fmt.Errorf("birddeploy: copy operator snippets: %w", err)
}
for rel, content := range fragments {
if rel == "bird.conf" || strings.HasPrefix(rel, "_") {
continue
}
if countIncludeTokens(content) > 0 {
return fmt.Errorf("birddeploy: nested include in preview fragment %q is unsupported", rel)
}
clean := filepath.Clean(filepath.FromSlash(rel))
if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) || filepath.Ext(clean) != ".conf" {
return fmt.Errorf("birddeploy: unsafe preview path %q", rel)
}
dst := filepath.Join(gen, clean)
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return fmt.Errorf("birddeploy: create fragment directory: %w", err)
}
if err := writeFile(dst, []byte(content)); err != nil {
return fmt.Errorf("birddeploy: write %s: %w", rel, err)
}
}
var rewriteErr error
if !standaloneIncludesOnly(main) {
return fmt.Errorf("birddeploy: unsupported include syntax in bird.conf")
}
rewritten := includeLine.ReplaceAllStringFunc(main, func(line string) string {
m := includeLine.FindStringSubmatch(line)
inc := m[2]
if filepath.IsAbs(inc) {
rel, err := filepath.Rel(active, inc)
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
rewriteErr = fmt.Errorf("birddeploy: external absolute include %q is unsupported", inc)
return line
}
inc = rel
}
clean := filepath.Clean(filepath.FromSlash(inc))
if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) {
rewriteErr = fmt.Errorf("birddeploy: unsupported relative include %q", inc)
return line
}
return m[1] + filepath.ToSlash(filepath.Join(gen, clean)) + m[3]
}) })
if rewriteErr != nil {
return rewriteErr
}
if err := writeFile(filepath.Join(gen, ".evobgp-owned"), []byte("evobgp-generation-v1\n")); err != nil {
return fmt.Errorf("birddeploy: mark generation: %w", err)
}
if err := writeFile(filepath.Join(gen, "bird.conf"), []byte(rewritten)); err != nil {
return fmt.Errorf("birddeploy: write generated bird.conf: %w", err)
}
return syncTree(gen)
}
func standaloneIncludesOnly(config string) bool {
tokens := includeTokenOffsets(config)
matches := includeLine.FindAllStringIndex(config, -1)
if len(tokens) != len(matches) {
return false
}
for i, match := range matches {
word := strings.Index(config[match[0]:match[1]], "include")
if word < 0 || tokens[i] != match[0]+word {
return false
}
}
return true
}
func countIncludeTokens(config string) int { return len(includeTokenOffsets(config)) }
// includeTokenOffsets skips BIRD comments and strings, retaining exact offsets
// so a commented standalone include cannot mask a live inline include.
func includeTokenOffsets(config string) []int {
var offsets []int
quote := byte(0)
lineComment, blockComment := false, false
for i := 0; i < len(config); {
c := config[i]
if lineComment {
if c == '\n' {
lineComment = false
}
i++
continue
}
if blockComment {
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
blockComment = false
i += 2
} else {
i++
}
continue
}
if quote != 0 {
// BIRD quoted text does not use C-style backslash escapes.
// Skipping the following quote could hide a live include.
if c == quote {
quote = 0
}
i++
continue
}
if c == '#' {
lineComment = true
i++
continue
}
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
blockComment = true
i += 2
continue
}
if c == '"' || c == '\'' {
quote = c
i++
continue
}
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
start := i
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
i++
}
if config[start:i] == "include" {
offsets = append(offsets, start)
}
continue
}
i++
}
return offsets
}
func syncTree(root string) error {
var dirs []string
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
dirs = append(dirs, path)
}
return nil
}); err != nil {
return err
}
sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) })
for _, dir := range dirs {
if err := syncDir(dir); err != nil {
return fmt.Errorf("sync %s: %w", dir, err)
}
} }
return nil return nil
} }
func restoreLKG(lkg, active string) error { func pruneGenerations(active, current string, previous []byte) error {
if _, err := os.Stat(filepath.Join(lkg, "bird.conf")); err != nil { entries, err := os.ReadDir(active)
return err
}
_ = filepath.WalkDir(lkg, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
suffix, _ := filepath.Rel(lkg, path)
out := filepath.Join(active, suffix)
_ = os.MkdirAll(filepath.Dir(out), 0o755)
b, err := os.ReadFile(path)
if err != nil { if err != nil {
return err
}
var stale []string
for _, entry := range entries {
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), ".evobgp-generation-") {
continue
}
path := filepath.Join(active, entry.Name())
if path == current || strings.Contains(string(previous), filepath.ToSlash(path)+"/") {
continue
}
marker, err := os.ReadFile(filepath.Join(path, ".evobgp-owned"))
if errors.Is(err, fs.ErrNotExist) {
continue
}
if err != nil {
return err
}
if string(marker) != "evobgp-generation-v1\n" {
continue
}
stale = append(stale, path)
}
sort.Strings(stale)
for _, path := range stale {
if err := os.RemoveAll(path); err != nil {
return fmt.Errorf("remove %s: %w", path, err)
}
}
return nil return nil
} }
return os.WriteFile(out, b, 0o644)
}) func replaceMain(active string, content []byte) (bool, error) {
return nil tmp, err := os.CreateTemp(active, ".evobgp-bird-conf-")
if err != nil {
return false, err
}
defer func() { _ = os.Remove(tmp.Name()) }()
if err := tmp.Chmod(0o644); err != nil {
_ = tmp.Close()
return false, err
}
if _, err := tmp.Write(content); err != nil {
_ = tmp.Close()
return false, err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return false, err
}
if err := tmp.Close(); err != nil {
return false, err
}
if err := os.Rename(tmp.Name(), filepath.Join(active, "bird.conf")); err != nil {
return false, err
}
return true, syncDir(active)
}
func writeSynced(path string, b []byte) error {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return err
}
if _, err := f.Write(b); err != nil {
_ = f.Close()
return err
}
if err := f.Sync(); err != nil {
_ = f.Close()
return err
}
return f.Close()
} }
+279
View File
@@ -0,0 +1,279 @@
//go:build linux
package birddeploy
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"testing"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/store"
)
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
t.Helper()
bin := t.TempDir()
parse := filepath.Join(bin, "bird")
configure := filepath.Join(bin, "birdc")
parseScript := `#!/bin/sh
if grep -q FAIL_PARSE "$2"; then exit 1; fi
exit 0
`
configureScript := fmt.Sprintf(`#!/bin/sh
root=%q
if [ -e "$root/.configure-wait" ]; then
rm "$root/.configure-wait"
touch "$root/.configure-started"
exec sleep 5
fi
if [ -e "$root/.configure-fail-once" ]; then
rm "$root/.configure-fail-once"
exit 1
fi
mkdir "$root/.configuring" || exit 1
sleep 0.05
rmdir "$root/.configuring"
`, active)
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
}
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
}
func revision(label string) *store.Revision {
return &store.Revision{PreviewFragments: map[string]string{
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
"bird.d/evobgp_new.conf": "# " + label + "\n",
}}
}
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
t.Helper()
active := t.TempDir()
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
t.Fatal(err)
}
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
return active, fakeCtl(t, active), old
}
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
active, ctl, _ := setupActive(t)
for _, label := range []string{"one", "two", "three"} {
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
t.Fatal(err)
}
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(main), "# "+label) {
t.Fatalf("wrong active generation: %s", main)
}
if strings.Contains(string(main), `include "bird.d/`) {
t.Fatalf("include not pinned: %s", main)
}
}
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
if err != nil {
t.Fatal(err)
}
if len(gens) != 2 {
t.Fatalf("want active and previous generations, got %d", len(gens))
}
for _, gen := range gens {
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("removed fragment retained in %s: %v", gen, err)
}
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
t.Fatalf("operator fragment missing: %v", err)
}
}
}
func TestApplyRevisionFaults(t *testing.T) {
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
t.Run(mode, func(t *testing.T) {
active, ctl, old := setupActive(t)
rev := revision(mode)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cfg := Config{ActiveDir: active}
switch mode {
case "parse":
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
case "write":
rev.PreviewFragments["../escape.conf"] = "bad"
case "disk-full":
cfg.writeFile = func(path string, b []byte) error {
if filepath.Base(path) == "previous-bird.conf" {
return syscall.ENOSPC
}
return writeSynced(path, b)
}
case "configure":
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
t.Fatal(err)
}
case "cancel":
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
t.Fatal(err)
}
}
result := make(chan error, 1)
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
if mode == "cancel" {
deadline := time.After(3 * time.Second)
for {
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
break
}
select {
case <-deadline:
t.Fatal("configure did not start")
case <-time.After(10 * time.Millisecond):
}
}
cancel()
}
err := <-result
if err == nil {
t.Fatal("expected apply failure")
}
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
if readErr != nil {
t.Fatal(readErr)
}
if string(main) != string(old) {
t.Fatalf("old config not restored: %s; error: %v", main, err)
}
if mode == "configure" || mode == "cancel" {
if !strings.Contains(err.Error(), "recovery configure succeeded") {
t.Fatalf("unconfirmed recovery: %v", err)
}
}
})
}
}
func TestApplyRevisionSerializes(t *testing.T) {
active, ctl, _ := setupActive(t)
var wg sync.WaitGroup
errs := make(chan error, 2)
for _, label := range []string{"first", "second"} {
wg.Add(1)
go func(label string) {
defer wg.Done()
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
}(label)
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
}
func TestLockHelperProcess(t *testing.T) {
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
if active == "" {
return
}
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
if err != nil {
t.Fatal(err)
}
defer func() { _ = lock.Close() }()
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
t.Fatal(err)
}
for {
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
return
}
time.Sleep(10 * time.Millisecond)
}
}
func TestApplyRevisionCrossProcessLock(t *testing.T) {
active, ctl, old := setupActive(t)
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
if err := cmd.Wait(); err != nil {
t.Errorf("lock helper: %v", err)
}
}()
deadline := time.After(3 * time.Second)
for {
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
break
}
select {
case <-deadline:
t.Fatal("lock helper did not become ready")
case <-time.After(10 * time.Millisecond):
}
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("want lock timeout, got %v", err)
}
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if string(main) != string(old) {
t.Fatalf("apply bypassed cross-process lock: %s", main)
}
}
func TestRealBirdParseGeneration(t *testing.T) {
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
if bin == "" {
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
}
active := t.TempDir()
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
t.Fatal(err)
}
ctl := fakeCtl(t, active)
ctl.Bird = bin
rev := &store.Revision{PreviewFragments: map[string]string{
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
}}
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
t.Fatal(err)
}
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
t.Fatalf("activated config fails real bird -p: %v", err)
}
}
+204
View File
@@ -0,0 +1,204 @@
package birddeploy
import (
"context"
"os"
"path/filepath"
"strings"
"testing"
"evobgp/internal/store"
)
func TestWriteGenerationPinsCompleteIncludeTree(t *testing.T) {
active := t.TempDir()
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte("# operator"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(active, "bird.d", "evobgp_removed.conf"), []byte("# removed"), 0o644); err != nil {
t.Fatal(err)
}
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
fragments := map[string]string{"bird.conf": "include \"bird.d/*.conf\";\n", "bird.d/evobgp_new.conf": "# new"}
if err := writeGeneration(active, gen, fragments); err != nil {
t.Fatal(err)
}
main, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(main), filepath.ToSlash(filepath.Join(gen, "bird.d", "*.conf"))) {
t.Fatalf("include is not pinned to generation: %s", main)
}
for _, name := range []string{"operator.conf", "evobgp_new.conf"} {
if _, err := os.Stat(filepath.Join(gen, "bird.d", name)); err != nil {
t.Fatalf("%s missing: %v", name, err)
}
}
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !os.IsNotExist(err) {
t.Fatalf("removed fragment retained: %v", err)
}
}
func TestWriteGenerationRejectsUnsafePaths(t *testing.T) {
for _, bad := range []string{"../escape.conf", "bird.d/../../escape.conf", "other.conf"} {
t.Run(bad, func(t *testing.T) {
active := t.TempDir()
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", bad: "bad"}
if err := writeGeneration(active, gen, fragments); err == nil {
t.Fatal("unsafe path accepted")
}
})
}
}
func TestWriteGenerationRejectsExternalAndNestedIncludes(t *testing.T) {
for _, main := range []string{
`include "/outside/bird.d/peer.conf";` + "\n",
`include "other/peer.conf";` + "\n",
`include "bird.d/peer.conf"; include "other.conf";` + "\n",
`router id 192.0.2.1; include "/outside/peer.conf";` + "\n",
"/*\ninclude \"bird.d/decoy.conf\";\n*/\nrouter id 192.0.2.1; include \"/outside/live.conf\";\n",
} {
active := t.TempDir()
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main}); err == nil {
t.Fatalf("include accepted: %s", main)
}
}
active := t.TempDir()
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte(`define X=1; include "/outside/foo.conf";`), 0o644); err != nil {
t.Fatal(err)
}
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
t.Fatal("nested operator include accepted")
}
}
func TestCountIncludeTokensIgnoresCommentsAndStrings(t *testing.T) {
content := "# include \"ignored.conf\";\n" +
"/* include \"ignored.conf\"; */\n" +
"define note = \"include in string\";\n" +
"include \"bird.d/real.conf\";\n"
if got := countIncludeTokens(content); got != 1 {
t.Fatalf("want one include, got %d", got)
}
}
func TestIncludeAfterBackslashStringCannotHideNestedInclude(t *testing.T) {
content := `define note = "\";` + "\n" + `include "/outside/hidden.conf";` + "\n"
if got := countIncludeTokens(content); got != 1 {
t.Fatalf("live include hidden after quoted backslash: %d", got)
}
active := t.TempDir()
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", "bird.d/evobgp_test.conf": content}
if err := writeGeneration(active, gen, fragments); err == nil {
t.Fatal("nested external include accepted after quoted backslash")
}
}
func TestWriteGenerationRewritesAbsoluteActiveInclude(t *testing.T) {
active := t.TempDir()
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
inc := filepath.ToSlash(filepath.Join(active, "bird.d", "peer.conf"))
main := "include \"" + inc + "\";\n"
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main, "bird.d/peer.conf": "# peer"}); err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(b), filepath.ToSlash(filepath.Join(gen, "bird.d", "peer.conf"))) {
t.Fatalf("absolute include not rewritten: %s", b)
}
}
func TestWriteGenerationRejectsSymlinkBirdDRoot(t *testing.T) {
active := t.TempDir()
outside := t.TempDir()
if err := os.WriteFile(filepath.Join(outside, "operator.conf"), []byte("# operator"), 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(outside, filepath.Join(active, "bird.d")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
old := []byte("router id 192.0.2.2;\n")
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
t.Fatal(err)
}
gen := filepath.Join(active, ".evobgp-generation-test")
if err := os.Mkdir(gen, 0o755); err != nil {
t.Fatal(err)
}
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
t.Fatal("bird.d symlink accepted")
}
current, err := os.ReadFile(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if string(current) != string(old) {
t.Fatalf("active config changed: %s", current)
}
if _, err := os.ReadFile(filepath.Join(outside, "operator.conf")); err != nil {
t.Fatalf("operator snippet changed: %v", err)
}
}
func TestApplyRevisionRejectsSymlinkMain(t *testing.T) {
active := t.TempDir()
target := filepath.Join(t.TempDir(), "operator-bird.conf")
old := []byte("router id 192.0.2.2;\n")
if err := os.WriteFile(target, old, 0o644); err != nil {
t.Fatal(err)
}
if err := os.Symlink(target, filepath.Join(active, "bird.conf")); err != nil {
t.Skipf("symlinks unavailable: %v", err)
}
err := ApplyRevision(context.Background(), nil, &store.Revision{PreviewFragments: map[string]string{"bird.conf": "router id 192.0.2.1;\n"}}, Config{ActiveDir: active})
if err == nil || !strings.Contains(err.Error(), "regular file") {
t.Fatalf("want explicit symlink rejection, got %v", err)
}
info, err := os.Lstat(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if info.Mode()&os.ModeSymlink == 0 {
t.Fatal("active symlink replaced")
}
current, err := os.ReadFile(target)
if err != nil {
t.Fatal(err)
}
if string(current) != string(old) {
t.Fatalf("operator target changed: %s", current)
}
}
+35
View File
@@ -0,0 +1,35 @@
//go:build linux
package birddeploy
import (
"context"
"errors"
"os"
"time"
"golang.org/x/sys/unix"
)
func lockTarget(ctx context.Context, path string) (*os.File, error) {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, err
}
for {
err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
if err == nil {
return f, nil
}
if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) {
_ = f.Close()
return nil, err
}
select {
case <-ctx.Done():
_ = f.Close()
return nil, ctx.Err()
case <-time.After(25 * time.Millisecond):
}
}
}
+13
View File
@@ -0,0 +1,13 @@
//go:build !linux && !windows
package birddeploy
import (
"context"
"errors"
"os"
)
func lockTarget(context.Context, string) (*os.File, error) {
return nil, errors.New("birddeploy: cross-process apply lock is unsupported on this platform")
}
+36
View File
@@ -0,0 +1,36 @@
//go:build windows
package birddeploy
import (
"context"
"errors"
"os"
"time"
"golang.org/x/sys/windows"
)
func lockTarget(ctx context.Context, path string) (*os.File, error) {
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
if err != nil {
return nil, err
}
var overlapped windows.Overlapped
for {
err = windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped)
if err == nil {
return f, nil
}
if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
_ = f.Close()
return nil, err
}
select {
case <-ctx.Done():
_ = f.Close()
return nil, ctx.Err()
case <-time.After(25 * time.Millisecond):
}
}
}
+14
View File
@@ -0,0 +1,14 @@
//go:build !windows
package birddeploy
import "os"
func syncDir(path string) error {
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return f.Sync()
}
+6
View File
@@ -0,0 +1,6 @@
//go:build windows
package birddeploy
// Windows does not support syncing a directory handle through os.File.Sync.
func syncDir(string) error { return nil }
+78 -12
View File
@@ -3,34 +3,100 @@ package bundle
import ( import (
"fmt" "fmt"
"os" "os"
pathpkg "path"
"path/filepath" "path/filepath"
"strings" "strings"
) )
// WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed). // WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed).
func WriteExtractedFiles(root string, v *VerifiedContents) error { func WriteExtractedFiles(root string, v *VerifiedContents) error {
return WriteExtractedFilesWithLimits(root, v, Limits{})
}
// WriteExtractedFilesWithLimits confines all filesystem operations to an open
// directory handle. Replacing existing files uses unlink + exclusive creation,
// so existing hard links and symlinks cannot redirect writes outside the root.
func WriteExtractedFilesWithLimits(root string, v *VerifiedContents, limits Limits) error {
l, err := limits.Normalized()
if err != nil {
return err
}
if v == nil {
return fmt.Errorf("bundle: nil verified contents")
}
if len(v.Manifest.Files) > l.MaxEntries-2 {
return ErrLimit
}
seen := map[string]bool{}
var total int64
for _, fe := range v.Manifest.Files {
if err := validatePath(fe.Path, l.MaxPathBytes); err != nil {
return err
}
if seen[strings.ToLower(fe.Path)] {
return fmt.Errorf("bundle: duplicate path %q", fe.Path)
}
seen[strings.ToLower(fe.Path)] = true
data, ok := v.Files[fe.Path]
if !ok {
return fmt.Errorf("bundle: missing file %q", fe.Path)
}
if int64(len(data)) > l.MaxFileBytes || int64(len(data)) > l.MaxExpandedBytes-total {
return ErrLimit
}
total += int64(len(data))
}
root = filepath.Clean(root) root = filepath.Clean(root)
if err := os.MkdirAll(root, 0o755); err != nil {
return err
}
confined, err := os.OpenRoot(root)
if err != nil {
return err
}
defer func() { _ = confined.Close() }()
for _, fe := range v.Manifest.Files { for _, fe := range v.Manifest.Files {
data, ok := v.Files[fe.Path] data, ok := v.Files[fe.Path]
if !ok { if !ok {
return fmt.Errorf("bundle: missing file %q", fe.Path) return fmt.Errorf("bundle: missing file %q", fe.Path)
} }
rel := pathpkg.Clean(strings.TrimPrefix(fe.Path, "/")) parts := strings.Split(fe.Path, "/")
if rel == "." || strings.HasPrefix(rel, "..") { for i := 1; i < len(parts); i++ {
return fmt.Errorf("bundle: unsafe path %q", fe.Path) dir := filepath.FromSlash(strings.Join(parts[:i], "/"))
} if err := confined.Mkdir(dir, 0o755); err != nil && !os.IsExist(err) {
dest := filepath.Join(root, filepath.FromSlash(rel))
relToRoot, err := filepath.Rel(root, dest)
if err != nil || strings.HasPrefix(relToRoot, "..") {
return fmt.Errorf("bundle: path escapes root: %q", fe.Path)
}
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err return err
} }
if err := os.WriteFile(dest, data, 0o644); err != nil { info, err := confined.Lstat(dir)
if err != nil {
return err return err
} }
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("bundle: unsafe directory %q", dir)
}
}
dest := filepath.FromSlash(fe.Path)
info, err := confined.Lstat(dest)
if err == nil {
if !info.Mode().IsRegular() {
return fmt.Errorf("bundle: unsafe destination %q", dest)
}
if err = confined.Remove(dest); err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
f, err := confined.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return err
}
_, writeErr := f.Write(data)
closeErr := f.Close()
if writeErr != nil {
return writeErr
}
if closeErr != nil {
return closeErr
}
} }
return nil return nil
} }
@@ -0,0 +1,120 @@
package bundle
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"io"
"os"
"reflect"
"sort"
"testing"
)
// legacyManifestV1 mirrors the decoder fields present in the locally available
// v1.0.0 source. It deliberately has no future capability or deployment fields.
type legacyManifestV1 struct {
RevisionID string `json:"revision_id"`
SpeakerID string `json:"speaker_id"`
CreatedAt string `json:"created_at"`
Files []struct {
Path string `json:"path"`
SHA256 string `json:"sha256"`
} `json:"files"`
Algorithm string `json:"signature_algorithm"`
PublicKeyBase64 string `json:"public_key_base64"`
}
func TestLegacyCompatibilityFrozenSignedCurrentBaseline(t *testing.T) {
raw, err := os.ReadFile("testdata/current_baseline_signed_bundle.tar.gz")
if err != nil {
t.Fatal(err)
}
pub := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{0x42}, ed25519.SeedSize)).Public().(ed25519.PublicKey)
verified, err := VerifyGzippedTar(raw, pub)
if err != nil {
t.Fatalf("frozen signed baseline rejected: %v", err)
}
if verified.Manifest.RevisionID != "revision-legacy" || verified.Manifest.SpeakerID != "speaker-legacy" || len(verified.Files) != 2 {
t.Fatalf("frozen baseline contents: %+v", verified.Manifest)
}
}
func TestLegacyCompatibilityBundleWire(t *testing.T) {
seed := bytes.Repeat([]byte{0x42}, ed25519.SeedSize)
priv := ed25519.NewKeyFromSeed(seed)
pub := priv.Public().(ed25519.PublicKey)
fragments := map[string]string{
"bird.conf": "router id 192.0.2.1;\n",
"bird.d/evobgp_prefixes_v4.conf": "# revision\n",
"_preview": "must not ship",
}
raw, err := BuildGzippedTar("revision-legacy", "speaker-legacy", fragments, priv)
if err != nil {
t.Fatal(err)
}
gr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
t.Fatal(err)
}
defer func() { _ = gr.Close() }()
tr := tar.NewReader(gr)
entries := map[string][]byte{}
var order []string
for {
h, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
t.Fatal(err)
}
body, err := io.ReadAll(tr)
if err != nil {
t.Fatal(err)
}
order = append(order, h.Name)
entries[h.Name] = body
}
wantOrder := []string{"manifest.json", "bundle.sig", "bird.conf", "bird.d/evobgp_prefixes_v4.conf"}
if !reflect.DeepEqual(order, wantOrder) {
t.Fatalf("legacy tar layout: got %v, want %v", order, wantOrder)
}
var top map[string]json.RawMessage
if err := json.Unmarshal(entries["manifest.json"], &top); err != nil {
t.Fatal(err)
}
wantFields := []string{"created_at", "files", "public_key_base64", "revision_id", "signature_algorithm", "speaker_id"}
fields := make([]string, 0, len(top))
for k := range top {
fields = append(fields, k)
}
sort.Strings(fields)
if !reflect.DeepEqual(fields, wantFields) {
t.Fatalf("manifest fields: got %v, want %v", fields, wantFields)
}
var m legacyManifestV1
if err := json.Unmarshal(entries["manifest.json"], &m); err != nil {
t.Fatal(err)
}
if m.RevisionID != "revision-legacy" || m.SpeakerID != "speaker-legacy" || m.CreatedAt == "" || m.Algorithm != "ed25519" || m.PublicKeyBase64 != base64.StdEncoding.EncodeToString(pub) {
t.Fatalf("legacy manifest fields: %+v", m)
}
if !ed25519.Verify(pub, entries["manifest.json"], entries["bundle.sig"]) {
t.Fatal("legacy Ed25519 verifier rejected current signed manifest")
}
if len(m.Files) != 2 {
t.Fatalf("legacy manifest file count: %d", len(m.Files))
}
for _, f := range m.Files {
sum := sha256.Sum256(entries[f.Path])
if hex.EncodeToString(sum[:]) != f.SHA256 {
t.Fatalf("legacy SHA-256 mismatch for %q", f.Path)
}
}
}
+103
View File
@@ -0,0 +1,103 @@
package bundle
import (
"errors"
"fmt"
"io"
"path"
"strings"
)
// ErrLimit reports that a transport, archive or extraction budget was exceeded.
var ErrLimit = errors.New("bundle: resource limit exceeded")
// Limits bounds both archive transport and decoded tar data, including metadata.
// Zero fields select defaults. These ceilings permit large (1M-prefix) configs;
// they are safety ceilings, not a promise of suitability for a small-RAM node.
type Limits struct {
MaxCompressedBytes int64
MaxExpandedBytes int64
MaxFileBytes int64
MaxManifestBytes int64
MaxEntries int
MaxPathBytes int
}
// DefaultLimits permits large production configs while bounding untrusted input.
func DefaultLimits() Limits {
return Limits{MaxCompressedBytes: 128 << 20, MaxExpandedBytes: 512 << 20, MaxFileBytes: 256 << 20, MaxManifestBytes: 2 << 20, MaxEntries: 4096, MaxPathBytes: 1024}
}
// Normalized fills zero fields with defaults and rejects invalid budgets.
func (l Limits) Normalized() (Limits, error) {
d := DefaultLimits()
values := []struct {
v *int64
fallback int64
}{{&l.MaxCompressedBytes, d.MaxCompressedBytes}, {&l.MaxExpandedBytes, d.MaxExpandedBytes}, {&l.MaxFileBytes, d.MaxFileBytes}, {&l.MaxManifestBytes, d.MaxManifestBytes}}
for _, x := range values {
if *x.v < 0 || *x.v == int64(^uint64(0)>>1) {
return l, fmt.Errorf("bundle: invalid byte limit")
}
if *x.v == 0 {
*x.v = x.fallback
}
}
if l.MaxEntries < 0 || l.MaxPathBytes < 0 {
return l, fmt.Errorf("bundle: invalid entry/path limit")
}
if l.MaxEntries == 0 {
l.MaxEntries = d.MaxEntries
}
if l.MaxPathBytes == 0 {
l.MaxPathBytes = d.MaxPathBytes
}
return l, nil
}
func validatePath(name string, limit int) error {
if name == "" || len(name) > limit || path.IsAbs(name) || path.Clean(name) != name || name == "." || strings.ContainsAny(name, "\\:\x00") {
return fmt.Errorf("bundle: unsafe path %q", name)
}
for _, part := range strings.Split(name, "/") {
if part == ".." || strings.TrimRight(part, " .") != part {
return fmt.Errorf("bundle: unsafe path %q", name)
}
base := strings.ToUpper(strings.SplitN(part, ".", 2)[0])
if base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || (len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9') {
return fmt.Errorf("bundle: reserved path %q", name)
}
}
return nil
}
type limitedWriter struct {
w io.Writer
remaining int64
}
func (w *limitedWriter) Write(p []byte) (int, error) {
if int64(len(p)) > w.remaining {
return 0, ErrLimit
}
n, err := w.w.Write(p)
w.remaining -= int64(n)
return n, err
}
// ReadCompressed rejects oversized input while reading, including chunked HTTP
// bodies and local files whose size changes after stat.
func ReadCompressed(r io.Reader, limits Limits) ([]byte, error) {
l, err := limits.Normalized()
if err != nil {
return nil, err
}
b, err := io.ReadAll(io.LimitReader(r, l.MaxCompressedBytes+1))
if err != nil {
return nil, err
}
if int64(len(b)) > l.MaxCompressedBytes {
return nil, ErrLimit
}
return b, nil
}
+286
View File
@@ -0,0 +1,286 @@
package bundle
import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/ed25519"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
)
type testEntry struct {
header tar.Header
body []byte
}
func archiveEntries(t *testing.T, entries []testEntry) []byte {
t.Helper()
var b bytes.Buffer
gz := gzip.NewWriter(&b)
tw := tar.NewWriter(gz)
for _, entry := range entries {
h := entry.header
if h.Typeflag == 0 {
h.Typeflag = tar.TypeReg
}
h.Size = int64(len(entry.body))
if err := tw.WriteHeader(&h); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(entry.body); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
if err := gz.Close(); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func testKey() ed25519.PrivateKey {
return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize))
}
func TestBundleResourceLimits(t *testing.T) {
pub := testKey().Public().(ed25519.PublicKey)
cases := []struct {
name string
entries []testEntry
limits Limits
}{
{"file", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxFileBytes: 1024}},
{"expanded", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 8192)}}, Limits{MaxExpandedBytes: 4096}},
{"manifest", []testEntry{{header: tar.Header{Name: manifestName}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxManifestBytes: 1024}},
{"entries", []testEntry{{header: tar.Header{Name: "a"}}, {header: tar.Header{Name: "b"}}, {header: tar.Header{Name: "c"}}}, Limits{MaxEntries: 2}},
{"pax-metadata", []testEntry{{header: tar.Header{Name: "bird.conf", Format: tar.FormatPAX, PAXRecords: map[string]string{"comment": strings.Repeat("a", 8192)}}}}, Limits{MaxExpandedBytes: 4096}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
raw := archiveEntries(t, tc.entries)
if _, err := VerifyGzippedTarWithLimits(raw, pub, tc.limits); !errors.Is(err, ErrLimit) {
t.Fatalf("want limit, got %v", err)
}
})
}
if _, err := VerifyGzippedTarWithLimits(make([]byte, 65), pub, Limits{MaxCompressedBytes: 64}); !errors.Is(err, ErrLimit) {
t.Fatalf("compressed cap: %v", err)
}
}
func TestBundleRejectsUnsafeEntries(t *testing.T) {
pub := testKey().Public().(ed25519.PublicKey)
for _, name := range []string{"../escape", "/etc/bird.conf", `bird.d\escape`, "C:/escape", "bird.d/../escape", "a//b", "a/./b", "NUL", "bird.conf:stream", "a. /b"} {
t.Run(name, func(t *testing.T) {
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: name}, body: []byte("x")}})
if _, err := VerifyGzippedTar(raw, pub); err == nil {
t.Fatal("unsafe path accepted")
}
})
}
for _, kind := range []byte{tar.TypeSymlink, tar.TypeLink, tar.TypeDir, tar.TypeFifo} {
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf", Typeflag: kind, Linkname: "outside"}}})
if _, err := VerifyGzippedTar(raw, pub); err == nil {
t.Fatalf("type %d accepted", kind)
}
}
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf"}}, {header: tar.Header{Name: "bird.conf"}}})
if _, err := VerifyGzippedTar(raw, pub); err == nil || !strings.Contains(err.Error(), "duplicate") {
t.Fatalf("duplicate accepted: %v", err)
}
}
func TestBundleChecksTrailerAndTrailingExpansion(t *testing.T) {
priv := testKey()
pub := priv.Public().(ed25519.PublicKey)
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "router id 192.0.2.1;"}, priv)
if err != nil {
t.Fatal(err)
}
corrupt := append([]byte{}, raw...)
corrupt[len(corrupt)-8] ^= 0xff
if _, err = VerifyGzippedTar(corrupt, pub); err == nil {
t.Fatal("gzip checksum ignored after tar EOF")
}
var extra bytes.Buffer
gz := gzip.NewWriter(&extra)
if _, err = gz.Write(bytes.Repeat([]byte{'x'}, 32<<10)); err != nil {
t.Fatal(err)
}
if err = gz.Close(); err != nil {
t.Fatal(err)
}
joined := append(append([]byte{}, raw...), extra.Bytes()...)
if _, err = VerifyGzippedTarWithLimits(joined, pub, Limits{MaxExpandedBytes: 8 << 10}); !errors.Is(err, ErrLimit) {
t.Fatalf("trailing gzip member bypassed budget: %v", err)
}
}
func TestBundleRejectsPAXSparseRegularFile(t *testing.T) {
// archive/tar expands PAX sparse holes even though the returned type is a
// regular file. Build a physical one-byte entry with a 4096-byte logical size.
record := func(key, value string) string {
body := key + "=" + value + "\n"
n := len(body) + 2
for {
next := len(fmt.Sprint(n)) + 1 + len(body)
if next == n {
return fmt.Sprintf("%d %s", n, body)
}
n = next
}
}
pax := record("GNU.sparse.size", "4096") + record("GNU.sparse.numblocks", "1") + record("GNU.sparse.map", "0,1")
var physical bytes.Buffer
tw := tar.NewWriter(&physical)
for _, entry := range []testEntry{{header: tar.Header{Name: "PaxHeader", Size: int64(len(pax))}, body: []byte(pax)}, {header: tar.Header{Name: "bird.conf", Size: 1}, body: []byte("x")}} {
if err := tw.WriteHeader(&entry.header); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(entry.body); err != nil {
t.Fatal(err)
}
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
data := physical.Bytes()
data[156] = tar.TypeXHeader
for i := 148; i < 156; i++ {
data[i] = ' '
}
sum := 0
for _, b := range data[:512] {
sum += int(b)
}
copy(data[148:156], fmt.Sprintf("%06o\x00 ", sum))
h, err := tar.NewReader(bytes.NewReader(data)).Next()
if err != nil || h.Typeflag != tar.TypeReg || h.Size != 4096 {
t.Fatalf("invalid sparse regression fixture %+v %v", h, err)
}
var compressed bytes.Buffer
gz := gzip.NewWriter(&compressed)
if _, err = gz.Write(data); err != nil {
t.Fatal(err)
}
if err = gz.Close(); err != nil {
t.Fatal(err)
}
if _, err = VerifyGzippedTar(compressed.Bytes(), testKey().Public().(ed25519.PublicKey)); err == nil || !strings.Contains(err.Error(), "sparse") {
t.Fatalf("PAX sparse accepted: %v", err)
}
}
func TestBundleRejectsUnmanifestedPayload(t *testing.T) {
priv := testKey()
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "x"}, priv)
if err != nil {
t.Fatal(err)
}
gr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil {
t.Fatal(err)
}
defer func() { _ = gr.Close() }()
tr := tar.NewReader(gr)
var entries []testEntry
for {
h, e := tr.Next()
if e == io.EOF {
break
}
if e != nil {
t.Fatal(e)
}
b, e := io.ReadAll(tr)
if e != nil {
t.Fatal(e)
}
entries = append(entries, testEntry{header: *h, body: b})
}
entries = append(entries, testEntry{header: tar.Header{Name: "unsigned.conf"}, body: []byte("unsigned")})
if _, err = VerifyGzippedTar(archiveEntries(t, entries), priv.Public().(ed25519.PublicKey)); err == nil {
t.Fatal("unmanifested payload accepted")
}
}
func TestExtractionDoesNotFollowLinks(t *testing.T) {
root := t.TempDir()
outside := filepath.Join(t.TempDir(), "outside")
if err := os.WriteFile(outside, []byte("preserve"), 0o600); err != nil {
t.Fatal(err)
}
dest := filepath.Join(root, "bird.conf")
if err := os.Link(outside, dest); err != nil {
if runtime.GOOS == "linux" {
t.Fatalf("Linux link regression requires hard links: %v", err)
}
t.Skipf("hard links unavailable: %v", err)
}
v := &VerifiedContents{Manifest: Manifest{Files: []FileEntry{{Path: "bird.conf"}}}, Files: map[string][]byte{"bird.conf": []byte("replacement")}}
if err := WriteExtractedFiles(root, v); err != nil {
t.Fatal(err)
}
got, err := os.ReadFile(outside)
if err != nil || string(got) != "preserve" {
t.Fatalf("hard link modified outside: %s %v", got, err)
}
if err = os.Remove(dest); err != nil {
t.Fatal(err)
}
if err = os.Symlink(outside, dest); err != nil {
if runtime.GOOS == "linux" {
t.Fatalf("Linux link regression requires symlinks: %v", err)
}
t.Skipf("symlinks unavailable: %v", err)
}
if err = WriteExtractedFiles(root, v); err == nil {
t.Fatal("symlink destination accepted")
}
got, err = os.ReadFile(outside)
if err != nil || string(got) != "preserve" {
t.Fatal("symlink modified outside")
}
}
func TestBundleBuilderBoundsAndPortablePaths(t *testing.T) {
for _, name := range []string{"/bird.conf", "a/../bird.conf", manifestName, sigFileName, `bird.d\bad`} {
if _, err := BuildGzippedTar("r", "s", map[string]string{name: "x"}, testKey()); err == nil {
t.Fatalf("invalid builder path accepted: %q", name)
}
}
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": strings.Repeat("x", 1025)}, testKey(), Limits{MaxFileBytes: 1024}); !errors.Is(err, ErrLimit) {
t.Fatalf("builder file cap %v", err)
}
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": "x"}, testKey(), Limits{MaxExpandedBytes: 1024}); !errors.Is(err, ErrLimit) {
t.Fatalf("builder tar overhead cap %v", err)
}
}
// This is a decoded-size regression using repeated representative route lines,
// not a unique-prefix cardinality test or a BIRD validity/performance benchmark.
func TestDefaultLimitsAllowMillionRepresentativeIPv6RouteLines(t *testing.T) {
if testing.Short() {
t.Skip("large representative config")
}
config := strings.Repeat("route 2001:db8:1234:5678::/64 blackhole;\n", 1_000_000)
raw, err := BuildGzippedTar("large", "speaker", map[string]string{"bird.conf": config}, testKey())
if err != nil {
t.Fatal(err)
}
v, err := VerifyGzippedTar(raw, testKey().Public().(ed25519.PublicKey))
if err != nil {
t.Fatal(err)
}
if len(v.Files["bird.conf"]) != len(config) {
t.Fatal("large config truncated")
}
}
+35 -8
View File
@@ -9,7 +9,6 @@ import (
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"path"
"sort" "sort"
"strings" "strings"
"time" "time"
@@ -36,22 +35,47 @@ const manifestName = "manifest.json"
// BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files. // BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files.
func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) { func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
return BuildGzippedTarWithLimits(revisionID, speakerID, fragments, priv, Limits{})
}
// BuildGzippedTarWithLimits preserves the manifest/signature wire format while
// refusing payloads that exceed the receiving node's configured budgets.
func BuildGzippedTarWithLimits(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey, limits Limits) ([]byte, error) {
l, err := limits.Normalized()
if err != nil {
return nil, err
}
if len(priv) != ed25519.PrivateKeySize { if len(priv) != ed25519.PrivateKeySize {
return nil, fmt.Errorf("bundle: invalid ed25519 private key size") return nil, fmt.Errorf("bundle: invalid ed25519 private key size")
} }
pub := priv.Public().(ed25519.PublicKey) pub := priv.Public().(ed25519.PublicKey)
norm := make(map[string]string, len(fragments)) norm := make(map[string]string)
seen := map[string]bool{}
var total int64
for p, content := range fragments { for p, content := range fragments {
p = path.Clean(strings.TrimPrefix(p, "/"))
if p == "." || strings.HasPrefix(p, "..") {
return nil, fmt.Errorf("bundle: invalid path %q", p)
}
if strings.HasPrefix(p, "_") { if strings.HasPrefix(p, "_") {
continue // UI-only preview aggregate, not shipped to nodes continue // UI-only preview aggregate, not shipped to nodes
} }
if len(norm) >= l.MaxEntries-2 {
return nil, ErrLimit
}
if err := validatePath(p, l.MaxPathBytes); err != nil {
return nil, err
}
if p == manifestName || p == sigFileName || seen[strings.ToLower(p)] {
return nil, fmt.Errorf("bundle: duplicate or reserved path %q", p)
}
seen[strings.ToLower(p)] = true
if int64(len(content)) > l.MaxFileBytes || int64(len(content)) > l.MaxExpandedBytes-total {
return nil, ErrLimit
}
total += int64(len(content))
norm[p] = content norm[p] = content
} }
if len(norm) > l.MaxEntries-2 {
return nil, ErrLimit
}
var files []FileEntry var files []FileEntry
var payload [][]byte var payload [][]byte
@@ -75,11 +99,14 @@ func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string,
if err != nil { if err != nil {
return nil, err return nil, err
} }
if int64(len(manifestJSON)) > l.MaxManifestBytes {
return nil, ErrLimit
}
sig := ed25519.Sign(priv, manifestJSON) sig := ed25519.Sign(priv, manifestJSON)
var buf bytes.Buffer var buf bytes.Buffer
gw := gzip.NewWriter(&buf) gw := gzip.NewWriter(&limitedWriter{w: &buf, remaining: l.MaxCompressedBytes})
tw := tar.NewWriter(gw) tw := tar.NewWriter(&limitedWriter{w: gw, remaining: l.MaxExpandedBytes})
add := func(name string, body []byte) error { add := func(name string, body []byte) error {
hdr := &tar.Header{ hdr := &tar.Header{
Binary file not shown.
+92 -18
View File
@@ -11,44 +11,95 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"path"
"strings" "strings"
) )
// VerifiedContents is the result of verifying a downloaded bundle on evobgp-node. // VerifiedContents contains only payloads covered by a verified signed manifest.
type VerifiedContents struct { type VerifiedContents struct {
Manifest Manifest Manifest Manifest
Files map[string][]byte Files map[string][]byte
} }
// VerifyGzippedTar checks Ed25519 signature over manifest.json and SHA-256 of each file. // VerifyGzippedTar verifies the unchanged signed manifest format with default limits.
func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, error) { func VerifyGzippedTar(raw []byte, pub ed25519.PublicKey) (*VerifiedContents, error) {
gr, err := gzip.NewReader(bytes.NewReader(bundle)) return VerifyGzippedTarWithLimits(raw, pub, Limits{})
}
// VerifyGzippedTarWithLimits verifies the signature and hashes under explicit
// compressed, expanded, per-entry and metadata budgets.
func VerifyGzippedTarWithLimits(raw []byte, pub ed25519.PublicKey, limits Limits) (*VerifiedContents, error) {
l, err := limits.Normalized()
if err != nil {
return nil, err
}
if int64(len(raw)) > l.MaxCompressedBytes {
return nil, ErrLimit
}
if len(pub) != ed25519.PublicKeySize {
return nil, fmt.Errorf("bundle: invalid public key size")
}
gr, err := gzip.NewReader(bytes.NewReader(raw))
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer func() { _ = gr.Close() }() defer func() { _ = gr.Close() }()
decoded := &io.LimitedReader{R: gr, N: l.MaxExpandedBytes + 1}
var manifestRaw []byte tr := tar.NewReader(decoded)
var sig []byte var manifestRaw, sig []byte
files := make(map[string][]byte) files := map[string][]byte{}
tr := tar.NewReader(gr) seen := map[string]bool{}
count := 0
var logicalBytes int64
for { for {
hdr, err := tr.Next() hdr, err := tr.Next()
if decoded.N == 0 {
return nil, ErrLimit
}
if err == io.EOF { if err == io.EOF {
break break
} }
if err != nil { if err != nil {
return nil, err return nil, err
} }
name := path.Clean(hdr.Name) count++
if strings.HasPrefix(name, "..") { if count > l.MaxEntries {
return nil, fmt.Errorf("bundle: illegal tar entry %q", hdr.Name) return nil, ErrLimit
} }
data, err := io.ReadAll(tr) if err = validatePath(hdr.Name, l.MaxPathBytes); err != nil {
return nil, err
}
name := hdr.Name
folded := strings.ToLower(name)
if seen[folded] {
return nil, fmt.Errorf("bundle: duplicate tar entry %q", name)
}
seen[folded] = true
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
return nil, fmt.Errorf("bundle: nonregular tar entry %q", name)
}
for key := range hdr.PAXRecords {
if strings.HasPrefix(key, "GNU.sparse.") {
return nil, fmt.Errorf("bundle: sparse entry %q is unsupported", name)
}
}
max := l.MaxFileBytes
if name == manifestName {
max = l.MaxManifestBytes
}
if name == sigFileName {
max = ed25519.SignatureSize
}
if hdr.Size < 0 || hdr.Size > max || hdr.Size >= decoded.N || hdr.Size > l.MaxExpandedBytes-logicalBytes {
return nil, ErrLimit
}
logicalBytes += hdr.Size
data, err := io.ReadAll(io.LimitReader(tr, max+1))
if err != nil { if err != nil {
return nil, err return nil, err
} }
if int64(len(data)) > max || decoded.N == 0 {
return nil, ErrLimit
}
switch name { switch name {
case manifestName: case manifestName:
manifestRaw = data manifestRaw = data
@@ -58,17 +109,36 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
files[name] = data files[name] = data
} }
} }
if len(manifestRaw) == 0 || len(sig) == 0 { // tar EOF precedes the gzip trailer. Drain through the same byte budget so
// checksums, extra gzip members and trailing decompression are also bounded.
if _, err = io.Copy(io.Discard, decoded); err != nil {
return nil, err
}
if decoded.N == 0 {
return nil, ErrLimit
}
if len(manifestRaw) == 0 || len(sig) != ed25519.SignatureSize {
return nil, fmt.Errorf("bundle: missing manifest or signature") return nil, fmt.Errorf("bundle: missing manifest or signature")
} }
if !ed25519.Verify(pub, manifestRaw, sig) { if !ed25519.Verify(pub, manifestRaw, sig) {
return nil, fmt.Errorf("bundle: ed25519 signature mismatch") return nil, fmt.Errorf("bundle: ed25519 signature mismatch")
} }
var m Manifest var m Manifest
if err := json.Unmarshal(manifestRaw, &m); err != nil { if err = json.Unmarshal(manifestRaw, &m); err != nil {
return nil, err return nil, err
} }
if len(m.Files) > l.MaxEntries-2 {
return nil, ErrLimit
}
listed := map[string]bool{}
for _, fe := range m.Files { for _, fe := range m.Files {
if err = validatePath(fe.Path, l.MaxPathBytes); err != nil {
return nil, err
}
if fe.Path == manifestName || fe.Path == sigFileName || listed[strings.ToLower(fe.Path)] {
return nil, fmt.Errorf("bundle: duplicate or reserved manifest path %q", fe.Path)
}
listed[strings.ToLower(fe.Path)] = true
body, ok := files[fe.Path] body, ok := files[fe.Path]
if !ok { if !ok {
return nil, fmt.Errorf("bundle: missing file %q", fe.Path) return nil, fmt.Errorf("bundle: missing file %q", fe.Path)
@@ -78,10 +148,13 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path) return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path)
} }
} }
if len(listed) != len(files) {
return nil, fmt.Errorf("bundle: payload is not covered by manifest")
}
return &VerifiedContents{Manifest: m, Files: files}, nil return &VerifiedContents{Manifest: m, Files: files}, nil
} }
// ParsePublicKeyHex decodes a 64-char hex Ed25519 public key (32 bytes). // ParsePublicKeyHex decodes a 32-byte Ed25519 public key from hexadecimal.
func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) { func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X")) s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X"))
raw, err := hex.DecodeString(s) raw, err := hex.DecodeString(s)
@@ -94,7 +167,8 @@ func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
return ed25519.PublicKey(raw), nil return ed25519.PublicKey(raw), nil
} }
// PublicKeyFromManifestBase64 uses the key embedded in the signed manifest (cross-check with expected pubkey optional). // PublicKeyFromManifestBase64 decodes the manifest key. Callers must still trust
// an independently provisioned key when authenticating the bundle.
func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) { func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) {
if m.PublicKeyB64 == "" { if m.PublicKeyB64 == "" {
return nil, fmt.Errorf("bundle: manifest missing public_key_base64") return nil, fmt.Errorf("bundle: manifest missing public_key_base64")
+265 -21
View File
@@ -8,48 +8,292 @@ import (
"path" "path"
"sort" "sort"
"strings" "strings"
"time"
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
) )
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree (idempotent via schema_migrations). const (
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) error { postgresMigrationLockKey int64 = 0x45766f4247504d47 // "EvoBGPMG", scoped to the database.
if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil { migrationLockTimeout = 30 * time.Second
return fmt.Errorf("db: schema_migrations: %w", err) migrationCleanupTimeout = 5 * time.Second
} )
// PostgreSQL migrations are transaction-only. Non-transactional statements
// need an explicit, separately reviewed execution path.
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree.
// Each migration and its version row commit atomically under a session advisory lock.
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) (retErr error) {
entries, err := fs.ReadDir(fsys, subdir) entries, err := fs.ReadDir(fsys, subdir)
if err != nil { if err != nil {
return err return fmt.Errorf("db: read migrations: %w", err)
} }
var ups []string var ups []string
for _, e := range entries { for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") { if !e.IsDir() && strings.HasSuffix(e.Name(), ".up.sql") {
continue
}
ups = append(ups, e.Name()) ups = append(ups, e.Name())
} }
}
sort.Strings(ups) sort.Strings(ups)
for _, name := range ups { // Include pool acquisition in the lock timeout. The lock precedes even
ver := strings.TrimSuffix(name, ".up.sql") // schema_migrations creation, because another process may start from an empty DB.
var dummy int lockCtx, cancelLock := context.WithTimeout(ctx, migrationLockTimeout)
err := pool.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, ver).Scan(&dummy) conn, err := pool.Acquire(lockCtx)
if err != nil {
cancelLock()
return fmt.Errorf("db: acquire migration connection: %w", err)
}
defer conn.Release()
if _, err := conn.Exec(lockCtx, `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
cancelLock()
// Cancellation may race with a successful server-side lock acquisition.
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
defer closeCancel()
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
return errors.Join(fmt.Errorf("db: acquire migration lock: %w", err), fmt.Errorf("db: close migration connection: %w", closeErr))
}
return fmt.Errorf("db: acquire migration lock: %w", err)
}
cancelLock()
defer func() {
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
defer cancel()
var unlocked bool
if err := conn.QueryRow(cleanupCtx, `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
if err == nil { if err == nil {
continue err = errors.New("lock was not held")
} }
if !errors.Is(err, pgx.ErrNoRows) { retErr = errors.Join(retErr, fmt.Errorf("db: release migration lock: %w", err))
return err // Never return a possibly locked session to the pool.
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
defer closeCancel()
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
retErr = errors.Join(retErr, fmt.Errorf("db: close migration connection: %w", closeErr))
} }
}
}()
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil {
return fmt.Errorf("db: schema_migrations: %w", err)
}
for _, name := range ups {
body, err := fs.ReadFile(fsys, path.Join(subdir, name)) body, err := fs.ReadFile(fsys, path.Join(subdir, name))
if err != nil { if err != nil {
return err return fmt.Errorf("db: read migration %s: %w", name, err)
} }
if _, err := pool.Exec(ctx, string(body)); err != nil { if bad := unsupportedMigrationStatement(string(body)); bad != "" {
return fmt.Errorf("db: migration %s uses unsupported non-transactional SQL %q", name, bad)
}
if err := applyPostgresMigration(ctx, conn, strings.TrimSuffix(name, ".up.sql"), body); err != nil {
return fmt.Errorf("db: migrate %s: %w", name, err) return fmt.Errorf("db: migrate %s: %w", name, err)
} }
if _, err := pool.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, ver); err != nil {
return err
}
} }
return nil return nil
} }
// unsupportedMigrationStatement inspects statement starts after masking SQL
// comments and quoted strings. In particular, a comment after a semicolon
// must not conceal COMMIT or another command that would escape our transaction.
func unsupportedMigrationStatement(sql string) string {
const (
normal = iota
singleQuote
doubleQuote
lineComment
blockComment
dollarQuote
)
state := normal
blockDepth := 0
dollarTag := ""
escapeString := false
var clean strings.Builder
for i := 0; i < len(sql); {
switch state {
case normal:
switch {
case sql[i] == '\'':
escapeString = i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !isSQLIdentifierByte(sql[i-2]))
state = singleQuote
clean.WriteByte(' ')
i++
case sql[i] == '"':
state = doubleQuote
clean.WriteByte(' ')
i++
case strings.HasPrefix(sql[i:], "--"):
state = lineComment
clean.WriteByte(' ')
i += 2
case strings.HasPrefix(sql[i:], "/*"):
state = blockComment
blockDepth = 1
clean.WriteByte(' ')
i += 2
case sql[i] == '$' && (i == 0 || !isSQLIdentifierByte(sql[i-1])):
end := i + 1
if end < len(sql) && isDollarTagStartByte(sql[end]) {
end++
for end < len(sql) && isDollarTagByte(sql[end]) {
end++
}
}
if end < len(sql) && sql[end] == '$' {
dollarTag = sql[i : end+1]
state = dollarQuote
clean.WriteByte(' ')
i = end + 1
} else {
clean.WriteByte(sql[i])
i++
}
default:
clean.WriteByte(sql[i])
i++
}
case singleQuote:
switch {
case escapeString && sql[i] == '\\' && i+1 < len(sql):
i += 2
case sql[i] == '\'' && i+1 < len(sql) && sql[i+1] == '\'':
i += 2
case sql[i] == '\'':
state = normal
i++
default:
i++
}
case doubleQuote:
if sql[i] == '"' {
if i+1 < len(sql) && sql[i+1] == '"' {
i += 2
} else {
state = normal
i++
}
} else {
i++
}
case lineComment:
if sql[i] == '\n' || sql[i] == '\r' {
clean.WriteByte('\n')
state = normal
}
i++
case blockComment:
switch {
case strings.HasPrefix(sql[i:], "/*"):
blockDepth++
i += 2
case strings.HasPrefix(sql[i:], "*/"):
blockDepth--
i += 2
if blockDepth == 0 {
state = normal
}
default:
i++
}
case dollarQuote:
if strings.HasPrefix(sql[i:], dollarTag) {
i += len(dollarTag)
state = normal
} else {
i++
}
}
}
for _, statement := range strings.Split(clean.String(), ";") {
words := strings.Fields(strings.ToUpper(statement))
if isUnsupportedMigrationCommand(words) {
return strings.Join(words[:min(len(words), 4)], " ")
}
}
return ""
}
func isSQLIdentifierByte(b byte) bool {
return isDollarTagByte(b) || b == '$'
}
func isDollarTagStartByte(b byte) bool {
return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b == '_' || b >= 0x80
}
func isDollarTagByte(b byte) bool {
return isDollarTagStartByte(b) || b >= '0' && b <= '9'
}
func isUnsupportedMigrationCommand(words []string) bool {
if len(words) == 0 {
return false
}
second := ""
if len(words) > 1 {
second = words[1]
}
switch words[0] {
case "BEGIN", "COMMIT", "END", "ROLLBACK", "ABORT", "SAVEPOINT", "REINDEX", "VACUUM", "CLUSTER":
return true
case "START", "PREPARE":
return second == "TRANSACTION"
case "RELEASE":
return second == "SAVEPOINT" || second != ""
case "ALTER":
return second == "SYSTEM"
case "CREATE":
if second == "DATABASE" || second == "TABLESPACE" {
return true
}
idx := 2
if second == "UNIQUE" {
idx = 3
}
return len(words) > idx && words[idx-1] == "INDEX" && words[idx] == "CONCURRENTLY"
case "DROP":
return second == "DATABASE" || second == "TABLESPACE" || second == "INDEX" && len(words) > 2 && words[2] == "CONCURRENTLY"
case "REFRESH":
return len(words) > 3 && second == "MATERIALIZED" && words[2] == "VIEW" && words[3] == "CONCURRENTLY"
case "DISCARD":
return second == "ALL"
}
return false
}
func applyPostgresMigration(ctx context.Context, conn *pgxpool.Conn, version string, body []byte) (retErr error) {
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
committed := false
defer func() {
if committed {
return
}
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
defer cancel()
if err := tx.Rollback(cleanupCtx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
retErr = errors.Join(retErr, fmt.Errorf("db: rollback migration %s: %w", version, err))
}
}()
var present int
err = tx.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, version).Scan(&present)
if err == nil {
return nil
}
if !errors.Is(err, pgx.ErrNoRows) {
return err
}
if _, err := tx.Exec(ctx, string(body)); err != nil {
return err
}
if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, version); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
committed = true
return nil
}
+63
View File
@@ -0,0 +1,63 @@
package db
import (
"io/fs"
"path"
"strings"
"testing"
"evobgp/migrations"
)
func TestPostgresEmbeddedMigrationsTransactional(t *testing.T) {
entries, err := fs.ReadDir(migrations.Postgres, "postgres")
if err != nil {
t.Fatal(err)
}
for _, entry := range entries {
if !strings.HasSuffix(entry.Name(), ".up.sql") {
continue
}
body, err := fs.ReadFile(migrations.Postgres, path.Join("postgres", entry.Name()))
if err != nil {
t.Fatal(err)
}
if bad := unsupportedMigrationStatement(string(body)); bad != "" {
t.Errorf("%s contains unsupported transaction control or DDL: %q", entry.Name(), bad)
}
}
}
func TestPostgresMigrationRejectsTransactionControl(t *testing.T) {
for _, statement := range []string{
"COMMIT;", "END;", "ROLLBACK;", "ABORT;", "BEGIN;",
"CREATE INDEX CONCURRENTLY idx ON t (id);", "VACUUM t;",
"CREATE TABLE t(id int); /* boundary */ COMMIT; SELECT 1/0;",
"/* comment */COMMIT;",
"-- comment\nCOMMIT;",
"/* nested /* comment */ still comment */ END;",
"CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$",
"-- comment\rCOMMIT;",
`foo$E'abc\'; COMMIT;`,
`éE'abc\'; COMMIT;`,
} {
if unsupportedMigrationStatement(statement) == "" {
t.Errorf("did not reject %q", statement)
}
}
}
func TestPostgresMigrationIgnoresQuotedControlWords(t *testing.T) {
for _, statement := range []string{
"SELECT 'COMMIT;'::text;",
"SELECT $$COMMIT;$$;",
"SELECT $tag$END;$tag$;",
"SELECT E'ABORT;\\'';",
"SELECT \"ROLLBACK\" FROM t;",
"-- COMMIT;\nSELECT 1;",
} {
if bad := unsupportedMigrationStatement(statement); bad != "" {
t.Errorf("rejected quoted or commented SQL %q: %s", statement, bad)
}
}
}
+24 -11
View File
@@ -25,17 +25,7 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if max := os.Getenv("EVOBGP_DB_MAX_CONNS"); max != "" { applyPostgresPoolConfig(cfg)
if n, err := strconv.Atoi(strings.TrimSpace(max)); err == nil && n > 0 {
cfg.MaxConns = int32(n)
}
}
if min := os.Getenv("EVOBGP_DB_MIN_CONNS"); min != "" {
if n, err := strconv.Atoi(strings.TrimSpace(min)); err == nil && n >= 0 {
cfg.MinConns = int32(n)
}
}
cfg.MaxConnLifetime = 30 * time.Minute
pool, err := pgxpool.NewWithConfig(ctx, cfg) pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -47,6 +37,29 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
return pool, nil return pool, nil
} }
// applyPostgresPoolConfig sets pool limits. An explicit EVOBGP_DB_MAX_CONNS is kept.
// When it is unset, the pool defaults to 8 connections so it can cover the default
// worker count instead of pgx's default of 4.
func applyPostgresPoolConfig(cfg *pgxpool.Config) {
if cfg == nil {
return
}
if max := strings.TrimSpace(os.Getenv("EVOBGP_DB_MAX_CONNS")); max != "" {
if n, err := strconv.Atoi(max); err == nil && n > 0 {
cfg.MaxConns = int32(n)
}
} else {
cfg.MaxConns = 8
}
if min := strings.TrimSpace(os.Getenv("EVOBGP_DB_MIN_CONNS")); min != "" {
if n, err := strconv.Atoi(min); err == nil && n >= 0 {
cfg.MinConns = int32(n)
}
}
cfg.MaxConnLifetime = 30 * time.Minute
cfg.MaxConnIdleTime = 5 * time.Minute
}
// OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on). // OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on).
func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) { func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) {
dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:")) dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:"))
+34
View File
@@ -0,0 +1,34 @@
package db
import (
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestApplyPostgresPoolConfigDefaultAndExplicit(t *testing.T) {
t.Setenv("EVOBGP_DB_MAX_CONNS", "")
t.Setenv("EVOBGP_DB_MIN_CONNS", "")
cfg, err := pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
if err != nil {
t.Fatal(err)
}
applyPostgresPoolConfig(cfg)
if cfg.MaxConns != 8 {
t.Fatalf("default max conns: %d", cfg.MaxConns)
}
if cfg.MaxConnIdleTime != 5*time.Minute {
t.Fatalf("idle time: %s", cfg.MaxConnIdleTime)
}
t.Setenv("EVOBGP_DB_MAX_CONNS", "25")
cfg, err = pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
if err != nil {
t.Fatal(err)
}
applyPostgresPoolConfig(cfg)
if cfg.MaxConns != 25 {
t.Fatalf("explicit max conns overridden: %d", cfg.MaxConns)
}
}
+227
View File
@@ -0,0 +1,227 @@
package db
import (
"context"
"crypto/rand"
"encoding/hex"
"errors"
"io/fs"
"os"
"strings"
"sync"
"testing"
"testing/fstest"
"time"
"evobgp/migrations"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestPostgresMigrationsIntegration(t *testing.T) {
pool := isolatedMigrationPool(t)
ctx := context.Background()
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
t.Fatalf("first migration pass: %v", err)
}
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
t.Fatalf("second migration pass: %v", err)
}
files, err := fs.ReadDir(migrations.Postgres, "postgres")
if err != nil {
t.Fatal(err)
}
want := 0
for _, file := range files {
if strings.HasSuffix(file.Name(), ".up.sql") {
want++
}
}
var got int
if err := pool.QueryRow(ctx, `SELECT count(*)::int FROM schema_migrations`).Scan(&got); err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("applied migrations = %d, want %d", got, want)
}
var extension string
if err := pool.QueryRow(ctx, `SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements'`).Scan(&extension); err != nil {
t.Fatalf("pg_stat_statements extension missing: %v", err)
}
}
func TestPostgresMigrationConcurrentApplyIntegration(t *testing.T) {
pool := isolatedMigrationPool(t)
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE migration_probe (id integer PRIMARY KEY); SELECT pg_sleep(0.1);`)}}
var wg sync.WaitGroup
errors := make(chan error, 2)
for i := 0; i < 2; i++ {
wg.Add(1)
go func() {
defer wg.Done()
errors <- ApplyPostgresMigrations(context.Background(), pool, files, "sql")
}()
}
wg.Wait()
close(errors)
for err := range errors {
if err != nil {
t.Fatal(err)
}
}
assertMigrationVersionCount(t, pool, 1)
}
func TestPostgresMigrationRollbackIntegration(t *testing.T) {
pool := isolatedMigrationPool(t)
files := fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); SELECT 1 / 0;`)}}
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil {
t.Fatal("expected migration error")
}
assertMigrationVersionCount(t, pool, 0)
var exists bool
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
t.Fatal(err)
}
if exists {
t.Fatal("failed migration left a table behind")
}
// A transaction terminator hidden behind a comment must be rejected before
// the first DDL statement can run.
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); /* boundary */ COMMIT; SELECT 1 / 0;`)}}
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
t.Fatalf("comment-hidden COMMIT error = %v", err)
}
assertMigrationVersionCount(t, pool, 0)
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
t.Fatal(err)
}
if exists {
t.Fatal("rejected migration left a table behind")
}
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$`)}}
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
t.Fatalf("identifier-embedded dollar tag error = %v", err)
}
assertMigrationVersionCount(t, pool, 0)
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('foo$tag$') IS NOT NULL`).Scan(&exists); err != nil {
t.Fatal(err)
}
if exists {
t.Fatal("identifier-embedded dollar tag left a table behind")
}
}
func TestPostgresMigrationIdempotentIntegration(t *testing.T) {
pool := isolatedMigrationPool(t)
files := fstest.MapFS{"sql/000001_once.up.sql": {Data: []byte(`CREATE TABLE migration_once (id integer); INSERT INTO migration_once VALUES (1);`)}}
for i := 0; i < 2; i++ {
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
t.Fatal(err)
}
}
assertMigrationVersionCount(t, pool, 1)
var rows int
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM migration_once`).Scan(&rows); err != nil {
t.Fatal(err)
}
if rows != 1 {
t.Fatalf("migration rows = %d, want 1", rows)
}
}
func TestPostgresMigrationLockCancellationIntegration(t *testing.T) {
pool := isolatedMigrationPool(t)
locker, err := pool.Acquire(context.Background())
if err != nil {
t.Fatal(err)
}
defer locker.Release()
if _, err := locker.Exec(context.Background(), `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
t.Fatal(err)
}
defer func() {
_, _ = locker.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey)
}()
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE lock_probe (id integer);`)}}
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
defer cancel()
err = ApplyPostgresMigrations(ctx, pool, files, "sql")
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("lock wait error = %v, want context deadline exceeded", err)
}
var exists bool
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('schema_migrations') IS NOT NULL`).Scan(&exists); err != nil {
t.Fatal(err)
}
if exists {
t.Fatal("schema_migrations was created before migration lock acquisition")
}
var unlocked bool
if err := locker.QueryRow(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
t.Fatalf("release held lock: unlocked=%v err=%v", unlocked, err)
}
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
t.Fatalf("apply after lock release: %v", err)
}
}
func isolatedMigrationPool(t *testing.T) *pgxpool.Pool {
t.Helper()
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
admin, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatal(err)
}
var nonce [8]byte
if _, err := rand.Read(nonce[:]); err != nil {
admin.Close()
t.Fatal(err)
}
schema := "evobgp_mig_test_" + hex.EncodeToString(nonce[:])
if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil {
admin.Close()
t.Fatal(err)
}
var pool *pgxpool.Pool
t.Cleanup(func() {
if pool != nil {
pool.Close()
}
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cleanupCancel()
if _, err := admin.Exec(cleanupCtx, `DROP SCHEMA `+schema+` CASCADE`); err != nil {
t.Errorf("drop isolated schema %s: %v", schema, err)
}
admin.Close()
})
cfg, err := pgxpool.ParseConfig(dsn)
if err != nil {
t.Fatal(err)
}
if cfg.ConnConfig.RuntimeParams == nil {
cfg.ConnConfig.RuntimeParams = make(map[string]string)
}
cfg.ConnConfig.RuntimeParams["search_path"] = schema
cfg.MaxConns = 3
pool, err = pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
return pool
}
func assertMigrationVersionCount(t *testing.T, pool *pgxpool.Pool, want int) {
t.Helper()
var got int
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM schema_migrations`).Scan(&got); err != nil {
t.Fatal(err)
}
if got != want {
t.Fatalf("migration versions = %d, want %d", got, want)
}
}
@@ -0,0 +1,39 @@
package httpapi
import (
"testing"
"evobgp/internal/store"
)
// These regressions ensure unobserved live state cannot inherit persisted health.
func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) {
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
cases := []struct {
name string
views []speakerBGPLive
}{
{name: "no_collector"},
{name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}},
{name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, peer, tc.views)
if row["session_state"] == "Established" {
t.Fatal("S01: persisted Established is exposed without a current session observation")
}
})
}
}
func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) {
sp := &store.Speaker{ID: "master", Role: "master"}
// collectSpeakerLiveStatus constructs this view when no poll result exists.
view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"}
row := speakerLiveStatusJSON(sp, view, nil)
if row["bgp_poll_ok"] == true || row["agent_ok"] == true {
t.Fatal("S01: absent poll is exposed as successful BGP/agent health")
}
}
+8 -39
View File
@@ -7,9 +7,9 @@ import (
"time" "time"
"evobgp/internal/db" "evobgp/internal/db"
"evobgp/internal/httpclient"
"evobgp/internal/jobs" "evobgp/internal/jobs"
"evobgp/internal/observability" "evobgp/internal/observability"
"evobgp/internal/pipeline"
"evobgp/internal/repository" "evobgp/internal/repository"
"evobgp/internal/store" "evobgp/internal/store"
@@ -18,7 +18,7 @@ import (
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03). // NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
func NewCDNHTTPClient() *http.Client { func NewCDNHTTPClient() *http.Client {
return httpclient.New(httpclient.DefaultTimeout) return pipeline.NewCDNHTTPClient()
} }
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys). // BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
@@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
backend = mem backend = mem
} }
cdnHTTP := NewCDNHTTPClient() // Configured DoH profiles and RIPE service overrides retain the previous
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP} // general HTTP/HTTPS transport, including explicitly configured private
// endpoints. CDN calls independently secure this client in UpstreamHTTPDo.
workerHTTP := pipeline.NewPipelineHTTPClient()
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP}
reg := jobs.NewRegistry(wk.Process) reg := jobs.NewRegistry(wk.Process)
wk.Registry = reg wk.Registry = reg
if pool != nil { if pool != nil {
audit := repository.NewJobAuditWriter(pool) reg.UseDurableQueue(backend)
jobMeta := func(j *jobs.Job) map[string]any {
if j == nil {
return nil
}
st := j.Snapshot()
meta, _ := st["meta"].(map[string]any)
return meta
}
reg.SetPersistHooks(
func(j *jobs.Job) {
if j == nil {
return
}
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
},
func(j *jobs.Job) {
if j == nil {
return
}
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
},
func(j *jobs.Job) {
if j == nil {
return
}
st := j.Snapshot()
status, _ := st["status"].(string)
var errMsg *string
if e, ok := st["error"].(string); ok && e != "" {
errMsg = &e
}
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
},
)
} }
observability.RegisterStoreBackend(backend) observability.RegisterStoreBackend(backend)
return backend, reg, pool, nil return backend, reg, pool, nil
+110
View File
@@ -0,0 +1,110 @@
package httpapi
import (
"context"
"encoding/base64"
"net"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
"evobgp/internal/httpclient"
"evobgp/internal/jobs"
"evobgp/internal/pipeline"
"evobgp/internal/store"
"github.com/miekg/dns"
)
func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) {
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
t.Setenv("EVOBGP_BIRDC_SOCKET", "")
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
var calls atomic.Int32
resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns"))
if err != nil {
w.WriteHeader(400)
return
}
query := new(dns.Msg)
if err = query.Unpack(raw); err != nil || len(query.Question) == 0 {
w.WriteHeader(400)
return
}
answer := new(dns.Msg)
answer.SetReply(query)
if query.Question[0].Qtype == dns.TypeA {
answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}}
}
body, err := answer.Pack()
if err != nil {
t.Error(err)
return
}
w.Header().Set("Content-Type", "application/dns-message")
_, _ = w.Write(body)
}))
defer resolver.Close()
st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true})
if err != nil {
t.Fatal(err)
}
defer func() {
reg.RequestCancelAll()
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if e := reg.Drain(ctx); e != nil {
t.Error(e)
}
}()
tenant, _, _, _, _ := st.DemoIDs()
profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL})
if err != nil {
t.Fatal(err)
}
mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}})
if err != nil {
t.Fatal(err)
}
if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil {
t.Fatal(err)
}
j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID})
if err != nil {
t.Fatal(err)
}
deadline := time.After(5 * time.Second)
ticker := time.NewTicker(5 * time.Millisecond)
defer ticker.Stop()
for {
snap := j.Snapshot()
status := snap["status"]
if status == jobs.StatusSucceeded {
break
}
if status == jobs.StatusFailed || status == jobs.StatusCancelled {
t.Fatalf("worker refresh failed: %+v", snap)
}
select {
case <-deadline:
t.Fatal("worker did not finish")
case <-ticker.C:
}
}
if calls.Load() < 2 {
t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load())
}
// The same general-client choice must not remove CDN's independent ACL.
cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") }))
defer cdn.Close()
req, err := http.NewRequest(http.MethodGet, cdn.URL, nil)
if err != nil {
t.Fatal(err)
}
if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil {
t.Fatal("private CDN accepted")
}
}
+81
View File
@@ -0,0 +1,81 @@
package httpapi
import (
"context"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"evobgp/internal/store"
)
func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) {
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1)))
}))
defer srv.Close()
st := store.NewMemory()
st.SeedDemo()
tenant, _, _, _, _ := st.DemoIDs()
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true})
if err != nil {
t.Fatal(err)
}
s := &Server{store: st, cdnHTTP: srv.Client()}
body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"})
if err != nil {
t.Fatal(err)
}
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body)))
req.SetPathValue("module_id", mod.ID)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
recorder := httptest.NewRecorder()
s.handlePreviewCDNSource(recorder, req)
if recorder.Code != http.StatusBadGateway {
t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String())
}
}
func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) {
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1")
m := store.NewMemory()
m.SeedDemo()
tenant, module, _, _, _ := m.DemoIDs()
var calls atomic.Int32
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
w.WriteHeader(http.StatusNotFound)
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
w.(http.Flusher).Flush()
// No EOF: an unbounded drain would wait until the caller deadline.
<-r.Context().Done()
}))
defer upstream.Close()
s := &Server{store: m, cdnHTTP: upstream.Client()}
request := func() *http.Request {
r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`))
r.SetPathValue("module_id", module)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
}
first := httptest.NewRecorder()
firstRequest := request()
ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second)
defer cancel()
s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx))
if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil {
t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err())
}
second := httptest.NewRecorder()
s.handlePreviewCDNSource(second, request())
if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" {
t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load())
}
}
+1
View File
@@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler {
} }
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id") w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After")
if r.Method == http.MethodOptions { if r.Method == http.MethodOptions {
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
return return
+56
View File
@@ -0,0 +1,56 @@
package httpapi
import (
"errors"
"net/http"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool {
j, found, err := s.jobs.ReplayContext(r.Context(), in)
if err != nil {
writeJobEnqueueError(w, err)
return true
}
if !found {
return false
}
w.Header().Set("Location", "/v1/jobs/"+j.ID)
snap := j.Snapshot()
body := map[string]any{"job_id": j.ID, "status": snap["status"]}
if j.Kind == jobs.KindTenantRefresh {
ids := snap["meta"].(map[string]any)["module_ids"]
n := 0
switch v := ids.(type) {
case []any:
n = len(v)
case []string:
n = len(v)
}
body["module_ids"] = ids
body["modules_due"] = n
}
writeJSON(w, http.StatusAccepted, body)
return true
}
func writeJobEnqueueError(w http.ResponseWriter, err error) {
switch {
case errors.Is(err, store.ErrJobConflict):
writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request")
case errors.Is(err, store.ErrJobQueueFull):
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full")
default:
writeInternalError(w, "job_enqueue", err)
}
}
func writeJobReadError(w http.ResponseWriter, err error) {
if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
return
}
writeInternalError(w, "job_read", err)
}
+99
View File
@@ -0,0 +1,99 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
type unavailableJobQueue struct{ store.JobQueue }
func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) {
return store.JobRecord{}, false, errors.New("database unavailable")
}
func TestJobEnqueueFailureNeverAccepts(t *testing.T) {
r := jobs.NewRegistry(nil)
r.UseDurableQueue(unavailableJobQueue{})
s := &Server{jobs: r}
req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"}))
w := httptest.NewRecorder()
s.handleBirdReload(w, req)
if w.Code == http.StatusAccepted || w.Code < 500 {
t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String())
}
}
func TestAutomaticEnqueueFailureIsVisible(t *testing.T) {
r := jobs.NewRegistry(nil)
r.UseDurableQueue(unavailableJobQueue{})
s := &Server{jobs: r}
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil)
s.enqueuePeerReconcile(w, req, "tenant", "peer_create")
if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" {
t.Fatalf("scheduling failure hidden: %q", got)
}
}
type selectorBackend struct {
store.Backend
modules []*store.Module
reads int
}
func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules }
func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) {
m := store.NewMemory()
b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}}
r := jobs.NewRegistry(nil)
r.UseDurableQueue(m)
s := &Server{store: b, jobs: r}
request := func() *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`))
req.Header.Set("Idempotency-Key", "same-request")
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"}))
w := httptest.NewRecorder()
s.handleTenantRefresh(w, req)
return w
}
first := request()
if first.Code != http.StatusAccepted {
t.Fatalf("first=%d %s", first.Code, first.Body.String())
}
var accepted map[string]any
if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil {
t.Fatal(err)
}
claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim %+v %v", claimed, err)
}
claimed[0].Status = "succeeded"
if err = m.FinishJob(context.Background(), claimed[0]); err != nil {
t.Fatal(err)
}
b.modules = nil
reads := b.reads
replay := request()
if replay.Code != http.StatusAccepted {
t.Fatalf("replay=%d %s", replay.Code, replay.Body.String())
}
var got map[string]any
if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil {
t.Fatal(err)
}
if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads {
t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads)
}
}
+114
View File
@@ -0,0 +1,114 @@
package httpapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"evobgp/internal/jobs"
"evobgp/internal/store"
)
func TestJobReportDurableProgressAndTenantScope(t *testing.T) {
ctx := context.Background()
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply})
if err != nil {
t.Fatal(err)
}
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim: %v %v", claimed, err)
}
f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken}
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil {
t.Fatal(err)
}
// A fresh registry has no in-process job or progress metadata.
r := jobs.NewRegistry(nil)
r.UseDurableQueue(m)
s := &Server{store: m, jobs: r}
request := func(tenantID, jobID string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil)
req.SetPathValue("job_id", jobID)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"}))
w := httptest.NewRecorder()
s.handleGetJobReport(w, req)
return w
}
check := func(id string) {
t.Helper()
w := request(tenant, id)
if w.Code != http.StatusOK {
t.Fatalf("report: %d %s", w.Code, w.Body.String())
}
var body struct {
Progress []struct {
State, Confirmation string
JobID string `json:"job_id"`
} `json:"deployment_progress"`
}
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
t.Fatal(err)
}
if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" {
t.Fatalf("lost persisted progress: %s", w.Body.String())
}
}
check(j.ID)
recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
if err != nil {
t.Fatal(err)
}
check(recovery.ID)
// A later explicit completion releases the delivery reservation. Once recovery
// has its own persisted row, its report must not show the parent's old state.
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
t.Fatal(err)
}
recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute)
if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID {
t.Fatalf("recovery claim: %v %v", recoveryRows, err)
}
rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken}
rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision)
if err != nil {
t.Fatal(err)
}
if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil {
t.Fatal(err)
}
w := request(tenant, recovery.ID)
var report struct {
Progress []struct {
State string
JobID string `json:"job_id"`
} `json:"deployment_progress"`
}
if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil {
t.Fatal(err)
}
if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" {
t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String())
}
if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound {
t.Fatalf("cross-tenant report: %d", w.Code)
}
foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
if err != nil {
t.Fatal(err)
}
if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound {
t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String())
}
}
@@ -0,0 +1,89 @@
package httpapi
import (
"context"
"crypto/ed25519"
"encoding/base64"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"net/http/httptest"
"evobgp/internal/bundle"
)
// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old
// CLI against today's HTTP handlers. It never calls apply or touches host BIRD.
func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) {
binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN")
if binary == "" {
t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set")
}
binary, err := filepath.Abs(binary)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(binary); err != nil {
t.Fatal(err)
}
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer srv.Close()
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node")
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(srv.Handler())
defer server.Close()
run := func(wantSuccess bool, args ...string) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput()
if ctx.Err() != nil {
t.Fatalf("legacy CLI timed out: %v", ctx.Err())
}
if wantSuccess && err != nil {
t.Fatalf("legacy CLI failed: %v: %s", err, output)
}
if !wantSuccess && err == nil {
t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output)
}
}
path := filepath.Join(t.TempDir(), "current-bundle.tar.gz")
run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path)
run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
if err != nil {
t.Fatal(err)
}
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker {
t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err)
}
// The archive is valid, but verification with an unrelated key must fail.
run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize))
for _, token := range []string{"invalid-node", "other-node"} {
denied := filepath.Join(t.TempDir(), "denied.tar.gz")
run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied)
if _, err := os.Stat(denied); !os.IsNotExist(err) {
t.Fatalf("unauthorized CLI wrote a bundle: %v", err)
}
}
// A damaged payload must not pass the old verifier with the correct key.
if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil {
t.Fatal(err)
}
run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
}
+310
View File
@@ -0,0 +1,310 @@
package httpapi
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/binary"
"encoding/hex"
"errors"
"fmt"
"sort"
"strings"
"sync"
"evobgp/internal/bundle"
"evobgp/internal/pipeline"
"evobgp/internal/store"
)
const (
// nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained.
nodeBundleCacheMaxEntries = 32
// nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB).
nodeBundleCacheMaxBytes = 64 << 20
)
// errNodeBundleRevisionMissing is returned when the revision cannot be loaded.
// It is not cached: the next pull tries again.
var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found")
// nodeBundleOpError carries the writeInternalError operation name for a failed build.
type nodeBundleOpError struct {
op string
err error
}
func (e *nodeBundleOpError) Error() string {
if e == nil || e.err == nil {
return "httpapi: node bundle"
}
return fmt.Sprintf("httpapi: %s: %v", e.op, e.err)
}
func (e *nodeBundleOpError) Unwrap() error { return e.err }
// nodeBundleBuilder produces the gzip body served to evobgp-node.
// The default implementation is bundle.BuildGzippedTar, whose archive contains
// manifest.json and bundle.sig. Tests replace it to count rebuilds.
type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error)
// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the
// build that filled the entry, signature included. Errors are not stored.
// Concurrent loads of one key share a single build.
type nodeBundleCache struct {
mu sync.Mutex
maxN int
maxBytes int
total int
items map[string]*nodeBundleEntry
lru []string // index 0 is the oldest idle entry
flights map[string]*nodeBundleFlight
}
type nodeBundleEntry struct {
body []byte
}
type nodeBundleFlight struct {
done chan struct{}
body []byte
err error
joined int
}
func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache {
return &nodeBundleCache{
maxN: maxEntries,
maxBytes: maxBytes,
items: make(map[string]*nodeBundleEntry),
flights: make(map[string]*nodeBundleFlight),
}
}
// nodeBundleCacheKey identifies one servable bundle.
// overlayFP is a digest of the speaker overlay output (router id / source addresses,
// node IPv4 fallback, peer and discovery render). Revision content itself is immutable,
// so the revision id stands in for the shared prefix files.
func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string {
var b strings.Builder
b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3)
b.WriteString(tenantID)
b.WriteByte(0)
b.WriteString(revisionID)
b.WriteByte(0)
b.WriteString(speakerID)
b.WriteByte(0)
b.WriteString(overlayFP)
return b.String()
}
// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle
// would apply on top of an immutable revision. Empty base fragments are enough:
// bird.conf and the peers fragment are fully re-rendered from tenant settings,
// speaker source addresses, and the live peer set. Revision file bytes are not
// part of this digest.
func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) {
frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil)
if err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
keys := make([]string, 0, len(frags))
for k := range frags {
keys = append(keys, k)
}
sort.Strings(keys)
h := sha256.New()
for _, k := range keys {
if err := hashWriteString(h, k); err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
if err := hashWriteString(h, frags[k]); err != nil {
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
}
}
return hex.EncodeToString(h.Sum(nil)), nil
}
func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], uint64(len(s)))
if _, err := h.Write(buf[:]); err != nil {
return err
}
_, err := h.Write([]byte(s))
return err
}
// do returns the cached body or runs build once for key.
// build's bool is false when the result must not be stored (failure is also not stored).
// Callers must not mutate the returned slice.
func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) {
if c == nil {
return nil, errors.New("httpapi: nil node bundle cache")
}
if ctx == nil {
ctx = context.Background()
}
c.mu.Lock()
if e, ok := c.items[key]; ok {
c.touchLocked(key)
body = e.body
c.mu.Unlock()
return body, nil
}
if f, ok := c.flights[key]; ok {
f.joined++
c.mu.Unlock()
return waitNodeBundleFlight(ctx, f)
}
if err = ctx.Err(); err != nil {
c.mu.Unlock()
return nil, err
}
f := &nodeBundleFlight{done: make(chan struct{})}
c.flights[key] = f
c.mu.Unlock()
var cacheOK bool
var finished bool
finish := func() {
if finished {
return
}
finished = true
out := body
c.mu.Lock()
delete(c.flights, key)
if err == nil && cacheOK {
if stored, ok := c.storeLocked(key, out); ok {
out = stored
}
}
f.body = out
f.err = err
close(f.done)
c.mu.Unlock()
body = out
}
defer func() {
if rec := recover(); rec != nil {
body = nil
cacheOK = false
err = fmt.Errorf("httpapi: node bundle build: %v", rec)
}
finish()
}()
// Detach from the leader's cancel so one disconnected agent does not fail the shared build.
body, cacheOK, err = build(context.WithoutCancel(ctx))
return
}
func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) {
select {
case <-f.done:
return f.body, f.err
case <-ctx.Done():
select {
case <-f.done:
return f.body, f.err
default:
return nil, ctx.Err()
}
}
}
// storeLocked retains body when it fits under both caps, evicting the oldest idle
// entries first. body is copied. The stored slice is returned.
func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) {
n := len(body)
if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes {
return nil, false
}
if e, ok := c.items[key]; ok {
c.total -= len(e.body)
delete(c.items, key)
c.removeLRULocked(key)
}
for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) {
if len(c.lru) == 0 {
return nil, false
}
c.deleteOldestLocked()
}
if len(c.items) >= c.maxN || c.total+n > c.maxBytes {
return nil, false
}
stored := make([]byte, n)
copy(stored, body)
c.items[key] = &nodeBundleEntry{body: stored}
c.total += n
c.lru = append(c.lru, key)
return stored, true
}
func (c *nodeBundleCache) touchLocked(key string) {
c.removeLRULocked(key)
c.lru = append(c.lru, key)
}
func (c *nodeBundleCache) deleteOldestLocked() {
if len(c.lru) == 0 {
return
}
key := c.lru[0]
c.lru = c.lru[1:]
if e, ok := c.items[key]; ok {
c.total -= len(e.body)
delete(c.items, key)
}
}
func (c *nodeBundleCache) removeLRULocked(key string) {
for i, k := range c.lru {
if k != key {
continue
}
copy(c.lru[i:], c.lru[i+1:])
c.lru = c.lru[:len(c.lru)-1]
return
}
}
func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) {
if s != nil && s.nodeBundleBuild != nil {
return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv)
}
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv)
}
// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip.
// The bool is false when overlay inputs moved during the load, so the caller must not
// cache the body under the key captured before the load. Failed loads are not cacheable.
func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) {
if err := ctx.Err(); err != nil {
return nil, false, err
}
fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
rev, err := s.store.GetRevision(tenantID, revisionID)
if err != nil {
return nil, false, errNodeBundleRevisionMissing
}
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
if err != nil {
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
}
body, err := s.buildNodeBundle(revisionID, speakerID, overlaid)
if err != nil {
return nil, false, &nodeBundleOpError{op: "internal", err: err}
}
cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0
return body, cacheable, nil
}
+791
View File
@@ -0,0 +1,791 @@
package httpapi
import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"crypto/ed25519"
"encoding/base64"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"evobgp/internal/bundle"
"evobgp/internal/signing"
"evobgp/internal/store"
)
func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) {
a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay")
if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") {
t.Fatal("tenants collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") {
t.Fatal("revisions collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") {
t.Fatal("speakers collided")
}
if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") {
t.Fatal("overlay inputs collided")
}
}
func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) {
c := newNodeBundleCache(8, 1<<20)
var builds atomic.Int32
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("signed-body"), true, nil
})
if err != nil {
t.Fatal(err)
}
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("other"), true, nil
})
if err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
}
if !bytes.Equal(got, []byte("signed-body")) {
t.Fatalf("hit body = %q", got)
}
t.Run("singleflight", func(t *testing.T) {
c := newNodeBundleCache(8, 1<<20)
var builds atomic.Int32
release := make(chan struct{})
const n = 8
var wg sync.WaitGroup
bodies := make([][]byte, n)
errs := make([]error, n)
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
<-release
return []byte("signed-bundle"), true, nil
})
}(i)
}
deadline := time.Now().Add(2 * time.Second)
for {
c.mu.Lock()
joined := 0
if f := c.flights["same"]; f != nil {
joined = f.joined
}
c.mu.Unlock()
if builds.Load() > 1 {
close(release)
wg.Wait()
t.Fatalf("builds = %d before followers joined", builds.Load())
}
if builds.Load() == 1 && joined == n-1 {
break
}
if time.Now().After(deadline) {
close(release)
wg.Wait()
t.Fatalf("builds=%d joined=%d", builds.Load(), joined)
}
time.Sleep(time.Millisecond)
}
close(release)
wg.Wait()
if builds.Load() != 1 {
t.Fatalf("builds = %d, want 1", builds.Load())
}
for i := 0; i < n; i++ {
if errs[i] != nil {
t.Fatalf("waiter %d: %v", i, errs[i])
}
if !bytes.Equal(bodies[i], []byte("signed-bundle")) {
t.Fatalf("waiter %d body = %q", i, bodies[i])
}
}
})
t.Run("canceled waiter does not drop the build", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
release := make(chan struct{})
var builds atomic.Int32
leaderErr := make(chan error, 1)
go func() {
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
<-release
return []byte("body"), true, nil
})
leaderErr <- err
}()
deadline := time.Now().Add(2 * time.Second)
for {
c.mu.Lock()
_, ok := c.flights["k"]
c.mu.Unlock()
if ok || time.Now().After(deadline) {
break
}
time.Sleep(time.Millisecond)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("nope"), true, nil
})
if !errors.Is(err, context.Canceled) {
t.Fatalf("follower err = %v", err)
}
close(release)
if err := <-leaderErr; err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatalf("builds = %d", builds.Load())
}
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
builds.Add(1)
return []byte("other"), true, nil
})
if err != nil {
t.Fatal(err)
}
if builds.Load() != 1 {
t.Fatal("successful build was not cached after a canceled waiter")
}
})
t.Run("errors are not cached", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
calls := 0
boom := errors.New("httpapi: bundle build failed")
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return nil, true, boom
})
if !errors.Is(err, boom) {
t.Fatalf("err = %v", err)
}
_, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) {
calls++
return nil, false, errNodeBundleRevisionMissing
})
if !errors.Is(err, errNodeBundleRevisionMissing) {
t.Fatalf("err = %v", err)
}
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("ok"), true, nil
})
if err != nil {
t.Fatal(err)
}
if string(got) != "ok" || calls != 3 {
t.Fatalf("calls=%d body=%q", calls, got)
}
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("other"), true, nil
})
if err != nil || calls != 3 {
t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err)
}
})
t.Run("unstorable success is not cached", func(t *testing.T) {
c := newNodeBundleCache(4, 1024)
calls := 0
for i := 0; i < 2; i++ {
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
calls++
return []byte("fresh"), false, nil
})
if err != nil || string(got) != "fresh" {
t.Fatalf("i=%d err=%v body=%q", i, err, got)
}
}
if calls != 2 {
t.Fatalf("calls = %d, want 2", calls)
}
})
}
func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) {
t.Run("entry cap evicts oldest idle", func(t *testing.T) {
c := newNodeBundleCache(2, 100)
put := func(key, body string) {
t.Helper()
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
return []byte(body), true, nil
})
if err != nil || string(got) != body {
t.Fatalf("put %s: %v %q", key, err, got)
}
}
put("a", "A")
put("b", "B")
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
t.Fatal("recent entry was rebuilt")
return nil, false, nil
}); err != nil {
t.Fatal(err)
}
put("c", "C")
assertBundleCacheAccounting(t, c)
if _, ok := c.items["b"]; ok {
t.Fatal("idle entry b was kept")
}
if _, ok := c.items["a"]; !ok || c.items["c"] == nil {
t.Fatalf("kept %v", cacheKeys(c))
}
if len(c.items) > 2 {
t.Fatalf("entry cap exceeded: %d", len(c.items))
}
})
t.Run("byte cap", func(t *testing.T) {
c := newNodeBundleCache(8, 10)
put := func(key string, n int) {
t.Helper()
body := bytes.Repeat([]byte("x"), n)
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
return body, true, nil
})
if err != nil || !bytes.Equal(got, body) {
t.Fatalf("put %s: %v len=%d", key, err, len(got))
}
assertBundleCacheAccounting(t, c)
}
put("a", 4)
put("b", 4)
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
t.Fatal("rebuilt a")
return nil, false, nil
}); err != nil {
t.Fatal(err)
}
put("c", 4)
assertBundleCacheAccounting(t, c)
if _, ok := c.items["b"]; ok {
t.Fatal("byte cap kept idle entry b")
}
if len(c.items) != 2 || c.total != 8 {
t.Fatalf("n=%d total=%d", len(c.items), c.total)
}
put("d", 6)
put("e", 6)
assertBundleCacheAccounting(t, c)
if len(c.items) != 1 || c.total != 6 {
t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c))
}
})
t.Run("entry larger than the byte cap is not stored", func(t *testing.T) {
c := newNodeBundleCache(4, 8)
calls := 0
big := bytes.Repeat([]byte("y"), 9)
for i := 0; i < 2; i++ {
got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) {
calls++
return big, true, nil
})
if err != nil || !bytes.Equal(got, big) {
t.Fatalf("i=%d err=%v", i, err)
}
}
if calls != 2 {
t.Fatalf("oversize body was cached, calls=%d", calls)
}
if len(c.items) != 0 || c.total != 0 {
t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total)
}
})
}
func TestNodeBundleOverlayFingerprintInputs(t *testing.T) {
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, rev, speaker := m.DemoIDs()
base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if base != again {
t.Fatal("fingerprint changed without an overlay input change")
}
sp, err := m.GetSpeaker(tenant, speaker)
if err != nil {
t.Fatal(err)
}
dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{
LastDispatchAt: "2026-01-01T00:00:00Z",
LastDispatchStatus: "ok",
SyncStatus: "synced",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got)
}
peers := m.ListPeers(tenant)
if len(peers) == 0 {
t.Fatal("expected demo peers")
}
state := "Active"
if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
t.Fatalf("session state changed fingerprint: %v %s", err, got)
}
if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil {
t.Fatal(err)
}
routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if routed == base {
t.Fatal("router id setting did not change the overlay fingerprint")
}
nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil {
t.Fatal(err)
}
nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if nodeFP == routed {
t.Fatal("node ipv4 did not change the overlay fingerprint")
}
sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.9",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil {
t.Fatal(err)
}
sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if sourceFP == nodeFP {
t.Fatal("source ipv4 did not change the overlay fingerprint")
}
nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.8",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil {
t.Fatal(err)
}
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP {
t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP)
}
v6 := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.8",
BirdBgpSourceIPv4: "203.0.113.40",
BirdBgpSourceIPv6: "2001:db8::40",
})
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil {
t.Fatal(err)
}
v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
// Source v6 changes local v6. When no peer policy overrides it, the rendered
// peers fragment can stay the same. A difference means the overlay emitted it;
// equality means a rebuild would not change the bundle.
_ = v6FP
if _, err := m.CreatePeer(tenant, &store.BGPPeer{
Name: "cache-extra",
Neighbor: "203.0.113.50",
RemoteASN: 64512,
Enabled: true,
}); err != nil {
t.Fatal(err)
}
withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
if err != nil {
t.Fatal(err)
}
if withPeer == v6FP {
t.Fatal("added peer did not change the overlay fingerprint")
}
sp2, err := m.CreateSpeaker(tenant, &store.Speaker{
Role: "replica",
Endpoint: "https://203.0.113.2",
MetaJSON: v6,
})
if err != nil {
t.Fatal(err)
}
other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev)
if err != nil {
t.Fatal(err)
}
if other == withPeer {
t.Fatal("speakers with the same addresses shared an overlay fingerprint")
}
otherRev, err := m.CreateRollbackRevision(tenant, rev)
if err != nil {
t.Fatal(err)
}
revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev)
if err != nil {
t.Fatal(err)
}
if revFP == withPeer {
t.Fatal("revision id did not change the overlay fingerprint")
}
}
func TestHandleNodeBundleCache(t *testing.T) {
srv, ts, ids := startNodeBundleServer(t)
if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes {
t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes)
}
var builds atomic.Int32
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
builds.Add(1)
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
}
client := ts.Client()
status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099")
if status != http.StatusNotFound {
t.Fatalf("missing revision status %d: %s", status, raw)
}
if builds.Load() != 0 {
t.Fatal("missing revision built a bundle")
}
body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 1 {
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
}
if !bytes.Equal(body1, body2) {
t.Fatal("cache hit was not byte-for-byte identical")
}
verifyNodeBundle(t, srv, body2)
sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{
Role: "replica",
Endpoint: "https://203.0.113.2",
MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}),
})
if err != nil {
t.Fatal(err)
}
bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev)
if builds.Load() != 2 {
t.Fatalf("second speaker builds=%d", builds.Load())
}
if bytes.Equal(body1, bodyB) {
t.Fatal("different speakers returned the same bundle")
}
if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") {
t.Fatal("speaker B bundle missing its router id")
}
body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 2 || !bytes.Equal(body1, body3) {
t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3))
}
meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
t.Fatal(err)
}
body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 3 {
t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load())
}
if bytes.Equal(body1, body4) {
t.Fatal("changed overlay input reused the previous bundle")
}
conf := bundleFile(t, body4, "bird.conf")
if !strings.Contains(conf, "198.51.100.9") {
t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf)
}
verifyNodeBundle(t, srv, body4)
body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 3 || !bytes.Equal(body4, body5) {
t.Fatal("rebuilt bundle was not cached")
}
source := store.SpeakerMetaJSON(store.SpeakerMeta{
NodeIPv4: "198.51.100.9",
BirdBgpSourceIPv4: "203.0.113.40",
})
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil {
t.Fatal(err)
}
body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 4 {
t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load())
}
conf = bundleFile(t, body6, "bird.conf")
if !strings.Contains(conf, "203.0.113.40") {
t.Fatalf("source ipv4 not applied: %s", conf)
}
verifyNodeBundle(t, srv, body6)
otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev)
if err != nil {
t.Fatal(err)
}
bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev)
if builds.Load() != 5 {
t.Fatalf("other revision builds=%d", builds.Load())
}
if bytes.Equal(body6, bodyRev) {
t.Fatal("revisions shared a bundle body")
}
cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
if builds.Load() != 5 || !bytes.Equal(body6, cached) {
t.Fatal("original revision was evicted or rebuilt")
}
}
func TestHandleNodeBundleSingleflight(t *testing.T) {
srv, ts, ids := startNodeBundleServer(t)
var builds atomic.Int32
release := make(chan struct{})
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
builds.Add(1)
<-release
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
}
client := ts.Client()
const n = 2
bodies := make([][]byte, n)
errs := make([]error, n)
var wg sync.WaitGroup
wg.Add(n)
for i := 0; i < n; i++ {
go func(i int) {
defer wg.Done()
bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev)
}(i)
}
deadline := time.Now().Add(3 * time.Second)
joined := 0
for {
if builds.Load() > 1 {
close(release)
wg.Wait()
t.Fatalf("concurrent pulls built %d times", builds.Load())
}
srv.bundleCache.mu.Lock()
joined = 0
for _, f := range srv.bundleCache.flights {
joined += f.joined
}
srv.bundleCache.mu.Unlock()
if builds.Load() == 1 && joined >= 1 {
break
}
if time.Now().After(deadline) {
close(release)
wg.Wait()
t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined)
}
time.Sleep(time.Millisecond)
}
close(release)
wg.Wait()
if builds.Load() != 1 {
t.Fatalf("builds = %d, want 1", builds.Load())
}
for i := 0; i < n; i++ {
if errs[i] != nil {
t.Fatal(errs[i])
}
}
if !bytes.Equal(bodies[0], bodies[1]) {
t.Fatal("singleflight returned different bodies")
}
verifyNodeBundle(t, srv, bodies[0])
}
type nodeBundleIDs struct {
tenant, rev, speaker string
}
func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) {
t.Helper()
srv, err := New(Options{
InsecureDev: true,
SeedDemo: true,
BundleSeedHex: testBundleSeed,
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(srv.Close)
tenant, _, _, rev, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node")
ts := httptest.NewServer(srv.Handler())
t.Cleanup(ts.Close)
return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker}
}
func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte {
t.Helper()
status, body := pullNodeBundle(t, client, base, speaker, rev)
if status != http.StatusOK {
t.Fatalf("status %d: %s", status, body)
}
return body
}
func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
if err != nil {
t.Fatal(err)
}
req.Header.Set("Authorization", "Bearer nodekey")
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return resp.StatusCode, body
}
func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) {
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
if err != nil {
return nil, err
}
req.Header.Set("Authorization", "Bearer nodekey")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New(resp.Status + ": " + string(body))
}
return body, nil
}
func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) {
t.Helper()
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
if err != nil {
t.Fatal(err)
}
if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil {
t.Fatal(err)
}
}
func bundleFile(t *testing.T, gz []byte, name string) string {
t.Helper()
gr, err := gzip.NewReader(bytes.NewReader(gz))
if err != nil {
t.Fatal(err)
}
defer func() { _ = gr.Close() }()
tr := tar.NewReader(gr)
for {
hdr, err := tr.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
t.Fatal(err)
}
if hdr.Name != name {
continue
}
var b strings.Builder
if _, err := io.Copy(&b, tr); err != nil {
t.Fatal(err)
}
return b.String()
}
t.Fatalf("archive missing %s", name)
return ""
}
func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) {
t.Helper()
if len(c.items) != len(c.lru) {
t.Fatalf("items %d lru %d", len(c.items), len(c.lru))
}
sum := 0
seen := make(map[string]bool, len(c.lru))
for _, k := range c.lru {
e, ok := c.items[k]
if !ok {
t.Fatalf("lru key %q missing from items", k)
}
if seen[k] {
t.Fatalf("duplicate lru key %q", k)
}
seen[k] = true
sum += len(e.body)
}
if sum != c.total {
t.Fatalf("total=%d sum=%d", c.total, sum)
}
if c.maxN > 0 && len(c.items) > c.maxN {
t.Fatalf("entries %d over cap %d", len(c.items), c.maxN)
}
if c.maxBytes > 0 && c.total > c.maxBytes {
t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes)
}
}
func cacheKeys(c *nodeBundleCache) []string {
keys := make([]string, 0, len(c.items))
for k := range c.items {
keys = append(keys, k)
}
return keys
}
@@ -0,0 +1,98 @@
package httpapi
import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"io"
"net/http"
"net/http/httptest"
"testing"
"evobgp/internal/bundle"
)
func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) {
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer srv.Close()
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node")
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
t.Fatal(err)
}
handler := srv.Handler()
call := func(path, token string) (int, []byte, string) {
t.Helper()
req, err := http.NewRequest(http.MethodGet, path, nil)
if err != nil {
t.Fatal(err)
}
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
resp := w.Result()
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return resp.StatusCode, body, resp.Header.Get("Content-Type")
}
latestPath := "/v1/speakers/" + speaker + "/revisions/latest"
bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision
for _, path := range []string{latestPath, bundlePath} {
for _, tc := range []struct {
token string
want int
}{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} {
status, _, _ := call(path, tc.token)
if status != tc.want {
t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want)
}
}
}
status, raw, _ := call(latestPath, "legacy-node")
if status != http.StatusOK {
t.Fatalf("legacy latest: status %d: %s", status, raw)
}
var latest struct {
RevisionID string `json:"revision_id"`
PublishedAt string `json:"published_at"`
}
if err := json.Unmarshal(raw, &latest); err != nil {
t.Fatal(err)
}
if latest.RevisionID != revision || latest.PublishedAt == "" {
t.Fatalf("legacy latest response: %+v", latest)
}
status, raw, contentType := call("/v1/bundle/signing-public-key", "operator")
if status != http.StatusOK || contentType != "application/json" {
t.Fatalf("public key status %d, type %q: %s", status, contentType, raw)
}
var key struct {
PublicKeyBase64 string `json:"public_key_base64"`
}
if err := json.Unmarshal(raw, &key); err != nil {
t.Fatal(err)
}
pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64)
if err != nil || len(pub) != ed25519.PublicKeySize {
t.Fatalf("legacy public key: len=%d err=%v", len(pub), err)
}
status, raw, contentType = call(bundlePath, "legacy-node")
if status != http.StatusOK || contentType != "application/gzip" {
t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw)
}
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
if err != nil {
t.Fatalf("legacy signed bundle verification: %v", err)
}
if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 {
t.Fatalf("legacy bundle manifest: %+v", verified.Manifest)
}
}
+105
View File
@@ -0,0 +1,105 @@
package httpapi
import (
"net/http"
"os"
"strconv"
"sync"
"time"
)
type expensiveOperation int
const (
lookupOperation expensiveOperation = iota
cdnPreviewOperation
revisionPreviewOperation
)
type tenantWork struct {
start time.Time
counts [3]int
active int
}
// operationLimiter is deliberately local to one API process. It never queues
// waiters or evicts active tenants to admit new identities.
type operationLimiter struct {
mu sync.Mutex
tenants map[string]*tenantWork
active, maxActive, maxTenantActive, maxTenants int
rates [3]int
nextSweep time.Time
}
func newOperationLimiter() *operationLimiter {
return &operationLimiter{
tenants: make(map[string]*tenantWork), maxTenants: 4096,
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
}
}
func operationEnvLimit(name string, fallback, maximum int) int {
n, err := strconv.Atoi(os.Getenv(name))
if err != nil || n < 1 || n > maximum {
return fallback
}
return n
}
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
l.mu.Lock()
defer l.mu.Unlock()
if !now.Before(l.nextSweep) {
for key, entry := range l.tenants {
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
delete(l.tenants, key)
}
}
l.nextSweep = now.Add(time.Second)
}
if l.active >= l.maxActive {
return nil, 1
}
entry := l.tenants[tenant]
if entry == nil {
if len(l.tenants) >= l.maxTenants {
return nil, 1
}
entry = &tenantWork{start: now}
l.tenants[tenant] = entry
}
if !now.Before(entry.start.Add(time.Minute)) {
entry.start = now
entry.counts = [3]int{}
}
if entry.active >= l.maxTenantActive {
return nil, 1
}
if entry.counts[op] >= l.rates[op] {
wait := entry.start.Add(time.Minute).Sub(now)
return nil, max(1, int((wait+time.Second-1)/time.Second))
}
entry.counts[op]++
entry.active++
l.active++
var once sync.Once
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
}
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
if r.Context().Err() != nil {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
return nil, false
}
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
if release == nil {
w.Header().Set("Retry-After", strconv.Itoa(retry))
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
return nil, false
}
return release, true
}
+143
View File
@@ -0,0 +1,143 @@
package httpapi
import (
"context"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
"evobgp/internal/store"
)
func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) {
l := newOperationLimiter()
l.maxActive = 2
l.maxTenantActive = 1
l.rates = [3]int{1, 2, 2}
now := time.Now()
a, _ := l.acquire("a", lookupOperation, now)
if a == nil {
t.Fatal("first denied")
}
if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil {
t.Fatal("tenant concurrency bypass")
}
b, _ := l.acquire("b", revisionPreviewOperation, now)
if b == nil {
t.Fatal("other tenant denied")
}
if r, _ := l.acquire("c", lookupOperation, now); r != nil {
t.Fatal("global concurrency bypass")
}
a()
a()
b()
if l.active != 0 {
t.Fatal("release not idempotent")
}
if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 {
t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry)
}
c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second))
if c == nil {
t.Fatal("independent operation denied")
}
c()
d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute))
if d == nil {
t.Fatal("expired window not reset")
}
d()
}
func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) {
l := newOperationLimiter()
l.maxTenants = 1
now := time.Now()
a, _ := l.acquire("a", lookupOperation, now)
if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil {
t.Fatal("active entry evicted")
}
a()
b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute))
if b == nil || len(l.tenants) != 1 {
t.Fatal("idle expiry did not free bounded state")
}
b()
}
func TestOperationConcurrentAdmissionBound(t *testing.T) {
l := newOperationLimiter()
l.maxActive = 4
l.maxTenantActive = 4
l.rates = [3]int{100, 100, 100}
results := make(chan func(), 32)
var wg sync.WaitGroup
for range 32 {
wg.Add(1)
go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }()
}
wg.Wait()
close(results)
accepted := 0
for r := range results {
if r != nil {
accepted++
r()
}
}
if accepted != 4 || l.active != 0 {
t.Fatalf("accepted=%d active=%d", accepted, l.active)
}
}
type waitingRevisionStore struct {
store.Backend
entered chan struct{}
unblock chan struct{}
}
func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) {
s.entered <- struct{}{}
<-s.unblock
return s.Backend.GetRevision(tenant, id)
}
func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) {
t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1")
t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30")
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, _ := m.DemoIDs()
entered := make(chan struct{}, 2)
unblock := make(chan struct{})
s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}}
request := func() *http.Request {
r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil)
r.SetPathValue("revision_id", revision)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
}
ctx, cancel := context.WithCancel(request().Context())
defer cancel()
first := httptest.NewRecorder()
done := make(chan struct{})
go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }()
<-entered
cancel() // Backend deliberately ignores cancellation: capacity stays reserved.
second := httptest.NewRecorder()
s.handleRevisionPreview(second, request())
if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" {
close(unblock)
<-done
t.Fatalf("quota: %d %s", second.Code, second.Body.String())
}
close(unblock)
<-done
third := httptest.NewRecorder()
s.handleRevisionPreview(third, request())
if third.Code != http.StatusOK {
t.Fatalf("release failed: %d %s", third.Code, third.Body.String())
}
}
+1 -47
View File
@@ -6,7 +6,6 @@ import (
"strings" "strings"
"time" "time"
"evobgp/internal/birdfmt"
"evobgp/internal/store" "evobgp/internal/store"
) )
@@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any {
return m return m
} }
// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions.
func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) {
if s.store == nil || tenantID == "" {
return
}
knownNeighbors := map[string]struct{}{}
for _, p := range s.store.ListPeers(tenantID) {
if p == nil {
continue
}
if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok {
knownNeighbors[n] = struct{}{}
}
}
now := time.Now().UTC()
for _, v := range views {
for _, sess := range v.Sessions {
if !birdfmt.IsDynamicDiscoverySession(sess.Name) {
continue
}
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
if !ok {
continue
}
if _, known := knownNeighbors[neighbor]; known {
continue
}
_, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{
SpeakerID: v.SpeakerID,
NeighborID: strings.TrimSpace(sess.NeighborID),
Neighbor: neighbor,
RemoteASN: sess.NeighborAS,
ProtocolName: sess.Name,
SessionState: sess.State,
SeenAt: now,
})
}
}
}
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) { func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context()) a, ok := authFromContext(r.Context())
if !ok { if !ok {
@@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques
if status == "" { if status == "" {
status = store.PeerDiscoveryPending status = store.PeerDiscoveryPending
} }
// Refresh live discovery when listing pending.
if status == store.PeerDiscoveryPending || status == "all" {
views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true)
s.syncPeerDiscoveriesFromLive(a.TenantID, views)
}
listStatus := status listStatus := status
if status == "all" { if status == "all" {
listStatus = "" listStatus = ""
@@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque
writeStoreErr(w, err) writeStoreErr(w, err)
return return
} }
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve") s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve")
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{ s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
"peer_id": peer.ID, "peer_id": peer.ID,
"discovery_id": disc.ID, "discovery_id": disc.ID,
+164 -95
View File
@@ -1,47 +1,98 @@
package httpapi package httpapi
import ( import (
"context"
"net/netip" "net/netip"
"os"
"strings" "strings"
"sync"
"time" "time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/nodedispatch"
"evobgp/internal/store" "evobgp/internal/store"
) )
const peerLiveCacheTTL = 15 * time.Second
type speakerBGPLive struct { type speakerBGPLive struct {
SpeakerID string SpeakerID string
Label string Label string
Sessions []birdfmt.BGPSession Sessions []birdfmt.BGPSession
Error string Error string
Polled bool
ObservedAt time.Time
ReceivedAt time.Time
LastSuccessAt time.Time
Source string
ObservationOverride *liveObservation
} }
type peerLiveCacheEntry struct { type liveObservation struct {
at time.Time ObservedAt *string `json:"observed_at"`
views []speakerBGPLive ReceivedAt *string `json:"received_at"`
LastSuccessAt *string `json:"last_success_at"`
ExpiresAt *string `json:"expires_at,omitempty"`
Source string `json:"source"`
Quality string `json:"quality"`
ErrorCode string `json:"error_code,omitempty"`
} }
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry func liveTime(t time.Time) *string {
if t.IsZero() {
return nil
}
v := t.UTC().Format(time.RFC3339Nano)
return &v
}
func (v speakerBGPLive) observed() bool {
return v.Polled || v.Error != "" || len(v.Sessions) > 0
}
func (v speakerBGPLive) successful() bool {
return v.observed() && v.Error == ""
}
func (v speakerBGPLive) observation() liveObservation {
if v.ObservationOverride != nil {
return *v.ObservationOverride
}
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
if o.Source == "" {
o.Source = "none"
}
switch {
case v.successful():
o.Quality = "ok"
case v.Error != "" && !v.LastSuccessAt.IsZero():
o.Quality = "stale"
o.ErrorCode = liveErrorCode(v.Error)
case v.Error != "":
o.Quality = "unreachable"
o.ErrorCode = liveErrorCode(v.Error)
default:
o.ErrorCode = "not_polled"
}
return o
}
func liveErrorCode(err string) string {
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
return "not_configured"
}
return "poll_failed"
}
type peerSessionOnSpeaker struct { type peerSessionOnSpeaker struct {
SpeakerID string `json:"speaker_id"` SpeakerID string `json:"speaker_id"`
Label string `json:"label"` Label string `json:"label"`
State string `json:"state"` State string `json:"state"`
PollError string `json:"poll_error,omitempty"` PollError string `json:"poll_error,omitempty"`
Observation liveObservation `json:"observation"`
} }
type liveSpeakerPoll struct { type liveSpeakerPoll struct {
SpeakerID string `json:"speaker_id"` SpeakerID string `json:"speaker_id"`
Label string `json:"label"` Label string `json:"label"`
OK bool `json:"ok"` OK bool `json:"ok"`
SessionCount int `json:"session_count"` SessionCount *int `json:"session_count"`
PollError string `json:"poll_error,omitempty"` PollError string `json:"poll_error,omitempty"`
Observation liveObservation `json:"observation"`
} }
func speakerDisplayLabel(sp *store.Speaker) string { func speakerDisplayLabel(sp *store.Speaker) string {
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
return sp.ID return sp.ID
} }
func masterSpeakerID(speakers []*store.Speaker) string {
for _, sp := range speakers {
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
return sp.ID
}
}
return ""
}
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
if !fresh {
if v, ok := peerLiveCache.Load(tenantID); ok {
ent := v.(peerLiveCacheEntry)
if time.Since(ent.at) < peerLiveCacheTTL {
return ent.views
}
}
}
speakers := s.store.ListSpeakersForTenant(tenantID)
views := make([]speakerBGPLive, 0, len(speakers)+1)
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
v := speakerBGPLive{Label: "CP (local BIRD)"}
if mid := masterSpeakerID(speakers); mid != "" {
v.SpeakerID = mid
for _, sp := range speakers {
if sp != nil && sp.ID == mid {
v.Label = speakerDisplayLabel(sp)
break
}
}
}
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
if err != nil {
v.Error = err.Error()
} else {
v.Sessions = birdfmt.ParseBGPSessions(out)
}
views = append(views, v)
}
opts := nodedispatch.Options{Timeout: 8 * time.Second}
type resWrap struct {
sp *store.Speaker
res nodedispatch.BirdProtocolsResult
}
ch := make(chan resWrap, len(speakers))
var wg sync.WaitGroup
for _, sp := range speakers {
if sp == nil {
continue
}
meta := store.ParseSpeakerMeta(sp.MetaJSON)
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
continue
}
wg.Add(1)
go func(speaker *store.Speaker) {
defer wg.Done()
ch <- resWrap{
sp: speaker,
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
}
}(sp)
}
wg.Wait()
close(ch)
for rw := range ch {
views = append(views, speakerBGPLive{
SpeakerID: rw.sp.ID,
Label: speakerDisplayLabel(rw.sp),
Sessions: rw.res.Sessions,
Error: rw.res.Error,
})
}
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
return views
}
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll { func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
out := make([]liveSpeakerPoll, 0, len(views)) out := make([]liveSpeakerPoll, 0, len(views))
for _, v := range views { for _, v := range views {
var sessionCount *int
if v.successful() {
count := len(v.Sessions)
sessionCount = &count
}
out = append(out, liveSpeakerPoll{ out = append(out, liveSpeakerPoll{
SpeakerID: v.SpeakerID, SpeakerID: v.SpeakerID,
Label: v.Label, Label: v.Label,
OK: v.Error == "", OK: v.successful(),
SessionCount: len(v.Sessions), SessionCount: sessionCount,
PollError: v.Error, PollError: v.Error,
Observation: v.observation(),
}) })
} }
return out return out
@@ -183,11 +160,12 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
protoName := birdfmt.PeerProtocolName(peer.ID) protoName := birdfmt.PeerProtocolName(peer.ID)
for _, v := range views { for _, v := range views {
if v.Error != "" && len(v.Sessions) == 0 { if !v.successful() {
on = append(on, peerSessionOnSpeaker{ on = append(on, peerSessionOnSpeaker{
SpeakerID: v.SpeakerID, SpeakerID: v.SpeakerID,
Label: v.Label, Label: v.Label,
PollError: v.Error, PollError: v.Error,
Observation: v.observation(),
}) })
continue continue
} }
@@ -197,6 +175,7 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
SpeakerID: v.SpeakerID, SpeakerID: v.SpeakerID,
Label: v.Label, Label: v.Label,
State: "absent", State: "absent",
Observation: v.observation(),
}) })
continue continue
} }
@@ -204,6 +183,7 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
SpeakerID: v.SpeakerID, SpeakerID: v.SpeakerID,
Label: v.Label, Label: v.Label,
State: sess.State, State: sess.State,
Observation: v.observation(),
} }
on = append(on, hit) on = append(on, hit)
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") { if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
} }
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) { func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views) expected := expectedPeerViews(peer, views)
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
row["session_on_speakers"] = on row["session_on_speakers"] = on
row["established_on_speakers"] = establishedOn row["established_on_speakers"] = establishedOn
row["session_conflict"] = false row["session_conflict"] = false
row["session_mismatch"] = mismatch row["session_mismatch"] = mismatch
row["last_known_session_state"] = row["session_state"]
row["session_state"] = nil
if state != "" { if state != "" {
row["session_state"] = state row["session_state"] = state
} }
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
row["session_state"] = "absent"
}
row["live_observation"] = peerLiveObservation(expected)
if connLabel != "" { if connLabel != "" {
row["connected_speaker_label"] = connLabel row["connected_speaker_label"] = connLabel
} }
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID) row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
} }
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
return views
}
want := strings.TrimSpace(*peer.SpeakerID)
expected := make([]speakerBGPLive, 0, 1)
for _, v := range views {
if strings.EqualFold(v.SpeakerID, want) {
expected = append(expected, v)
}
}
return expected
}
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
for _, v := range views {
if !v.successful() {
return false
}
}
return true
}
func peerLiveObservation(views []speakerBGPLive) liveObservation {
if len(views) == 0 {
return speakerBGPLive{}.observation()
}
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
if result.Source == "" {
result.Source = "none"
}
var latestObserved, latestReceived, latestSuccess time.Time
allOK, allFailed, anyStale := true, true, false
for _, v := range views {
o := v.observation()
if v.ObservedAt.After(latestObserved) {
latestObserved = v.ObservedAt
}
if v.ReceivedAt.After(latestReceived) {
latestReceived = v.ReceivedAt
}
if v.LastSuccessAt.After(latestSuccess) {
latestSuccess = v.LastSuccessAt
}
allOK = allOK && v.successful()
allFailed = allFailed && o.Quality == "unreachable"
anyStale = anyStale || o.Quality == "stale"
if result.ErrorCode == "" && o.ErrorCode != "" {
result.ErrorCode = o.ErrorCode
}
if o.ExpiresAt != nil {
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
if err == nil {
previous := time.Time{}
if result.ExpiresAt != nil {
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
}
if previous.IsZero() || expires.Before(previous) {
result.ExpiresAt = o.ExpiresAt
}
}
}
}
result.ObservedAt = liveTime(latestObserved)
result.ReceivedAt = liveTime(latestReceived)
result.LastSuccessAt = liveTime(latestSuccess)
if len(views) > 1 {
result.Source = "mixed"
}
switch {
case allOK:
result.Quality = "ok"
result.ErrorCode = ""
case anyStale:
result.Quality = "stale"
case allFailed:
result.Quality = "unreachable"
}
return result
}
func peerLiveSpeakerIDOrNull(id string) any { func peerLiveSpeakerIDOrNull(id string) any {
if strings.TrimSpace(id) == "" { if strings.TrimSpace(id) == "" {
return nil return nil
+73
View File
@@ -2,11 +2,84 @@ package httpapi
import ( import (
"testing" "testing"
"time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/store" "evobgp/internal/store"
) )
func TestApplyPeerLiveFieldsObservationStates(t *testing.T) {
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
cases := []struct {
name string
views []speakerBGPLive
wantState any
wantQuality string
}{
{"no collector", nil, nil, "unknown"},
{"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"},
{"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"},
{"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, peer, tc.views)
if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" {
t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"])
}
observation := row["live_observation"].(liveObservation)
if observation.Quality != tc.wantQuality {
t.Fatalf("observation=%+v", observation)
}
})
}
}
func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) {
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
assigned := "speaker-a"
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned}
viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"}
viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"}
viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
cases := []struct {
name string
assigned bool
views []speakerBGPLive
wantState any
wantQuality string
wantObserved string
}{
{"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)},
{"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)},
{"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)},
{"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
candidate := *peer
if !tc.assigned {
candidate.SpeakerID = nil
}
row := map[string]any{"session_state": "Established"}
applyPeerLiveFields(row, &candidate, tc.views)
obs := row["live_observation"].(liveObservation)
if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality {
t.Fatalf("state=%v observation=%+v", row["session_state"], obs)
}
if tc.wantObserved == "" && obs.ObservedAt != nil {
t.Fatalf("unexpected timestamp %v", *obs.ObservedAt)
}
if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) {
t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved)
}
})
}
}
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) { func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
peer := &store.BGPPeer{ peer := &store.BGPPeer{
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
+42 -15
View File
@@ -9,23 +9,33 @@ import (
"time" "time"
) )
// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints. const maxAuthRateLimitClients = 4096
type authRateHit struct {
start time.Time
count int
}
// authRateLimiter keeps a bounded fixed-window counter per client IP.
type authRateLimiter struct { type authRateLimiter struct {
mu sync.Mutex mu sync.Mutex
hits map[string][]time.Time hits map[string]authRateHit
limit int limit int
window time.Duration window time.Duration
maxClients int
nextSweep time.Time
} }
func newAuthRateLimiter() *authRateLimiter { func newAuthRateLimiter() *authRateLimiter {
limit := 60 limit := 60
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 { if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 {
limit = n limit = n
} }
return &authRateLimiter{ return &authRateLimiter{
hits: make(map[string][]time.Time), hits: make(map[string]authRateHit),
limit: limit, limit: limit,
window: time.Minute, window: time.Minute,
maxClients: maxAuthRateLimitClients,
} }
} }
@@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool {
if l == nil { if l == nil {
return true return true
} }
now := time.Now() return l.allowAt(ip, time.Now())
}
func (l *authRateLimiter) allowAt(ip string, now time.Time) bool {
l.mu.Lock() l.mu.Lock()
defer l.mu.Unlock() defer l.mu.Unlock()
cut := now.Add(-l.window) if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) {
arr := l.hits[ip] l.nextSweep = time.Time{}
kept := arr[:0] for key, hit := range l.hits {
for _, t := range arr { expiry := hit.start.Add(l.window)
if t.After(cut) { if !now.Before(expiry) {
kept = append(kept, t) delete(l.hits, key)
} else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
l.nextSweep = expiry
} }
} }
if len(kept) >= l.limit { }
l.hits[ip] = kept hit, exists := l.hits[ip]
if !exists && len(l.hits) >= l.maxClients {
// Never evict an active counter: a new identity must wait for expiry.
return false return false
} }
kept = append(kept, now) if exists && hit.count >= l.limit {
l.hits[ip] = kept return false
}
if !exists {
hit.start = now
expiry := now.Add(l.window)
if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
l.nextSweep = expiry
}
}
hit.count++
l.hits[ip] = hit
return true return true
} }
+100
View File
@@ -0,0 +1,100 @@
package httpapi
import (
"bytes"
"context"
"errors"
"io"
"net"
"net/http"
"os"
"strconv"
"strings"
)
const requestBodyLimit int64 = 8 << 20
// withRequestBodyLimit must run inside authMiddleware. Reading the complete
// bounded body before dispatch prevents a valid JSON prefix from committing a
// mutation before a later size/read error is discovered. Content-Type is not a
// security boundary; CSV retains its additional row/field checks in its handler.
func withRequestBodyLimit(next http.Handler) http.Handler {
return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next)
}
func withBodyLimit(limit int64, next http.Handler) http.Handler {
return withBodyLimitAndConcurrency(limit, 4, next)
}
func requestBodyConcurrency() int {
n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT")))
if err == nil && n >= 1 && n <= 64 {
return n
}
return 4
}
func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler {
slots := make(chan struct{}, concurrency)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
original := r.Body
if original != nil {
defer func() { _ = original.Close() }()
}
if r.ContentLength > limit {
writeRequestBodyTooLarge(w)
return
}
if r.Context().Err() != nil {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
return
}
if r.Body == nil || r.Body == http.NoBody {
next.ServeHTTP(w, r)
return
}
select {
case slots <- struct{}{}:
defer func() { <-slots }()
default:
w.Header().Set("Retry-After", "1")
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full")
return
}
bounded := http.MaxBytesReader(w, r.Body, limit)
data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded})
var tooLarge *http.MaxBytesError
if errors.As(err, &tooLarge) {
writeRequestBodyTooLarge(w)
return
}
var timeout net.Error
if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
return
}
if err != nil {
writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body")
return
}
r.Body = io.NopCloser(bytes.NewReader(data))
r.ContentLength = int64(len(data))
next.ServeHTTP(w, r)
})
}
func writeRequestBodyTooLarge(w http.ResponseWriter) {
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit")
}
type requestContextReader struct {
ctx context.Context
reader io.Reader
}
func (r requestContextReader) Read(p []byte) (int, error) {
if err := r.ctx.Err(); err != nil {
return 0, err
}
return r.reader.Read(p)
}
+273
View File
@@ -0,0 +1,273 @@
package httpapi
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
)
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
const limit = 32
for _, unknown := range []bool{false, true} {
for _, contentType := range []string{"", "application/json", "text/plain"} {
for _, extra := range []int{0, 1} {
label := "known"
if unknown {
label = "unknown"
}
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
mutations := 0
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var input map[string]any
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
t.Error(err)
}
mutations++
w.WriteHeader(http.StatusCreated)
}))
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
if unknown {
req.ContentLength = -1
}
if contentType != "" {
req.Header.Set("Content-Type", contentType)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if extra == 0 {
if w.Code != http.StatusCreated || mutations != 1 {
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
}
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
}
})
}
}
}
}
type checkedRequestBody struct {
reads int
closed bool
reader io.Reader
}
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
type failingRequestReader struct{}
func (failingRequestReader) Read([]byte) (int, error) {
return 0, errors.New("untrusted detailed read failure")
}
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
for _, cancelled := range []bool{false, true} {
calls := 0
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
req.Body = b
req.ContentLength = -1
if cancelled {
ctx, cancel := context.WithCancel(req.Context())
cancel()
req = req.WithContext(ctx)
}
w := httptest.NewRecorder()
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
want := http.StatusBadRequest
if cancelled {
want = http.StatusRequestTimeout
}
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
}
if cancelled && b.reads != 0 {
t.Fatal("read body after cancellation")
}
}
}
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
b := &checkedRequestBody{reader: failingRequestReader{}}
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
req.Body = b
req.ContentLength = requestBodyLimit + 1
s := &Server{}
called := false
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
}
}
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
req.ContentLength = length
called := false
w := httptest.NewRecorder()
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
if w.Code != http.StatusRequestEntityTooLarge || called {
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
}
}
}
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
if err != nil {
t.Fatal(err)
}
defer s.Close()
tenant, _, _, _, _ := s.store.DemoIDs()
before := len(s.store.ListModules(tenant))
h := s.Handler()
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
for _, extra := range []int{1, 0} {
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
req.ContentLength = -1
req.Header.Set("Authorization", "Bearer dev")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if extra == 1 {
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
}
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
}
}
}
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
f, ok := w.(http.Flusher)
if !ok {
t.Fatal("response lost Flusher")
}
w.Header().Set("Content-Type", "text/event-stream")
_, _ = io.WriteString(w, "data: ready\n\n")
f.Flush()
})).ServeHTTP(w, req)
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
t.Fatal("stream response buffered")
}
}
type blockedAdmissionBody struct {
entered chan struct{}
resume chan struct{}
sent bool
}
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
if b.sent {
return 0, io.EOF
}
b.sent = true
close(b.entered)
<-b.resume
p[0] = 'x'
return 1, nil
}
func (*blockedAdmissionBody) Close() error { return nil }
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
readEntered, readResume := make(chan struct{}), make(chan struct{})
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
var once sync.Once
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/empty" {
once.Do(func() { close(handlerEntered) })
<-handlerResume
}
w.WriteHeader(http.StatusNoContent)
}))
first := httptest.NewRequest(http.MethodPost, "/first", nil)
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
first.ContentLength = 0 // A claimed zero must not bypass body admission.
done := make(chan struct{})
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
select {
case <-readEntered:
case <-time.After(3 * time.Second):
t.Fatal("first body did not start")
}
reject := func() {
t.Helper()
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
h.ServeHTTP(w, req)
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
}
}
reject()
// An empty legacy GET remains available while all body slots are occupied.
empty := httptest.NewRecorder()
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
if empty.Code != http.StatusNoContent {
t.Fatal("empty GET consumed body slot")
}
close(readResume)
select {
case <-handlerEntered:
case <-time.After(3 * time.Second):
t.Fatal("handler did not start")
}
reject() // The fully read buffer is still retained by the running handler.
close(handlerResume)
select {
case <-done:
case <-time.After(3 * time.Second):
t.Fatal("first request did not finish")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
if w.Code != http.StatusNoContent {
t.Fatalf("body slot not released: %d", w.Code)
}
}
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
for range 2 {
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
req.Body = io.NopCloser(failingRequestReader{})
req.ContentLength = -1
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusBadRequest {
t.Fatalf("read failure retained slot: %d", w.Code)
}
}
}
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
for _, tc := range []struct {
raw string
want int
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
t.Run(tc.raw, func(t *testing.T) {
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
if got := requestBodyConcurrency(); got != tc.want {
t.Fatalf("concurrency=%d want%d", got, tc.want)
}
})
}
}
+147 -79
View File
@@ -8,14 +8,12 @@ import (
"errors" "errors"
"io" "io"
"net/http" "net/http"
"os"
"sort" "sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/bundle"
"evobgp/internal/jobs" "evobgp/internal/jobs"
"evobgp/internal/observability" "evobgp/internal/observability"
"evobgp/internal/pipeline" "evobgp/internal/pipeline"
@@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler {
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md. // Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth. // Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone) s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
s.mux.Handle("/v1/", s.authMiddleware(wrappedV1)) s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1)))
return s.withCORS(observability.HTTPMiddleware(s.mux)) return s.withCORS(observability.HTTPMiddleware(s.mux))
} }
@@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule) m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
m.HandleFunc("GET /peers", s.handleListPeers) m.HandleFunc("GET /peers", s.handleListPeers)
m.HandleFunc("GET /speakers", s.handleListSpeakers) m.HandleFunc("GET /speakers", s.handleListSpeakers)
m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh)
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey) m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh) m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh) m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
@@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob) m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport) m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob) m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob)
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision) m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle) m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll) m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
@@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
allPeers := s.store.ListPeers(a.TenantID) allPeers := s.store.ListPeers(a.TenantID)
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID) allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r)) page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID))
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) if err != nil {
defer cancel() writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh) return
if fresh {
s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews)
} }
items := make([]map[string]any, 0, len(page)) items := make([]map[string]any, 0, len(page))
for _, p := range page { for _, p := range page {
@@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
}) })
} }
func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string {
if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") {
return s.liveBGPProtocolStatesFresh(r.Context())
}
if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok {
return cached
}
return s.liveBGPProtocolStatesFresh(r.Context())
}
func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string {
sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
if sock == "" {
return map[string]string{}
}
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
if err != nil {
return map[string]string{}
}
states := birdfmt.ParseBGPProtocolStates(out)
observability.SetBirdProtocolStates(states)
return states
}
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state. // parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
func parseBGPProtocolStates(output string) map[string]string { func parseBGPProtocolStates(output string) map[string]string {
out := make(map[string]string) out := make(map[string]string)
@@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) {
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
var liveByID map[string]map[string]any var liveByID map[string]map[string]any
if fresh { if fresh {
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second) var err error
defer cancel() _, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers)
liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers) if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
return
}
} }
items := make([]map[string]any, 0, len(speakers)) items := make([]map[string]any, 0, len(speakers))
for _, sp := range speakers { for _, sp := range speakers {
@@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) {
idemPtr = &idem idemPtr = &idem
} }
mid := mod.ID mid := mod.ID
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{ j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
"module_id": moduleID, "module_id": moduleID,
"trigger": "api", "trigger": "api",
}) })
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
} }
} }
moduleIDs := body.ModuleIDs moduleIDs := body.ModuleIDs
sort.Strings(body.ModuleIDs)
requestIDs := append([]string{}, body.ModuleIDs...)
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string
if idem != "" {
idemPtr = &idem
}
request := map[string]any{"module_ids": requestIDs, "trigger": "api"}
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) {
return
}
if len(moduleIDs) == 0 { if len(moduleIDs) == 0 {
now := time.Now().UTC() now := time.Now().UTC()
for _, mod := range s.store.ListModules(a.TenantID) { for _, mod := range s.store.ListModules(a.TenantID) {
@@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
} }
} }
} }
if len(moduleIDs) == 0 { if len(moduleIDs) == 0 && idem == "" {
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"}) writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
return return
} }
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
var idemPtr *string
if idem != "" {
idemPtr = &idem
}
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
"module_ids": moduleIDs, "module_ids": moduleIDs,
"trigger": "api", "trigger": "api",
}) }, request)
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusAccepted, map[string]any{ writeJSON(w, http.StatusAccepted, map[string]any{
"job_id": snap["job_id"], "job_id": snap["job_id"],
"status": snap["status"], "status": snap["status"],
"module_ids": moduleIDs, "module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"],
"modules_due": len(moduleIDs), "modules_due": len(moduleIDs),
}) })
} }
@@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any {
return m return m
} }
// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error). // enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation.
func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) { func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) {
if s.jobs == nil { if s.jobs == nil {
return return
} }
@@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
return return
} }
mid := moduleID mid := moduleID
key := "module_refresh:" + moduleID // Each mutation is a new logical refresh. Reusing a module-wide key would
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{ // incorrectly replay an older terminal job after later configuration changes.
_, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
"module_id": moduleID, "module_id": moduleID,
"trigger": trigger, "trigger": trigger,
}) })
s.reportAutomaticEnqueue(w, r, trigger, moduleID, err)
} }
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error). // enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
func (s *Server) enqueuePeerReconcile(tenantID, trigger string) { func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) {
if s.jobs == nil { if s.jobs == nil {
return return
} }
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{ _, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
"trigger": trigger, "trigger": trigger,
"job_title": "Обновление BGP пиров", "job_title": "Обновление BGP пиров",
}) })
s.reportAutomaticEnqueue(w, r, trigger, "", err)
}
func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) {
if err == nil {
return
}
w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed")
if a, ok := authFromContext(r.Context()); ok {
s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()})
}
} }
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
@@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") { if !s.requirePerm(w, a, "bgp:operations:read") {
return return
} }
release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation)
if !ok {
return
}
defer release()
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id")) rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
if err != nil { if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found") writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
@@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request)
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{ j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
"source_revision_id": revID, "source_revision_id": revID,
}) })
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) {
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
"revision_id": revID, "revision_id": revID,
"strategy": body.Strategy, "strategy": body.Strategy,
"trigger": "api", "trigger": "api",
}) })
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) {
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{ j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
"revision_id": revID, "revision_id": revID,
"speaker_id": spkID, "speaker_id": spkID,
"trigger": "api", "trigger": "api",
}) })
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) {
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil) j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
if err != nil { if err != nil {
writeInternalError(w, "internal", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) {
status := r.URL.Query().Get("status") status := r.URL.Query().Get("status")
kind := r.URL.Query().Get("kind") kind := r.URL.Query().Get("kind")
cursor := r.URL.Query().Get("cursor") cursor := r.URL.Query().Get("cursor")
list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit) list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit)
if err != nil {
writeInternalError(w, "jobs_list", err)
return
}
items := make([]map[string]any, 0, len(list)) items := make([]map[string]any, 0, len(list))
for _, j := range list { for _, j := range list {
items = append(items, j.Snapshot()) items = append(items, j.Snapshot())
@@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") { if !s.requirePerm(w, a, "bgp:operations:read") {
return return
} }
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id")) j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil { if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found") writeJobReadError(w, err)
return return
} }
writeJSON(w, http.StatusOK, j.Snapshot()) writeJSON(w, http.StatusOK, j.Snapshot())
@@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:read") { if !s.requirePerm(w, a, "bgp:operations:read") {
return return
} }
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id")) j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil { if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found") writeJobReadError(w, err)
return return
} }
snap := j.Snapshot() snap := j.Snapshot()
@@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
out["revision_id"] = v out["revision_id"] = v
} }
} }
sourceJobID := ""
if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" {
// Resolve through the tenant-scoped registry before exposing linked progress.
if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil {
writeJobReadError(w, err)
return
}
sourceJobID = source
out["source_deployment_job_id"] = source
}
rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID)
if err == nil && len(rows) == 0 && sourceJobID != "" {
rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID)
}
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable")
return
}
progress := make([]map[string]any, 0, len(rows))
for _, d := range rows {
progress = append(progress, map[string]any{
"job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID,
"generation": d.Generation, "state": d.State, "confirmation": d.Confirmation,
"error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound,
"health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID,
"health_source_at": d.HealthSourceAt, "health_error": d.HealthError,
})
}
out["deployment_progress"] = progress
writeJSON(w, http.StatusOK, out) writeJSON(w, http.StatusOK, out)
} }
@@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
if !s.requirePerm(w, a, "bgp:operations:admin") { if !s.requirePerm(w, a, "bgp:operations:admin") {
return return
} }
j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id")) j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil { if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "job not found") writeJobReadError(w, err)
return
}
writeJSON(w, http.StatusAccepted, j.Snapshot())
}
func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
return
}
if !s.requirePerm(w, a, "bgp:operations:admin") {
return
}
j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id"))
if err != nil {
if errors.Is(err, store.ErrJobQueueFull) {
writeJobEnqueueError(w, err)
return
}
if errors.Is(err, store.ErrJobConflict) {
writeProblem(w, http.StatusConflict, "Conflict", "job is not failed")
return
}
writeJobReadError(w, err)
return return
} }
writeJSON(w, http.StatusAccepted, j.Snapshot()) writeJSON(w, http.StatusAccepted, j.Snapshot())
@@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) {
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope") writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
return return
} }
rev, err := s.store.GetRevision(a.TenantID, rid) overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid)
if err != nil {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
return
}
frags := rev.PreviewFragments
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags)
if err != nil { if err != nil {
writeInternalError(w, "bundle overlay", err) writeInternalError(w, "bundle overlay", err)
return return
} }
frags = overlaid // Finished gzip, including manifest and bundle.sig. Revision bytes are immutable;
tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv) // the overlay fingerprint keeps speaker-specific renders from colliding.
tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) {
return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP)
})
if err != nil { if err != nil {
writeInternalError(w, "internal", err) if errors.Is(err, errNodeBundleRevisionMissing) {
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
return
}
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return
}
op := "internal"
var opErr *nodeBundleOpError
if errors.As(err, &opErr) && opErr.op != "" {
op = opErr.op
}
writeInternalError(w, op, err)
return return
} }
w.Header().Set("Content-Type", "application/gzip") w.Header().Set("Content-Type", "application/gzip")
+1 -18
View File
@@ -3,7 +3,6 @@ package httpapi
import ( import (
"context" "context"
"log" "log"
"net"
"net/http" "net/http"
"strings" "strings"
"time" "time"
@@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) {
} }
func clientIP(r *http.Request) string { func clientIP(r *http.Request) string {
if r == nil { return clientIPWithTrustedProxies(r, trustedProxyPrefixes())
return ""
}
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
parts := strings.Split(xff, ",")
if len(parts) > 0 {
return strings.TrimSpace(parts[0])
}
}
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
return xrip
}
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
if err != nil {
return strings.TrimSpace(r.RemoteAddr)
}
return host
} }
// initAuditPusher wires portal push when URL and secret are configured. // initAuditPusher wires portal push when URL and secret are configured.
+33 -24
View File
@@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail) writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
return return
} }
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil { release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation)
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail) if !ok {
return return
} }
defer release()
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id")) mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
if err != nil { if err != nil {
writeStoreErr(w, err) writeStoreErr(w, err)
@@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS") writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
return return
} }
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
return
}
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil) req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
if err != nil { if err != nil {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url") writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
@@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
_, _ = io.Copy(io.Discard, resp.Body) _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status)) writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
return return
} }
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) raw, err := pipeline.ReadCDNBody(resp.Body)
if err != nil { if err != nil {
writeBadGateway(w, "cdn preview read body", err) writeBadGateway(w, "cdn preview read body", err)
return return
@@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL}) s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create")
writeJSON(w, http.StatusCreated, cdnSourceJSON(x)) writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
} }
@@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID}) s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch")
writeJSON(w, http.StatusOK, cdnSourceJSON(x)) writeJSON(w, http.StatusOK, cdnSourceJSON(x))
} }
@@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID}) s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete")
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
@@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN}) s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create")
writeJSON(w, http.StatusCreated, asEntryJSON(x)) writeJSON(w, http.StatusCreated, asEntryJSON(x))
} }
@@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN}) s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch")
writeJSON(w, http.StatusOK, asEntryJSON(x)) writeJSON(w, http.StatusOK, asEntryJSON(x))
} }
@@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete")
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
@@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN}) s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create")
writeJSON(w, http.StatusCreated, domainEntryJSON(x)) writeJSON(w, http.StatusCreated, domainEntryJSON(x))
} }
@@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN}) s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch")
writeJSON(w, http.StatusOK, domainEntryJSON(x)) writeJSON(w, http.StatusOK, domainEntryJSON(x))
} }
@@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete")
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
@@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix}) s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create")
writeJSON(w, http.StatusCreated, ipRangeJSON(x)) writeJSON(w, http.StatusCreated, ipRangeJSON(x))
} }
@@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix}) s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch")
writeJSON(w, http.StatusOK, ipRangeJSON(x)) writeJSON(w, http.StatusOK, ipRangeJSON(x))
} }
@@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID}) s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete")
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
@@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
return return
} }
moduleID := r.PathValue("module_id") moduleID := r.PathValue("module_id")
res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body) res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body)
if err != nil { if err != nil {
if errors.Is(err, importer.ErrCSVTooLarge) {
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests")
return
}
if errors.Is(err, store.ErrInvalidInput) { if errors.Is(err, store.ErrInvalidInput) {
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty") writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
return return
@@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
if res.Imported > 0 { if res.Imported > 0 {
switch res.ModuleType { switch res.ModuleType {
case "AS_PREFIXES": case "AS_PREFIXES":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv")
case "DOMAINS": case "DOMAINS":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv")
case "IP_RANGES": case "IP_RANGES":
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv") s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv")
} }
} }
writeJSON(w, http.StatusOK, map[string]any{ writeJSON(w, http.StatusOK, map[string]any{
@@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor}) s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
s.enqueuePeerReconcile(a.TenantID, "peer_create") s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create")
writeJSON(w, http.StatusCreated, peerJSON(x)) writeJSON(w, http.StatusCreated, peerJSON(x))
} }
@@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor}) s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
s.enqueuePeerReconcile(a.TenantID, "peer_patch") s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch")
writeJSON(w, http.StatusOK, peerJSON(x)) writeJSON(w, http.StatusOK, peerJSON(x))
} }
@@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) {
return return
} }
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID}) s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
s.enqueuePeerReconcile(a.TenantID, "peer_delete") s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete")
w.WriteHeader(http.StatusNoContent) w.WriteHeader(http.StatusNoContent)
} }
@@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) {
return return
} }
if peerDiscoverySettingsChanged(body) { if peerDiscoverySettingsChanged(body) {
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings") s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings")
} }
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)}) s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
+53
View File
@@ -1,6 +1,7 @@
package httpapi package httpapi
import ( import (
"context"
"encoding/csv" "encoding/csv"
"encoding/json" "encoding/json"
"io" "io"
@@ -8,8 +9,60 @@ import (
"net/http/httptest" "net/http/httptest"
"strings" "strings"
"testing" "testing"
"evobgp/internal/importer"
"evobgp/internal/store"
) )
func TestCSVImportLimitsBeforeMutations(t *testing.T) {
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, _, _ := m.DemoIDs()
mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"})
if err != nil {
t.Fatal(err)
}
module := mod.ID
s := &Server{store: m}
before, err := m.ListASEntries(tenant, module)
if err != nil {
t.Fatal(err)
}
header := "asn,community\n"
validPrefix := header + "64512,\n"
oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n"
for _, tc := range []struct {
name, body string
unknownLength bool
status int
}{
{"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK},
{"oversize", oversized, false, http.StatusRequestEntityTooLarge},
{"unknown_length", oversized, true, http.StatusRequestEntityTooLarge},
{"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge},
{"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge},
} {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body))
if tc.unknownLength {
req.ContentLength = -1
req.TransferEncoding = []string{"chunked"}
}
req.SetPathValue("module_id", module)
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
w := httptest.NewRecorder()
s.handleImportModuleEntriesCSV(w, req)
if w.Code != tc.status {
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
}
after, e := m.ListASEntries(tenant, module)
if e != nil || len(after) != len(before) {
t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e)
}
})
}
}
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) { func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
srv, err := New(Options{ srv, err := New(Options{
InsecureDev: true, InsecureDev: true,
+5
View File
@@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) {
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required") writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
return return
} }
release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation)
if !ok {
return
}
defer release()
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q) res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
if err != nil { if err != nil {
if errors.Is(err, store.ErrInvalidInput) { if errors.Is(err, store.ErrInvalidInput) {
+2 -1
View File
@@ -1,6 +1,7 @@
package httpapi package httpapi
import ( import (
"context"
"encoding/json" "encoding/json"
"io" "io"
"net/http" "net/http"
@@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) {
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{ if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"}, {Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
+9 -6
View File
@@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
return return
} }
kind := "maintenance_policy_run" kind := "maintenance_policy_run"
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string var idemPtr *string
if idem != "" { if idem != "" {
@@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
if dryRun { if dryRun {
title = "Maintenance policy dry-run" title = "Maintenance policy dry-run"
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) {
return
}
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title, "policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
}) })
if err != nil { if err != nil {
writeInternalError(w, "maintenance_policy_enqueue", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -11,6 +11,7 @@ import (
"evobgp/internal/jobs" "evobgp/internal/jobs"
"evobgp/internal/pgmonitor" "evobgp/internal/pgmonitor"
"evobgp/internal/store"
) )
var ( var (
@@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) { if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
return return
} }
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key")) idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
var idemPtr *string var idemPtr *string
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
meta["actor_prefix"] = actorPrefix(a) meta["actor_prefix"] = actorPrefix(a)
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta) if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) {
return
}
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
return
}
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta)
if err != nil { if err != nil {
writeInternalError(w, "postgres_maint_enqueue", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
@@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
if idem != "" { if idem != "" {
idemPtr = &idem idemPtr = &idem
} }
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{ j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a), "policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
"job_title": "PostgreSQL cleanup (deprecated path)", "job_title": "PostgreSQL cleanup (deprecated path)",
}) })
if err != nil { if err != nil {
writeInternalError(w, "postgres_maint_enqueue", err) writeJobEnqueueError(w, err)
return return
} }
w.Header().Set("Location", "/v1/jobs/"+j.ID) w.Header().Set("Location", "/v1/jobs/"+j.ID)
+61 -8
View File
@@ -8,15 +8,17 @@ import (
"errors" "errors"
"net/http" "net/http"
"strings" "strings"
"sync"
"time" "time"
"evobgp/internal/audit" "evobgp/internal/audit"
"evobgp/internal/jobs" "evobgp/internal/jobs"
"evobgp/internal/logging"
"evobgp/internal/maintenance" "evobgp/internal/maintenance"
"evobgp/internal/pgmonitor" "evobgp/internal/pgmonitor"
"evobgp/internal/repository"
"evobgp/internal/runtimelogs" "evobgp/internal/runtimelogs"
"evobgp/internal/store" "evobgp/internal/store"
"evobgp/internal/telemetry"
"github.com/jackc/pgx/v5/pgxpool" "github.com/jackc/pgx/v5/pgxpool"
) )
@@ -30,7 +32,11 @@ type Server struct {
maintStats *maintenance.DBStatsProvider maintStats *maintenance.DBStatsProvider
jobs *jobs.Registry jobs *jobs.Registry
authLimiter *authRateLimiter authLimiter *authRateLimiter
operationOnce sync.Once
operationLimiter *operationLimiter
bundlePriv ed25519.PrivateKey bundlePriv ed25519.PrivateKey
bundleCache *nodeBundleCache
nodeBundleBuild nodeBundleBuilder
keyResolver *apiKeyResolver keyResolver *apiKeyResolver
firewallResolver *firewallTokenResolver firewallResolver *firewallTokenResolver
bundleSeedHex string bundleSeedHex string
@@ -39,6 +45,11 @@ type Server struct {
runtimeLogs *runtimelogs.Service runtimeLogs *runtimelogs.Service
runtimeLogsPolicyTenant string runtimeLogsPolicyTenant string
mux *http.ServeMux mux *http.ServeMux
backgroundMu sync.Mutex
backgroundStarted bool
backgroundClosed bool
backgroundCancel context.CancelFunc
telemetryCollector *telemetry.Collector
// Portal / dual-auth (JWT) configuration. // Portal / dual-auth (JWT) configuration.
jwtSecret string jwtSecret string
@@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) {
maintStats: maintStats, maintStats: maintStats,
jobs: reg, jobs: reg,
bundlePriv: priv, bundlePriv: priv,
bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes),
keyResolver: resolver, keyResolver: resolver,
firewallResolver: fwResolver, firewallResolver: fwResolver,
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex), bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
@@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) {
// Close releases database resources. // Close releases database resources.
func (s *Server) Close() { func (s *Server) Close() {
if s == nil {
return
}
s.backgroundMu.Lock()
if s.backgroundClosed {
s.backgroundMu.Unlock()
return
}
s.backgroundClosed = true
cancel := s.backgroundCancel
collector := s.telemetryCollector
s.backgroundMu.Unlock()
if cancel != nil {
cancel()
}
if collector != nil {
_ = collector.Close(context.Background())
}
// Do not close the pool while a worker is still persisting its final state.
if s.jobs != nil {
s.jobs.RequestCancelAll()
_ = s.jobs.Drain(context.Background())
}
if s.pgPool != nil { if s.pgPool != nil {
s.pgPool.Close() s.pgPool.Close()
} }
@@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store }
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all). // Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
func (s *Server) Jobs() *jobs.Registry { return s.jobs } func (s *Server) Jobs() *jobs.Registry { return s.jobs }
// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled. // StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled.
func (s *Server) StartBackground(ctx context.Context) { func (s *Server) StartBackground(ctx context.Context) {
if s != nil && s.pgPool != nil { if s == nil {
pgmonitor.StartScheduler(ctx, s.pgPool) return
}
s.backgroundMu.Lock()
defer s.backgroundMu.Unlock()
if s.backgroundClosed || s.backgroundStarted {
return
}
s.backgroundStarted = true
backgroundCtx, cancel := context.WithCancel(ctx)
s.backgroundCancel = cancel
if s.store != nil {
s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv())
}
if s.pgPool != nil {
pgmonitor.StartScheduler(backgroundCtx, s.pgPool)
if s.jobs != nil { if s.jobs != nil {
jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool)) jobs.StartDurableQueueLoop(backgroundCtx, s.jobs)
jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store)
} }
} }
if s != nil && s.maintConfig != nil && s.jobs != nil { if s != nil && s.maintConfig != nil && s.jobs != nil {
maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) { maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
key := idem key := idem
_, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{ _, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
"policy_id": policyID, "policy_id": policyID,
"dry_run": dryRun, "dry_run": dryRun,
"trigger": "scheduler", "trigger": "scheduler",
}) })
if err != nil {
logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err)
}
}, 30*time.Second) }, 30*time.Second)
} }
if s != nil && s.runtimeLogs != nil && s.store != nil { if s != nil && s.runtimeLogs != nil && s.store != nil {
runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{ runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{
Service: s.runtimeLogs, Service: s.runtimeLogs,
Store: s.store, Store: s.store,
PolicyTenant: s.runtimeLogsPolicyTenant, PolicyTenant: s.runtimeLogsPolicyTenant,
+60 -15
View File
@@ -6,7 +6,11 @@ import (
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"errors" "errors"
"net"
"net/http" "net/http"
"net/netip"
"net/url"
"strconv"
"strings" "strings"
"time" "time"
@@ -165,25 +169,66 @@ func publicControlPlaneURL(r *http.Request, override string) string {
if s := strings.TrimSpace(override); s != "" { if s := strings.TrimSpace(override); s != "" {
return strings.TrimRight(s, "/") return strings.TrimRight(s, "/")
} }
if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") { if r == nil {
return strings.TrimRight(origin, "/")
}
proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
if proto == "" {
proto = "https"
}
host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
if i := strings.Index(host, ","); i >= 0 {
host = strings.TrimSpace(host[:i])
}
if host == "" {
host = strings.TrimSpace(r.Host)
}
if host == "" {
return "" return ""
} }
if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) {
protos := r.Header.Values("X-Forwarded-Proto")
hosts := r.Header.Values("X-Forwarded-Host")
if len(protos) == 1 && len(hosts) == 1 {
proto := strings.TrimSpace(protos[0])
host := strings.TrimSpace(hosts[0])
if (proto == "http" || proto == "https") && validControlPlaneHost(host) {
return proto + "://" + host return proto + "://" + host
} }
}
}
if !validControlPlaneHost(r.Host) {
return ""
}
proto := "http"
if r.TLS != nil {
proto = "https"
}
return proto + "://" + r.Host
}
func validControlPlaneHost(host string) bool {
if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") {
return false
}
u, err := url.Parse("http://" + host)
if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
return false
}
name := u.Hostname()
if name == "" {
return false
}
if strings.HasPrefix(host, "[") {
if _, err := netip.ParseAddr(name); err != nil {
return false
}
} else if net.ParseIP(name) == nil {
for _, label := range strings.Split(name, ".") {
if label == "" || label[0] == '-' || label[len(label)-1] == '-' {
return false
}
for _, c := range label {
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
return false
}
}
}
}
if port := u.Port(); port != "" {
n, err := strconv.Atoi(port)
if err != nil || n < 1 || n > 65535 {
return false
}
}
return true
}
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) { func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
if s == nil || sp == nil || resp == nil { if s == nil || sp == nil || resp == nil {
+59 -73
View File
@@ -1,9 +1,7 @@
package httpapi package httpapi
import ( import (
"context"
"strings" "strings"
"sync"
"time" "time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
@@ -11,6 +9,34 @@ import (
"evobgp/internal/store" "evobgp/internal/store"
) )
type agentHealthPoll struct {
result nodedispatch.AgentHealthResult
at time.Time
lastSuccessAt time.Time
observationOverride *liveObservation
}
func agentHealthObservation(poll *agentHealthPoll) liveObservation {
if poll == nil {
return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"}
}
if poll.observationOverride != nil {
return *poll.observationOverride
}
o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"}
if !poll.result.OK {
o.Quality = "unreachable"
if !poll.lastSuccessAt.IsZero() {
o.Quality = "stale"
}
o.ErrorCode = liveErrorCode(poll.result.Error)
if poll.result.Error == "agent reported ok=false" {
o.ErrorCode = "reported_unhealthy"
}
}
return o
}
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) { func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
total = len(sessions) total = len(sessions)
for _, s := range sessions { for _, s := range sessions {
@@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
} }
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any { func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
var poll *agentHealthPoll
if health != nil {
poll = &agentHealthPoll{result: *health}
}
return speakerLiveStatusWithPoll(sp, view, poll)
}
func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any {
total, established := countBGPSessions(view.Sessions) total, established := countBGPSessions(view.Sessions)
bgpObservation := view.observation()
agentObservation := agentHealthObservation(health)
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil {
agentObservation = bgpObservation
}
m := map[string]any{ m := map[string]any{
"label": view.Label, "label": view.Label,
"bgp_poll_ok": view.Error == "", "bgp_poll_ok": view.successful(),
"bgp_sessions_total": total, "bgp_sessions_total": nil,
"bgp_established": established, "bgp_established": nil,
"bgp_observation": bgpObservation,
"agent_observation": agentObservation,
}
if view.successful() {
m["bgp_sessions_total"] = total
m["bgp_established"] = established
} }
if view.Error != "" { if view.Error != "" {
m["bgp_poll_error"] = view.Error m["bgp_poll_error"] = view.Error
} }
if health != nil { if health != nil {
m["agent_ok"] = health.OK m["agent_ok"] = health.result.OK
if health.Error != "" { if health.result.Error != "" {
m["agent_error"] = health.Error m["agent_error"] = health.result.Error
} }
if health.LastSyncAt != "" { if health.result.LastSyncAt != "" {
m["agent_last_sync_at"] = health.LastSyncAt m["agent_last_sync_at"] = health.result.LastSyncAt
} }
if health.LastAppliedRevisionID != "" { if health.result.LastAppliedRevisionID != "" {
m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID
} }
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") { } else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
m["agent_ok"] = view.Error == "" m["agent_ok"] = view.successful()
if view.Error != "" { if view.Error != "" {
m["agent_error"] = view.Error m["agent_error"] = view.Error
} }
@@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
m["agent_ok"] = false m["agent_ok"] = false
m["agent_error"] = "agent health not polled" m["agent_error"] = "agent health not polled"
} }
if len(view.Sessions) > 0 { if view.successful() {
sess := make([]map[string]any, 0, len(view.Sessions)) sess := make([]map[string]any, 0, len(view.Sessions))
for _, s := range view.Sessions { for _, s := range view.Sessions {
row := map[string]any{ row := map[string]any{
@@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
} }
return m return m
} }
func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any {
views := s.collectSpeakerBGPLive(ctx, tenantID, fresh)
viewByID := make(map[string]speakerBGPLive, len(views))
for _, v := range views {
if v.SpeakerID != "" {
viewByID[v.SpeakerID] = v
}
}
opts := nodedispatch.Options{Timeout: 8 * time.Second}
type healthWrap struct {
id string
h nodedispatch.AgentHealthResult
}
healthCh := make(chan healthWrap, len(speakers))
var wg sync.WaitGroup
for _, sp := range speakers {
if sp == nil {
continue
}
meta := store.ParseSpeakerMeta(sp.MetaJSON)
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
continue
}
wg.Add(1)
go func(speaker *store.Speaker) {
defer wg.Done()
healthCh <- healthWrap{
id: speaker.ID,
h: nodedispatch.FetchAgentHealth(ctx, speaker, opts),
}
}(sp)
}
wg.Wait()
close(healthCh)
healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers))
for hw := range healthCh {
healthByID[hw.id] = hw.h
}
out := make(map[string]map[string]any, len(speakers))
for _, sp := range speakers {
if sp == nil {
continue
}
view, ok := viewByID[sp.ID]
if !ok {
view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)}
}
var hp *nodedispatch.AgentHealthResult
if h, ok := healthByID[sp.ID]; ok {
hCopy := h
hp = &hCopy
}
out[sp.ID] = speakerLiveStatusJSON(sp, view, hp)
}
return out
}
+17
View File
@@ -2,12 +2,29 @@ package httpapi
import ( import (
"testing" "testing"
"time"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/nodedispatch" "evobgp/internal/nodedispatch"
"evobgp/internal/store" "evobgp/internal/store"
) )
func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) {
sp := &store.Speaker{ID: "master", Role: "master"}
unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil)
if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil {
t.Fatalf("unpolled=%#v", unpolled)
}
if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" {
t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"])
}
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil)
if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 {
t.Fatalf("empty=%#v", empty)
}
}
func TestCountBGPSessions(t *testing.T) { func TestCountBGPSessions(t *testing.T) {
total, est := countBGPSessions([]birdfmt.BGPSession{ total, est := countBGPSessions([]birdfmt.BGPSession{
{Name: "p1", State: "Established"}, {Name: "p1", State: "Established"},
+154
View File
@@ -0,0 +1,154 @@
package httpapi
import (
"context"
"net/http"
"time"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
"evobgp/internal/telemetry"
)
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
snapshots, err := s.store.ListTelemetry(ctx, tenant)
if err != nil {
return nil, nil, err
}
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
return views, statuses, nil
}
// snapshotObservation uses control-plane receipt time, never the remote clock.
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
if !o.ReceivedAt.IsZero() {
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
}
if v.Source == "" {
v.Source = "none"
}
switch {
case invalid != "":
v.ErrorCode = invalid
case o.ReceivedAt.IsZero():
v.ErrorCode = "not_polled"
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
case o.Error != "":
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
if !o.LastSuccessAt.IsZero() {
v.Quality = "stale"
}
default:
v.Quality = "ok"
}
return v
}
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
byID := make(map[string]*store.Speaker, len(speakers))
for _, sp := range speakers {
if sp != nil {
byID[sp.ID] = sp
}
}
grouped := make(map[string][]store.TelemetrySnapshot)
for _, snapshot := range snapshots {
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
continue
}
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
continue
}
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
}
views := make([]speakerBGPLive, 0, len(speakers)+1)
statuses := make(map[string]map[string]any, len(speakers))
appendView := func(id string, sp *store.Speaker) {
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
if sp == nil {
view.Label = "CP (local)"
}
var health *agentHealthPoll
rows := grouped[id]
// A restarted local collector may leave an expired authority behind.
// Only simultaneously current authorities conflict; never prefer one
// of two current BIRD nodes arbitrarily.
if len(rows) > 1 {
current := make([]store.TelemetrySnapshot, 0, len(rows))
for _, row := range rows {
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
current = append(current, row)
}
}
if len(current) > 0 {
rows = current
}
}
if len(rows) > 1 {
view.Error = "ambiguous telemetry authorities"
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
} else if len(rows) == 1 {
row := rows[0]
invalid := ""
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
invalid = "configuration_changed"
}
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
view.ObservationOverride = &bgp
view.Polled = !row.BGP.ReceivedAt.IsZero()
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
if bgp.Quality == "ok" {
view.Sessions = row.BGP.Sessions
} else {
view.Error = bgp.ErrorCode
}
if row.Kind == "remote_agent" {
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
if h.Quality != "ok" {
result.Error = h.ErrorCode
}
if h.Quality == "ok" && !result.OK {
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
}
health = &agentHealthPoll{result: result, observationOverride: &h}
}
}
views = append(views, view)
if sp != nil {
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
}
}
for _, sp := range speakers {
if sp != nil {
appendView(sp.ID, sp)
}
}
if len(grouped[""]) > 0 {
appendView("", nil)
}
return views, statuses
}
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
a, ok := authFromContext(r.Context())
if !ok {
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
return
}
if !s.requirePerm(w, a, "bgp:network:read") {
return
}
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
defer cancel()
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
if err != nil {
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
return
}
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
}
+126
View File
@@ -0,0 +1,126 @@
package httpapi
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
type telemetryHTTPStore struct {
store.Backend
discoveryWrites int
snapshotErr error
refreshCalls int
}
func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) {
s.discoveryWrites++
return s.Backend.UpsertPeerDiscovery(tenant, in)
}
func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) {
if s.snapshotErr != nil {
return nil, s.snapshotErr
}
return s.Backend.ListTelemetry(ctx, tenant)
}
func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) {
s.refreshCalls++
return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown)
}
func telemetryRequest(method, path, tenant string) *http.Request {
r := httptest.NewRequest(method, path, nil)
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"}))
}
func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) {
var calls atomic.Int64
agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
_, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`))
}))
defer agent.Close()
t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1")
mem := store.NewMemory()
mem.SeedDemo()
tenant, _, _, _, _ := mem.DemoIDs()
sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})})
if err != nil {
t.Fatal(err)
}
// Positive control: the legacy adapter reaches the fake configured agent.
if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK {
t.Fatal(result)
}
calls.Store(0)
st := &telemetryHTTPStore{Backend: mem}
server := &Server{store: st}
claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "")
if err != nil || len(claims) != 1 {
t.Fatalf("claim: %v %v", claims, err)
}
for i := 0; i < 20; i++ {
for _, tc := range []struct {
path string
handler http.HandlerFunc
}{
{"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers},
{"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries},
} {
w := httptest.NewRecorder()
tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant))
if w.Code != http.StatusOK {
t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
}
}
w := httptest.NewRecorder()
server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant))
if w.Code != http.StatusAccepted {
t.Fatalf("refresh: %d %s", w.Code, w.Body.String())
}
want := `"requested":false`
if i == 0 {
want = `"requested":true`
}
if !strings.Contains(w.Body.String(), want) {
t.Fatalf("refresh coalescing: %s", w.Body.String())
}
}
if calls.Load() != 0 || st.discoveryWrites != 0 {
t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites)
}
}
func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) {
mem := store.NewMemory()
mem.SeedDemo()
tenant, _, _, _, _ := mem.DemoIDs()
st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")}
server := &Server{store: st}
for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} {
w := httptest.NewRecorder()
handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant))
if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {
t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String())
}
}
r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil)
w := httptest.NewRecorder()
server.handleTelemetryRefresh(w, r)
if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 {
t.Fatal("unauthenticated refresh admitted")
}
r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT}))
w = httptest.NewRecorder()
server.handleTelemetryRefresh(w, r)
if w.Code != http.StatusForbidden || st.refreshCalls != 0 {
t.Fatal("unprivileged refresh admitted")
}
}
@@ -0,0 +1,30 @@
package httpapi
import (
"context"
"testing"
)
func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) {
s, err := New(Options{SeedDemo: true})
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s.StartBackground(ctx)
first := s.telemetryCollector
if first == nil {
t.Fatal("telemetry collector not started")
}
s.StartBackground(ctx)
if s.telemetryCollector != first {
t.Fatal("duplicate telemetry collector")
}
s.Close()
s.StartBackground(ctx)
if s.telemetryCollector != first {
t.Fatal("collector started after close")
}
s.Close()
}
+102
View File
@@ -0,0 +1,102 @@
package httpapi
import (
"evobgp/internal/birdfmt"
"evobgp/internal/store"
"testing"
"time"
)
func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) {
now := time.Now().UTC()
sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"}
observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"}
base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}}
cases := []struct {
name string
mutate func(*store.TelemetrySnapshot)
bgp, health bool
quality string
}{
{"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"},
{"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"},
{"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"},
{"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"},
{"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"},
{"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
snapshot := base
tc.mutate(&snapshot)
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute)
status := statuses[sp.ID]
if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality {
t.Fatalf("status=%+v observation=%+v", status, views[0].observation())
}
if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) {
t.Fatal("stale BGP exposed live sessions")
}
})
}
}
func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "master", Role: "master"}
for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} {
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute)
if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true {
t.Fatalf("unexpected healthy master: %+v", statuses)
}
}
}
func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "master", Role: "master"}
fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}}
old := fresh
old.LocalAuthority = "old"
old.BGP.ReceivedAt = now.Add(-2 * time.Minute)
_, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
if status[sp.ID]["bgp_poll_ok"] != true {
t.Fatal("expired authority hides current local snapshot")
}
old.BGP.ReceivedAt = now
_, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
if status[sp.ID]["bgp_poll_ok"] != false {
t.Fatal("two current local authorities are ambiguous")
}
}
func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) {
now := time.Now()
sp := &store.Speaker{ID: "replica", Role: "replica"}
row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}}
views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute)
observation := peerLiveObservation(views)
if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" {
t.Fatalf("changed config is not a failed current poll: %+v", observation)
}
}
func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) {
now := time.Now().UTC()
old := now.Add(-30 * time.Second)
a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "")
b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "")
want := old.Add(5 * time.Minute).Format(time.RFC3339Nano)
if a.ExpiresAt == nil || *a.ExpiresAt != want {
t.Fatalf("expiry ignores configured interval: %+v", a)
}
views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}}
combined := peerLiveObservation(views)
if combined.ExpiresAt == nil || *combined.ExpiresAt != want {
t.Fatalf("aggregate extends oldest source: %+v", combined)
}
missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "")
if missing.ExpiresAt != nil {
t.Fatal("unobserved source has expiry")
}
}
+91
View File
@@ -0,0 +1,91 @@
package httpapi
import (
"net"
"net/http"
"net/netip"
"os"
"strings"
)
// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid.
// An empty configuration deliberately treats every connection as direct.
func trustedProxyPrefixes() []netip.Prefix {
raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS"))
if raw == "" {
return nil
}
parts := strings.Split(raw, ",")
prefixes := make([]netip.Prefix, 0, len(parts))
for _, part := range parts {
prefix, err := netip.ParsePrefix(strings.TrimSpace(part))
if err != nil {
return nil
}
if prefix.Addr().Is4In6() {
if prefix.Bits() < 96 {
return nil
}
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
}
prefixes = append(prefixes, prefix.Masked())
}
return prefixes
}
func peerAddress(r *http.Request) (netip.Addr, bool) {
if r == nil {
return netip.Addr{}, false
}
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
if err != nil {
host = strings.TrimSpace(r.RemoteAddr)
}
ip, err := netip.ParseAddr(host)
if err != nil {
return netip.Addr{}, false
}
return ip.WithZone("").Unmap(), true
}
func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool {
for _, prefix := range prefixes {
if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) {
return true
}
}
return false
}
func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string {
peer, ok := peerAddress(r)
if !ok {
if r == nil {
return ""
}
return strings.TrimSpace(r.RemoteAddr)
}
if !isTrustedProxy(peer, prefixes) {
return peer.String()
}
values := r.Header.Values("X-Forwarded-For")
if len(values) != 1 || strings.TrimSpace(values[0]) == "" {
return peer.String()
}
hops := strings.Split(values[0], ",")
current := peer
for i := len(hops) - 1; i >= 0; i-- {
if !isTrustedProxy(current, prefixes) {
return current.String()
}
next, err := netip.ParseAddr(strings.TrimSpace(hops[i]))
if err != nil {
return peer.String()
}
if next.Zone() != "" {
return peer.String()
}
current = next.Unmap()
}
return current.String()
}
+129
View File
@@ -0,0 +1,129 @@
package httpapi
import (
"fmt"
"net/http/httptest"
"net/netip"
"testing"
"time"
)
func TestClientIPTrustedProxyChain(t *testing.T) {
prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
tests := []struct {
name string
remote string
xff string
want string
}{
{"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"},
{"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"},
{"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"},
{"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"},
{"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"},
{"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"},
{"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
r := httptest.NewRequest("GET", "http://example.test/", nil)
r.RemoteAddr = tt.remote
r.Header.Set("X-Forwarded-For", tt.xff)
if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want {
t.Fatalf("client IP = %q, want %q", got, tt.want)
}
})
}
}
func TestClientIPConfigurationFailsClosed(t *testing.T) {
r := httptest.NewRequest("GET", "http://example.test/", nil)
r.RemoteAddr = "10.0.0.2:1234"
r.Header.Set("X-Forwarded-For", "203.0.113.9")
r.Header.Set("X-Real-IP", "203.0.113.8")
if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" {
t.Fatalf("direct client IP = %q", got)
}
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid")
if got := clientIP(r); got != "10.0.0.2" {
t.Fatalf("invalid CIDR client IP = %q", got)
}
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128")
if got := clientIP(r); got != "203.0.113.9" {
t.Fatalf("mapped CIDR client IP = %q", got)
}
}
func TestPublicControlPlaneURLTrust(t *testing.T) {
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil)
r.Header.Set("Origin", "https://evil.example")
r.Header.Set("X-Forwarded-Proto", "https")
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
r.RemoteAddr = "198.51.100.8:1234"
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Fatalf("direct URL = %q", got)
}
r.RemoteAddr = "10.0.0.2:1234"
if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" {
t.Fatalf("proxied URL = %q", got)
}
if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" {
t.Fatalf("override URL = %q", got)
}
for _, bad := range []struct{ proto, host string }{
{"javascript", "bgp.example.test"},
{"https,http", "bgp.example.test"},
{"https", "evil.example/path"},
{"https", "[email protected]"},
{"https", "evil.example,bgp.example.test"},
{"https", "bgp.example.test:99999"},
{"https", "bgp.example.test:"},
} {
r.Header.Set("X-Forwarded-Proto", bad.proto)
r.Header.Set("X-Forwarded-Host", bad.host)
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got)
}
}
r.Header.Set("X-Forwarded-Proto", "https")
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
r.Header.Add("X-Forwarded-Host", "evil.example")
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
t.Fatalf("duplicate forwarded host = %q", got)
}
}
func TestAuthRateLimiterManyIdentities(t *testing.T) {
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32}
now := time.Unix(1000, 0)
for i := 0; i < 1000; i++ {
allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now)
if allowed != (i < l.maxClients) {
t.Fatalf("identity %d allowed = %t", i, allowed)
}
}
if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) {
t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits))
}
}
func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) {
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2}
now := time.Unix(1000, 0)
if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) {
t.Fatal("initial requests rejected")
}
if l.allowAt("a", now) || l.allowAt("c", now) {
t.Fatal("active limit or capacity bypassed")
}
if len(l.hits) != 2 {
t.Fatalf("map size = %d", len(l.hits))
}
if !l.allowAt("c", now.Add(time.Minute)) {
t.Fatal("expired entries did not release capacity")
}
if len(l.hits) != 1 {
t.Fatalf("expired entries remained: %d", len(l.hits))
}
}
+31
View File
@@ -0,0 +1,31 @@
package httpclient
import (
"errors"
"fmt"
"io"
"net/http"
)
// ErrResponseTooLarge means the complete decoded HTTP response exceeded its budget.
var ErrResponseTooLarge = errors.New("httpclient: response body exceeds limit")
// ReadResponseBody reads a complete bounded body, including one extra byte to
// distinguish an exact limit from truncation. Standard Transport gzip decoding
// occurs before this reader, so the limit covers decompressed data as well.
func ReadResponseBody(resp *http.Response, limit int64) ([]byte, error) {
if limit <= 0 || limit == int64(^uint64(0)>>1) {
return nil, fmt.Errorf("httpclient: invalid response limit")
}
if resp.ContentLength > limit {
return nil, ErrResponseTooLarge
}
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(body)) > limit {
return nil, ErrResponseTooLarge
}
return body, nil
}
+22
View File
@@ -0,0 +1,22 @@
package httpclient
import (
"errors"
"io"
"net/http"
"strings"
"testing"
)
func TestReadResponseBodyDoesNotAcceptValidPrefixTruncation(t *testing.T) {
for _, length := range []int64{-1, 9} {
resp := &http.Response{ContentLength: length, Body: io.NopCloser(strings.NewReader("{} "))}
if _, err := ReadResponseBody(resp, 8); !errors.Is(err, ErrResponseTooLarge) {
t.Fatalf("length=%d got %v", length, err)
}
}
resp := &http.Response{ContentLength: -1, Body: io.NopCloser(strings.NewReader("{} "))}
if b, err := ReadResponseBody(resp, 8); err != nil || len(b) != 8 {
t.Fatalf("exact limit failed: %q %v", b, err)
}
}
+90 -14
View File
@@ -1,8 +1,13 @@
package httpclient package httpclient
import ( import (
"errors"
"fmt"
"strings"
"sync" "sync"
"time" "time"
"evobgp/internal/observability"
) )
const ( const (
@@ -16,14 +21,77 @@ type hostBreaker struct {
openUntil time.Time openUntil time.Time
} }
var hostBreakers sync.Map // string -> *hostBreaker // ErrBreakerCapacity rejects new upstream identities when the bounded cache is full.
var ErrBreakerCapacity = errors.New("httpclient: upstream circuit capacity exhausted")
func breakerForHost(host string) *hostBreaker { type breakerEntry struct {
breaker *hostBreaker
lastUsed time.Time
active int
}
type breakerCache struct {
mu sync.Mutex
entries map[string]*breakerEntry
maximum int
idle time.Duration
nextSweep time.Time
}
var hostBreakers = breakerCache{entries: make(map[string]*breakerEntry), maximum: 1024, idle: 10 * time.Minute}
func (c *breakerCache) sweep(now time.Time) {
for host, e := range c.entries {
if e.active != 0 || now.Before(e.lastUsed.Add(c.idle)) {
continue
}
e.breaker.mu.Lock()
open := now.Before(e.breaker.openUntil)
e.breaker.mu.Unlock()
if !open {
delete(c.entries, host)
observability.RemoveUpstreamBreaker(host)
}
}
c.nextSweep = now.Add(time.Second)
}
func (c *breakerCache) acquire(host string, now time.Time) (*hostBreaker, string, func(), error) {
if len(host) > 254 {
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
}
host = strings.TrimSuffix(strings.ToLower(host), ".")
if len(host) > 253 {
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
}
if host == "" { if host == "" {
host = "_" host = "_"
} }
v, _ := hostBreakers.LoadOrStore(host, &hostBreaker{}) c.mu.Lock()
return v.(*hostBreaker) defer c.mu.Unlock()
if !now.Before(c.nextSweep) {
c.sweep(now)
}
e := c.entries[host]
if e == nil {
if len(c.entries) >= c.maximum {
return nil, host, nil, ErrBreakerCapacity
}
e = &breakerEntry{breaker: &hostBreaker{}}
c.entries[host] = e
}
e.active++
e.lastUsed = now
var once sync.Once
release := func() {
once.Do(func() {
c.mu.Lock()
defer c.mu.Unlock()
e.active--
e.lastUsed = time.Now()
})
}
return e.breaker, host, release, nil
} }
func (b *hostBreaker) allow() bool { func (b *hostBreaker) allow() bool {
@@ -51,23 +119,31 @@ func (b *hostBreaker) recordFailure() {
// SnapshotBreakers returns whether each known host breaker is currently open. // SnapshotBreakers returns whether each known host breaker is currently open.
func SnapshotBreakers() map[string]bool { func SnapshotBreakers() map[string]bool {
hostBreakers.mu.Lock()
defer hostBreakers.mu.Unlock()
now := time.Now()
hostBreakers.sweep(now)
out := map[string]bool{} out := map[string]bool{}
hostBreakers.Range(func(key, value any) bool { for host, e := range hostBreakers.entries {
host, _ := key.(string) b := e.breaker
b, _ := value.(*hostBreaker)
if b == nil {
return true
}
b.mu.Lock() b.mu.Lock()
open := time.Now().Before(b.openUntil) open := time.Now().Before(b.openUntil)
b.mu.Unlock() b.mu.Unlock()
out[host] = open out[host] = open
return true }
})
return out return out
} }
// ResetHostBreakers clears all circuit breakers (tests only). // ResetHostBreakers clears idle circuit breakers (tests only). In-flight
// references are retained so their completions cannot recreate orphan metrics.
func ResetHostBreakers() { func ResetHostBreakers() {
hostBreakers = sync.Map{} hostBreakers.mu.Lock()
defer hostBreakers.mu.Unlock()
for host, e := range hostBreakers.entries {
if e.active == 0 {
delete(hostBreakers.entries, host)
observability.RemoveUpstreamBreaker(host)
}
}
hostBreakers.nextSweep = time.Time{}
} }
+118
View File
@@ -0,0 +1,118 @@
package httpclient
import (
"context"
"errors"
"fmt"
"net/http"
"strings"
"testing"
"time"
"evobgp/internal/observability"
"github.com/prometheus/client_golang/prometheus"
)
func TestBreakerCanonicalHostAndActiveRetention(t *testing.T) {
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
now := time.Now()
first, key, release, err := c.acquire("Cache-Test.Example.", now)
if err != nil || key != "cache-test.example" {
t.Fatalf("canonical key=%s err=%v", key, err)
}
second, _, release2, err := c.acquire("cache-test.example", now)
if err != nil || first != second || len(c.entries) != 1 {
t.Fatal("alias bypassed shared breaker")
}
observability.SetUpstreamBreakerOpen(key, true)
first.openUntil = now.Add(3 * time.Minute)
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute)); !errors.Is(err, ErrBreakerCapacity) {
t.Fatalf("active entry evicted: %v", err)
}
release()
release()
release2()
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute+2*time.Second)); !errors.Is(err, ErrBreakerCapacity) {
t.Fatalf("open breaker evicted: %v", err)
}
_, _, done, err := c.acquire("other.example", now.Add(4*time.Minute))
if err != nil {
t.Fatal(err)
}
defer done()
if len(c.entries) != 1 || c.entries[key] != nil {
t.Fatal("idle closed entry retained")
}
families, err := prometheus.DefaultGatherer.Gather()
if err != nil {
t.Fatal(err)
}
for _, family := range families {
if strings.HasSuffix(family.GetName(), "upstream_breaker_open") {
for _, metric := range family.Metric {
for _, label := range metric.Label {
if label.GetName() == "host" && label.GetValue() == key {
t.Fatal("evicted host metric retained")
}
}
}
}
}
}
type breakerTransport func(*http.Request) (*http.Response, error)
func (f breakerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestBreakerCapacityRejectsBeforeDial(t *testing.T) {
ResetHostBreakers()
var releases []func()
defer func() {
for _, release := range releases {
release()
}
ResetHostBreakers()
}()
for i := 0; i < hostBreakers.maximum; i++ {
_, _, release, err := hostBreakers.acquire(fmt.Sprintf("capacity-%d.example", i), time.Now())
if err != nil {
t.Fatal(err)
}
releases = append(releases, release)
}
called := false
hc := &http.Client{Transport: breakerTransport(func(*http.Request) (*http.Response, error) { called = true; return nil, errors.New("unexpected dial") })}
req, _ := http.NewRequest(http.MethodGet, "https://overflow.example/", nil)
_, err := DoWithBreaker(context.Background(), hc, req, 1)
if !errors.Is(err, ErrBreakerCapacity) || called {
t.Fatalf("capacity err=%v dial=%t", err, called)
}
if len(SnapshotBreakers()) != hostBreakers.maximum {
t.Fatal("cache capacity changed")
}
ResetHostBreakers()
if len(SnapshotBreakers()) != hostBreakers.maximum {
t.Fatal("reset discarded in-flight references")
}
}
func TestBreakerRejectsOversizedIdentity(t *testing.T) {
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
if _, _, _, err := c.acquire(strings.Repeat("x", 254), time.Now()); err == nil || len(c.entries) != 0 {
t.Fatal("oversized identity retained")
}
name := strings.Repeat("a", 253)
first, key, release, err := c.acquire(name, time.Now())
if err != nil {
t.Fatal(err)
}
defer release()
second, alias, release2, err := c.acquire(strings.ToUpper(name)+".", time.Now())
if err != nil {
t.Fatal(err)
}
defer release2()
if first != second || key != alias {
t.Fatal("maximum length trailing-dot alias differs")
}
}
+8 -3
View File
@@ -54,7 +54,9 @@ func DoWithRetry(ctx context.Context, hc *http.Client, req *http.Request, maxAtt
continue continue
} }
if resp.StatusCode >= 500 { if resp.StatusCode >= 500 {
_, _ = io.Copy(io.Discard, resp.Body) // Draining is only an optimization for connection reuse, never an
// invitation to consume an unbounded error stream before retrying.
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
_ = resp.Body.Close() _ = resp.Body.Close()
lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status) lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status)
continue continue
@@ -72,8 +74,11 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
if req == nil || req.URL == nil { if req == nil || req.URL == nil {
return nil, fmt.Errorf("httpclient: nil request") return nil, fmt.Errorf("httpclient: nil request")
} }
host := req.URL.Hostname() br, host, release, err := hostBreakers.acquire(req.URL.Hostname(), time.Now())
br := breakerForHost(host) if err != nil {
return nil, err
}
defer release()
if !br.allow() { if !br.allow() {
observability.SetUpstreamBreakerOpen(host, true) observability.SetUpstreamBreakerOpen(host, true)
return nil, fmt.Errorf("httpclient: circuit open for %s", host) return nil, fmt.Errorf("httpclient: circuit open for %s", host)
+44
View File
@@ -0,0 +1,44 @@
package httpclient
import (
"context"
"io"
"net/http"
"testing"
)
type endlessRetryBody struct {
read int
closed bool
}
func (b *endlessRetryBody) Read(p []byte) (int, error) {
for i := range p {
p[i] = 'x'
}
b.read += len(p)
return len(p), nil
}
func (b *endlessRetryBody) Close() error { b.closed = true; return nil }
type retryTransport func(*http.Request) (*http.Response, error)
func (f retryTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestRetryDrainBoundsEndlessErrorBody(t *testing.T) {
b := &endlessRetryBody{}
c := &http.Client{Transport: retryTransport(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: 503, Status: "503 Service Unavailable", Body: b, Header: make(http.Header)}, nil
})}
req, err := http.NewRequest(http.MethodGet, "http://agent.example/bundle", nil)
if err != nil {
t.Fatal(err)
}
if _, err = DoWithRetry(context.Background(), c, req, 1); err == nil {
t.Fatal("upstream failure accepted")
}
if b.read != 64<<10 || !b.closed {
t.Fatalf("retry consumed %d bytes, closed=%v", b.read, b.closed)
}
var _ io.ReadCloser = b
}
+106 -8
View File
@@ -1,7 +1,10 @@
package importer package importer
import ( import (
"bytes"
"context"
"encoding/csv" "encoding/csv"
"errors"
"fmt" "fmt"
"io" "io"
"strconv" "strconv"
@@ -10,6 +13,18 @@ import (
"evobgp/internal/store" "evobgp/internal/store"
) )
// MaxCSVBytes is the complete request limit, including the optional header.
const MaxCSVBytes = 8 << 20
// MaxCSVRows bounds database work for a single import, excluding its header.
const MaxCSVRows = 10000
// MaxCSVFields bounds CSV parser allocations for delimiter-heavy records.
const MaxCSVFields = 64
// ErrCSVTooLarge means the entire import was rejected before creating entries.
var ErrCSVTooLarge = errors.New("importer: csv exceeds byte, row or field limit")
// Result of importing module entries from CSV. // Result of importing module entries from CSV.
type Result struct { type Result struct {
Imported int Imported int
@@ -17,19 +32,51 @@ type Result struct {
} }
// ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules. // ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules.
func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) { func ImportModuleEntriesCSV(ctx context.Context, st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
mod, err := st.GetModule(tenantID, moduleID) mod, err := st.GetModule(tenantID, moduleID)
if err != nil { if err != nil {
return nil, err return nil, err
} }
cr := csv.NewReader(io.LimitReader(body, 8<<20)) // Read one byte beyond the cap: a valid prefix must never be imported as if
// it were the complete request. Check all resource limits before mutations.
raw, err := io.ReadAll(io.LimitReader(body, MaxCSVBytes+1))
if err != nil {
return nil, fmt.Errorf("importer: read csv: %w", err)
}
if len(raw) > MaxCSVBytes {
return nil, ErrCSVTooLarge
}
if err := checkCSVFields(ctx, raw); err != nil {
return nil, err
}
cr := csv.NewReader(bytes.NewReader(raw))
cr.TrimLeadingSpace = true cr.TrimLeadingSpace = true
cr.FieldsPerRecord = -1 cr.FieldsPerRecord = -1
rows, err := cr.ReadAll() rows := make([][]string, 0)
headerRows := 0
for {
if err := ctx.Err(); err != nil {
return nil, err
}
record, err := cr.Read()
if err == io.EOF {
break
}
if err != nil { if err != nil {
return nil, fmt.Errorf("importer: invalid csv: %w", err) return nil, fmt.Errorf("importer: invalid csv: %w", err)
} }
if len(rows) == 0 && isCSVHeader(record) {
headerRows = 1
}
if len(rows)-headerRows >= MaxCSVRows {
return nil, ErrCSVTooLarge
}
rows = append(rows, record)
}
if len(rows) == 0 { if len(rows) == 0 {
return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput) return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput)
} }
@@ -63,18 +110,17 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
} }
start := 0 start := 0
if len(rows[0]) >= 2 { if isCSVHeader(rows[0]) {
key := strings.ToLower(strings.TrimSpace(rows[0][0]))
switch key {
case "asn", "domain", "iprange":
start = 1 start = 1
} }
}
imported := 0 imported := 0
switch mod.Type { switch mod.Type {
case "AS_PREFIXES": case "AS_PREFIXES":
for i := start; i < len(rows); i++ { for i := start; i < len(rows); i++ {
if err := ctx.Err(); err != nil {
return nil, err
}
rec := rows[i] rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue continue
@@ -97,6 +143,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
} }
case "DOMAINS": case "DOMAINS":
for i := start; i < len(rows); i++ { for i := start; i < len(rows); i++ {
if err := ctx.Err(); err != nil {
return nil, err
}
rec := rows[i] rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue continue
@@ -119,6 +168,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
} }
case "IP_RANGES": case "IP_RANGES":
for i := start; i < len(rows); i++ { for i := start; i < len(rows); i++ {
if err := ctx.Err(); err != nil {
return nil, err
}
rec := rows[i] rec := rows[i]
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) { if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
continue continue
@@ -145,3 +197,49 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
return &Result{Imported: imported, ModuleType: mod.Type}, nil return &Result{Imported: imported, ModuleType: mod.Type}, nil
} }
func isCSVHeader(record []string) bool {
if len(record) < 2 {
return false
}
switch strings.ToLower(strings.TrimSpace(record[0])) {
case "asn", "domain", "iprange":
return true
default:
return false
}
}
// checkCSVFields runs before encoding/csv allocates a string/index for each field.
// Quoted commas, escaped quotes and multiline fields don't start new fields.
// Syntax validation remains the responsibility of encoding/csv below.
func checkCSVFields(ctx context.Context, raw []byte) error {
quoted, fields := false, 1
for i := 0; i < len(raw); i++ {
if i%4096 == 0 {
if err := ctx.Err(); err != nil {
return err
}
}
switch raw[i] {
case '"':
if quoted && i+1 < len(raw) && raw[i+1] == '"' {
i++
continue
}
quoted = !quoted
case ',':
if !quoted {
fields++
if fields > MaxCSVFields {
return ErrCSVTooLarge
}
}
case '\n':
if !quoted {
fields = 1
}
}
}
return nil
}
+95
View File
@@ -0,0 +1,95 @@
package importer
import (
"context"
"errors"
"strings"
"testing"
"evobgp/internal/store"
)
type importBackend struct {
store.Backend
created int
onCreate func()
}
func (s *importBackend) GetModule(string, string) (*store.Module, error) {
return &store.Module{Type: "AS_PREFIXES"}, nil
}
func (s *importBackend) ListCommunities(string) ([]*store.Community, error) { return nil, nil }
func (s *importBackend) CreateASEntry(_, _ string, e *store.ASEntry) (*store.ASEntry, error) {
s.created++
if s.onCreate != nil {
s.onCreate()
}
return e, nil
}
func TestCSVByteLimitRejectsValidTruncatedPrefix(t *testing.T) {
header := "asn,community\n"
for _, extra := range []int{0, 1} {
s := &importBackend{}
body := header + strings.Repeat("\n", MaxCSVBytes-len(header)) + strings.Repeat("\n", extra)
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(body))
if extra == 0 && err != nil {
t.Fatalf("exact limit: %v", err)
}
if extra == 1 && !errors.Is(err, ErrCSVTooLarge) {
t.Fatalf("limit+1: %v", err)
}
if s.created != 0 {
t.Fatal("oversized import created entries")
}
}
}
func TestCSVRowLimitCheckedBeforeWrites(t *testing.T) {
for _, header := range []string{"", "asn,community\n"} {
for _, extra := range []int{0, 1} {
s := &importBackend{}
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(header+strings.Repeat("64512,\n", MaxCSVRows+extra)))
if extra == 0 && (err != nil || s.created != MaxCSVRows) {
t.Fatalf("exact rows: created=%d err=%v", s.created, err)
}
if extra == 1 && (!errors.Is(err, ErrCSVTooLarge) || s.created != 0) {
t.Fatalf("rows+1: created=%d err=%v", s.created, err)
}
}
}
}
func TestCSVCancellationStopsFurtherWrites(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
s := &importBackend{onCreate: cancel}
_, err := ImportModuleEntriesCSV(ctx, s, "tenant", "module", strings.NewReader("asn,community\n64512,\n64513,\n"))
if !errors.Is(err, context.Canceled) || s.created != 1 {
t.Fatalf("created=%d err=%v", s.created, err)
}
}
func TestCSVFieldLimitAndQuotedFields(t *testing.T) {
for _, tc := range []struct {
name, input string
oversized bool
}{
{"exact", "64512," + strings.Repeat(",", MaxCSVFields-2) + "\n", false},
{"over", "64512," + strings.Repeat(",", MaxCSVFields-1) + "\n", true},
{"many_fields", strings.Repeat(",", 1<<20) + "\n", true},
{"quoted_extra", "64512,,\"" + strings.Repeat(",", 128) + "\nquoted \"\"text\"\"\"\n", false},
} {
t.Run(tc.name, func(t *testing.T) {
s := &importBackend{}
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(tc.input))
if tc.oversized {
if !errors.Is(err, ErrCSVTooLarge) || s.created != 0 {
t.Fatalf("created=%d err=%v", s.created, err)
}
} else if err != nil || s.created != 1 {
t.Fatalf("created=%d err=%v", s.created, err)
}
})
}
}
+231
View File
@@ -0,0 +1,231 @@
package jobs
import (
"context"
"encoding/json"
"io"
"net/http"
"strings"
"sync/atomic"
"testing"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
"github.com/google/uuid"
)
type deployRoundTrip func(*http.Request) (*http.Response, error)
func (f deployRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestDeployApplyConfirmation(t *testing.T) {
for _, tc := range []struct {
name, response, confirmation, status string
dispatch bool
stale bool
wantApplied bool
}{
{name: "dispatch disabled", confirmation: "unconfirmed", status: StatusSucceeded},
{name: "bare success", dispatch: true, response: `{"ok":true}`, confirmation: "unconfirmed", status: StatusSucceeded},
{name: "matching ACK", dispatch: true, response: "matching", confirmation: "agent_reported", status: StatusSucceeded, wantApplied: true},
{name: "mismatched ACK", dispatch: true, response: `{"ok":true,"applied_revision_id":"other"}`, confirmation: "unconfirmed", status: StatusFailed},
{name: "stale desired", dispatch: true, response: "matching", confirmation: "unconfirmed", status: StatusFailed, stale: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
if tc.dispatch {
// A remote-only job must not touch the control plane BIRD tree.
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", t.TempDir())
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
} else {
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
}
m := store.NewMemory()
m.SeedDemo()
tenant, _, module, previous, speakerID := m.DemoIDs()
if err := m.SetLastAppliedRevision(tenant, speakerID, previous); err != nil {
t.Fatal(err)
}
revision := uuid.NewString()
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
t.Fatal(err)
}
newer := uuid.NewString()
if err := m.CreateRenderRevision(context.Background(), newer, tenant, module, &revision, "newer-hash", map[string]string{"bird.conf": "# newer\n"}, nil); err != nil {
t.Fatal(err)
}
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
if _, err := m.UpdateSpeaker(tenant, speakerID, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
t.Fatal(err)
}
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" || r.Header.Get("Authorization") != "Bearer secret" {
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
}
var body map[string]string
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["revision_id"] != revision {
t.Errorf("legacy sync body: %#v, %v", body, err)
}
if tc.stale {
if err := m.PublishRevisionForSpeaker(speakerID, newer); err != nil {
t.Error(err)
}
}
response := tc.response
if response == "matching" {
response = `{"ok":true,"applied_revision_id":"` + revision + `"}`
}
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
})}
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision, "speaker_id": speakerID}}
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
if j.Status != tc.status {
t.Fatalf("job status = %s, want %s; error=%v", j.Status, tc.status, j.Error)
}
sp, err := m.GetSpeaker(tenant, speakerID)
if err != nil {
t.Fatal(err)
}
want := previous
if tc.wantApplied {
want = revision
}
if sp.LastAppliedRevisionID == nil || *sp.LastAppliedRevisionID != want {
t.Fatalf("applied = %v, want %s", sp.LastAppliedRevisionID, want)
}
published, _, err := m.LatestPublishedRevision(speakerID)
if err != nil {
t.Fatal(err)
}
wantPublished := revision
if tc.stale {
wantPublished = newer
}
if published != wantPublished {
t.Fatalf("published = %s, want %s", published, wantPublished)
}
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
if len(results) != 1 || results[0].Confirmation != tc.confirmation {
t.Fatalf("results = %+v", results)
}
})
}
}
func TestDeployApplyPartialResult(t *testing.T) {
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
m := store.NewMemory()
m.SeedDemo()
tenant, _, module, previous, first := m.DemoIDs()
revision := uuid.NewString()
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
t.Fatal(err)
}
goodMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "good.example.test", AgentSecret: "secret"})
if _, err := m.UpdateSpeaker(tenant, first, &store.SpeakerPatch{MetaJSON: &goodMeta}); err != nil {
t.Fatal(err)
}
badMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "bad.example.test", AgentSecret: "secret"})
second, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: badMeta})
if err != nil {
t.Fatal(err)
}
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
response := `{"ok":true,"applied_revision_id":"` + revision + `"}`
if strings.HasPrefix(r.URL.Host, "bad.") {
response = `{"ok":false,"error":"apply failed"}`
}
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
})}
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
if j.Status != StatusFailed {
t.Fatalf("job status = %s, want failed", j.Status)
}
summary := j.Meta["apply_summary"].(map[string]any)
if summary["applied_count"] != 1 || summary["failed_count"] != 1 || summary["published_count"] != 2 {
t.Fatalf("partial summary = %#v", summary)
}
firstSp, _ := m.GetSpeaker(tenant, first)
secondSp, _ := m.GetSpeaker(tenant, second.ID)
if firstSp.LastAppliedRevisionID == nil || *firstSp.LastAppliedRevisionID != revision || secondSp.LastAppliedRevisionID != nil {
t.Fatalf("partial applied state: first=%v second=%v", firstSp.LastAppliedRevisionID, secondSp.LastAppliedRevisionID)
}
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
got := map[string]bool{}
for _, res := range results {
got[res.SpeakerID] = true
}
if len(results) != 2 || !got[first] || !got[second.ID] {
t.Fatalf("partial results=%+v", results)
}
}
func TestDeployApplyCancellationStopsFurtherPublication(t *testing.T) {
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
m := store.NewMemory()
m.SeedDemo()
tenant, _, module, previous, seeded := m.DemoIDs()
if err := m.DeleteSpeaker(tenant, seeded); err != nil {
t.Fatal(err)
}
revision := uuid.NewString()
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
t.Fatal(err)
}
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
speakerCount := speakerWakeLimit + 2
for range speakerCount {
if _, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: meta}); err != nil {
t.Fatal(err)
}
}
speakers := m.ListSpeakersForTenant(tenant)
if len(speakers) != speakerCount {
t.Fatalf("speakers=%d, want %d", len(speakers), speakerCount)
}
var j *Job
var calls atomic.Int32
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
// Cancel only once the in-flight wake limit is busy, so later speakers stay unpublished.
if calls.Add(1) == int32(speakerWakeLimit) {
j.RequestCancel()
}
<-r.Context().Done()
return nil, r.Context().Err()
})}
j = &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
if j.Status != StatusCancelled || int(calls.Load()) != speakerWakeLimit {
t.Fatalf("cancelled job status=%s dispatch calls=%d", j.Status, calls.Load())
}
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
gotResult := map[string]bool{}
for _, res := range results {
gotResult[res.SpeakerID] = res.Status == "cancelled"
}
if len(results) != speakerWakeLimit {
t.Fatalf("cancelled results=%+v", results)
}
for i, sp := range speakers {
got, _, err := m.LatestPublishedRevision(sp.ID)
if i < speakerWakeLimit {
if err != nil || got != revision || !gotResult[sp.ID] {
t.Fatalf("speaker %s published=%s err=%v result=%v", sp.ID, got, err, gotResult[sp.ID])
}
if sp.LastAppliedRevisionID != nil {
t.Fatalf("cancelled speaker %s marked applied", sp.ID)
}
continue
}
if err == nil {
t.Fatalf("speaker %s published %s after cancellation", sp.ID, got)
}
}
if j.Meta["apply_summary"].(map[string]any)["published_count"] != speakerWakeLimit {
t.Fatalf("cancelled summary = %#v", j.Meta["apply_summary"])
}
}
+126
View File
@@ -0,0 +1,126 @@
package jobs
import (
"context"
"errors"
"fmt"
"net/http"
"strconv"
"time"
"evobgp/internal/logging"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
func deploymentFenceForJob(j *Job) store.JobFence {
generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
round, _ := strconv.Atoi(j.metaString("reconcile_round"))
return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
}
func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
for {
d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if !errors.Is(err, store.ErrDeploymentBusy) {
return d, err
}
timer := time.NewTimer(time.Second)
select {
case <-ctx.Done():
timer.Stop()
return store.DeploymentSpeaker{}, ctx.Err()
case <-timer.C:
}
}
}
// ReconcileDeployments schedules at most three recovery rounds for current desired
// generations. Stable keys coalesce API replicas; the worker CAS checks generation
// again before publishing, and preserves the original desired request sequence.
func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
rows, err := st.ListPendingDeployments(ctx, 100)
if err != nil {
return err
}
for _, d := range rows {
key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
if err != nil {
return err
}
}
return nil
}
// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
// ingest/deploy/scheduler processes must not start this job-producing loop.
func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
if r == nil || r.queue == nil || st == nil {
return
}
r.durableMu.Lock()
if r.durableStop {
r.durableMu.Unlock()
return
}
r.durableLoopWG.Add(1)
r.durableMu.Unlock()
go func() {
defer r.durableLoopWG.Done()
ticker := time.NewTicker(10 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
}
r.durableMu.Lock()
stopped := r.durableStop
r.durableMu.Unlock()
if stopped {
return
}
qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
err := ObserveDeployments(qctx, st, nil)
if err == nil {
err = ReconcileDeployments(qctx, r, st)
}
cancel()
if err != nil && ctx.Err() == nil {
logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
}
}
}()
}
// ObserveDeployments samples compatible health before reconciling drift. The CP
// receipt time and optional agent source time are stored separately; missing
// legacy revision fields never manufacture an applied/verified confirmation.
func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
rows, err := st.ListDeploymentObservations(ctx, 10)
if err != nil {
return err
}
for _, d := range rows {
sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
if e != nil {
return e
}
observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
cancel()
observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
if h.OK && observation.RevisionID == "" {
observation.Error = "legacy health did not report an applied revision"
}
}
if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
return err
}
}
return nil
}
+140
View File
@@ -0,0 +1,140 @@
package jobs
import (
"context"
"io"
"net/http"
"strings"
"testing"
"time"
"evobgp/internal/store"
)
func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) {
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
ctx := context.Background()
r := NewRegistry(nil)
r.UseDurableQueue(m)
initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
if err != nil {
t.Fatal(err)
}
claim := func(owner string) *Job {
t.Helper()
rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute)
if e != nil || len(rows) != 1 {
t.Fatalf("claim: %+v %v", rows, e)
}
return jobFromRecord(rows[0])
}
first := claim("initial")
if first.ID != initial.ID {
t.Fatal("wrong initial job")
}
(&Worker{Store: m}).Process(first)
if err = m.FinishJob(ctx, recordFromJob(first)); err != nil {
t.Fatal(err)
}
// Enqueue an operator request before the synthetic recovery request. The latter
// must retain the earlier desired order despite its own larger queue sequence.
operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
if err != nil {
t.Fatal(err)
}
for range 2 {
if err = ReconcileDeployments(ctx, r, m); err != nil {
t.Fatal(err)
}
}
all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100)
if err != nil || len(all) != 3 {
t.Fatalf("reconcile dedupe: %d %v", len(all), err)
}
newer := claim("operator")
if newer.ID != operator.ID {
t.Fatal("wrong operator job")
}
recovery := claim("recovery")
f := deploymentFenceForJob(recovery)
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
if err != nil {
t.Fatal(err)
}
if d.RequestSequence != first.RequestSequence {
t.Fatal("recovery lost original desired order")
}
if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
t.Fatal(err)
}
newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision)
if err != nil {
t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err)
}
if newD.RequestSequence != newer.RequestSequence {
t.Fatal("operator desired sequence changed")
}
if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil {
t.Fatal("stale recovery generation reused")
}
}
func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) {
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
m := store.NewMemory()
m.SeedDemo()
tenant, _, _, revision, speaker := m.DemoIDs()
ctx := context.Background()
r := NewRegistry(nil)
r.UseDurableQueue(m)
_, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
if err != nil {
t.Fatal(err)
}
rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
if err != nil || len(rows) != 1 {
t.Fatal(err)
}
j := jobFromRecord(rows[0])
(&Worker{Store: m}).Process(j)
if err = m.FinishJob(ctx, recordFromJob(j)); err != nil {
t.Fatal(err)
}
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
t.Fatal(err)
}
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}`
client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) {
if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" {
t.Error("health must be authenticated GET")
}
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
})}
if err = ObserveDeployments(ctx, m, client); err != nil {
t.Fatal(err)
}
progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" {
t.Fatalf("legacy health confirmation: %+v %v", progress, err)
}
response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}`
if err = ObserveDeployments(ctx, m, client); err != nil {
t.Fatal(err)
}
progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID)
if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" {
t.Fatalf("late drift hidden: %+v", progress)
}
if err = ReconcileDeployments(ctx, r, m); err != nil {
t.Fatal(err)
}
jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100)
if len(jobs) != 1 {
t.Fatalf("drift recovery jobs: %d", len(jobs))
}
}
+193 -113
View File
@@ -2,136 +2,216 @@ package jobs
import ( import (
"context" "context"
"os" "errors"
"strconv"
"strings"
"time"
"evobgp/internal/logging" "evobgp/internal/logging"
"evobgp/internal/observability" "evobgp/internal/observability"
"evobgp/internal/repository" "evobgp/internal/store"
"sync/atomic"
"time"
) )
// Adopt registers a durable job_audit claim into the in-process registry and starts the worker. const durableLease = time.Minute
// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart.
func (r *Registry) Adopt(j *Job) bool { // StartDurableQueueLoop claims work only after reserving local capacity.
if r == nil || j == nil || j.ID == "" { func StartDurableQueueLoop(ctx context.Context, r *Registry) {
return false if r == nil || r.queue == nil {
return
} }
log := logging.With("component", "jobs.durable")
r.durableMu.Lock()
if r.durableStop {
r.durableMu.Unlock()
return
}
r.durableLoopWG.Add(2)
r.durableMu.Unlock()
go r.reapExpiredJobsLoop(ctx)
go func() {
defer r.durableLoopWG.Done()
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
if ctx.Err() != nil {
return
}
r.durableMu.Lock()
stopped := r.durableStop
r.durableMu.Unlock()
if stopped {
return
}
metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second)
if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil {
observability.RecordJobAuditDepth(counts)
}
metricCancel()
free := 0
reserve:
for free < cap(r.workerSem) {
select {
case r.workerSem <- struct{}{}:
free++
default:
break reserve
}
}
if free > 0 {
qctx, cancel := context.WithTimeout(ctx, 10*time.Second)
claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease)
cancel()
if err != nil {
log.Warn("claim failed", "err", err)
}
for i := len(claimed); i < free; i++ {
<-r.workerSem
}
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
for _, c := range claimed {
r.durableMu.Lock()
stopped = r.durableStop
if stopped || ctx.Err() != nil {
r.durableMu.Unlock()
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second)
if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil {
log.Warn("release during shutdown failed", "job_id", c.ID, "err", err)
}
releaseCancel()
<-r.workerSem
continue
}
j := jobFromRecord(c)
r.mu.Lock() r.mu.Lock()
if _, exists := r.byID[j.ID]; exists {
r.mu.Unlock()
return false
}
if j.Status == "" {
j.Status = StatusQueued
}
if j.Meta == nil {
j.Meta = map[string]any{}
}
r.byID[j.ID] = j r.byID[j.ID] = j
if j.IdempotencyKey != nil && *j.IdempotencyKey != "" {
r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j
}
if isRefreshKind(j.Kind) { if isRefreshKind(j.Kind) {
r.inflightRefresh[j.TenantID]++ r.inflightRefresh[j.TenantID]++
} }
workerStart := r.workerStart
r.mu.Unlock() r.mu.Unlock()
r.durableWG.Add(1)
if workerStart != nil { go r.runClaimed(j)
go func() { r.durableMu.Unlock()
r.workerSem <- struct{}{}
active := len(r.workerSem)
capacity := cap(r.workerSem)
observability.RecordJobQueueDepth(active, capacity)
defer func() {
<-r.workerSem
observability.RecordJobQueueDepth(len(r.workerSem), capacity)
}()
workerStart(j)
}()
}
return true
}
// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED).
// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m).
func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) {
if ctx == nil || reg == nil || audit == nil {
return
}
interval := 20 * time.Second
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" {
if d, err := time.ParseDuration(s); err == nil && d > 0 {
interval = d
} }
} }
staleAfter := 15 * time.Minute
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" {
if d, err := time.ParseDuration(s); err == nil && d > 0 {
staleAfter = d
}
}
limit := 8
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 {
limit = n
}
grace := 30 * time.Second
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" {
if d, err := time.ParseDuration(s); err == nil && d >= 0 {
grace = d
}
}
log := logging.With("component", "jobs.durable")
run := func() {
cctx, cancel := context.WithTimeout(ctx, 15*time.Second)
defer cancel()
n, err := audit.ReclaimStaleRunning(cctx, staleAfter)
if err != nil {
log.Warn("reclaim stale running failed", "err", err)
} else if n > 0 {
log.Info("reclaimed stale running jobs", "count", n)
}
claimed, err := audit.ClaimQueued(cctx, limit, grace)
if err != nil {
log.Warn("claim queued failed", "err", err)
return
}
for i := range claimed {
c := claimed[i]
j := &Job{
ID: c.ID,
TenantID: c.TenantID,
Kind: c.Kind,
Status: StatusQueued,
IdempotencyKey: c.IdempotencyKey,
ModuleID: c.ModuleID,
CreatedAt: c.CreatedAt,
Meta: c.Meta,
}
if !reg.Adopt(j) {
// Already local — leave DB running; local worker owns it.
continue
}
log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID)
}
counts, err := audit.CountByStatus(cctx)
if err == nil {
observability.RecordJobAuditDepth(counts)
}
}
go func() {
run()
t := time.NewTicker(interval)
defer t.Stop()
for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
return return
case <-t.C: case <-ticker.C:
run()
} }
} }
}() }()
} }
// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim.
func (r *Registry) reapExpiredJobsLoop(ctx context.Context) {
defer r.durableLoopWG.Done()
r.reapExpiredJobs(ctx)
reap := time.NewTicker(15 * time.Second)
stop := time.NewTicker(time.Second)
defer reap.Stop()
defer stop.Stop()
for {
if r.durableStopped() || ctx.Err() != nil {
return
}
select {
case <-ctx.Done():
return
case <-stop.C:
if r.durableStopped() {
return
}
case <-reap.C:
r.reapExpiredJobs(ctx)
}
}
}
func (r *Registry) durableStopped() bool {
if r == nil {
return true
}
r.durableMu.Lock()
defer r.durableMu.Unlock()
return r.durableStop
}
func (r *Registry) reapExpiredJobs(ctx context.Context) {
if r == nil || r.queue == nil {
return
}
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil {
logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err)
}
}
func (r *Registry) runClaimed(j *Job) {
log := logging.With("component", "jobs.durable")
var leaseHealthy atomic.Bool
leaseHealthy.Store(true)
defer func() {
r.mu.Lock()
delete(r.byID, j.ID)
r.mu.Unlock()
<-r.workerSem
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
r.durableWG.Done()
}()
stop := make(chan struct{})
done := make(chan struct{})
go func() {
defer close(done)
t := time.NewTicker(durableLease / 4)
defer t.Stop()
for {
select {
case <-stop:
return
case <-t.C:
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease)
if err == nil {
err = r.queue.CheckpointJob(ctx, recordFromJob(j))
}
cancel()
if err != nil {
leaseHealthy.Store(false)
j.RequestCancel()
log.Warn("heartbeat failed", "job_id", j.ID, "err", err)
return
}
if requested {
j.RequestCancel()
}
}
}
}()
r.workerStart(j)
close(stop)
<-done
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
r.durableMu.Lock()
stopped := r.durableStop
r.durableMu.Unlock()
c := recordFromJob(j)
if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed {
if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
log.Warn("release failed", "err", err)
}
return
}
if !leaseHealthy.Load() {
// Leave an uncertain lease for expiry/reclaim. This consumes an attempt,
// whereas a deliberate graceful shutdown releases without using the budget.
return
}
if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled {
c.Status = StatusFailed
msg := "worker returned without terminal status"
c.Error = &msg
}
if err := r.queue.FinishJob(ctx, c); err != nil {
log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err)
}
}
+132
View File
@@ -0,0 +1,132 @@
package jobs
import (
"context"
"errors"
"sync/atomic"
"testing"
"time"
"evobgp/internal/store"
)
type blockingClaimQueue struct {
store.JobQueue
entered chan int
resume chan struct{}
}
func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
q.entered <- limit
<-q.resume
return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
}
func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
m := store.NewMemory()
queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
if err != nil {
t.Fatal(err)
}
q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
var runs atomic.Int32
r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
r.UseDurableQueue(q)
StartDurableQueueLoop(context.Background(), r)
select {
case limit := <-q.entered:
if limit != cap(r.workerSem) {
t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
}
case <-time.After(3 * time.Second):
t.Fatal("no claim")
}
r.RequestCancelAll()
drained := make(chan error, 1)
go func() { drained <- r.Drain(context.Background()) }()
select {
case <-drained:
t.Fatal("drained before in-flight claim returned")
case <-time.After(20 * time.Millisecond):
}
close(q.resume)
select {
case err := <-drained:
if err != nil {
t.Fatal(err)
}
case <-time.After(3 * time.Second):
t.Fatal("shutdown did not drain")
}
if runs.Load() != 0 {
t.Fatal("started worker after stop")
}
got, err := m.GetJob(context.Background(), "tenant", queued.ID)
if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
t.Fatalf("not released: %+v %v", got, err)
}
}
func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
ctx := context.Background()
m := store.NewMemory()
key := "request"
a := NewRegistry(nil)
a.UseDurableQueue(m)
j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
if err != nil {
t.Fatal(err)
}
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
if err != nil || len(claimed) != 1 {
t.Fatalf("claim %+v %v", claimed, err)
}
claimed[0].Status = StatusSucceeded
claimed[0].Meta = map[string]any{"result": "ok"}
if err = m.FinishJob(ctx, claimed[0]); err != nil {
t.Fatal(err)
}
b := NewRegistry(nil)
b.UseDurableQueue(m)
got, err := b.GetContext(ctx, "tenant", j.ID)
if err != nil || got.Status != StatusSucceeded {
t.Fatalf("shared get %+v %v", got, err)
}
list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
if err != nil || len(list) != 1 {
t.Fatalf("shared list %+v %v", list, err)
}
replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
t.Fatalf("terminal replay %+v %v %v", replay, created, err)
}
if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
t.Fatalf("changed payload accepted: %v", err)
}
}
func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
ctx := context.Background()
m := store.NewMemory()
r := NewRegistry(nil)
r.UseDurableQueue(m)
parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
if err != nil {
t.Fatal(err)
}
key := "deploy_from:" + parent.ID
child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
if err != nil {
t.Fatal(err)
}
worker := &Worker{Store: m, Registry: r}
worker.Process(parent)
if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
t.Fatalf("did not recover child: %+v", parent.Snapshot())
}
items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
if err != nil || len(items) != 1 {
t.Fatalf("duplicate child: %+v %v", items, err)
}
}
+100 -3
View File
@@ -11,6 +11,7 @@ import (
"time" "time"
"evobgp/internal/observability" "evobgp/internal/observability"
"evobgp/internal/store"
"github.com/google/uuid" "github.com/google/uuid"
) )
@@ -38,6 +39,11 @@ type Job struct {
Error *string Error *string
ProgressPct *int16 ProgressPct *int16
Meta map[string]any Meta map[string]any
OwnerID string
FencingToken int64
RequestSequence int64
Attempt int
requestHash string
cancelRequested bool cancelRequested bool
mu sync.Mutex mu sync.Mutex
} }
@@ -225,6 +231,30 @@ type Registry struct {
// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic // have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
// deploy coalescing under tenantRefreshMu (instead of polling job statuses). // deploy coalescing under tenantRefreshMu (instead of polling job statuses).
inflightRefresh map[string]int inflightRefresh map[string]int
queue store.JobQueue
ownerID string
durableMu sync.Mutex
durableWG sync.WaitGroup
durableLoopWG sync.WaitGroup
durableStop bool
}
// UseDurableQueue makes the shared store authoritative for new work and job reads.
func (r *Registry) UseDurableQueue(q store.JobQueue) {
r.mu.Lock()
defer r.mu.Unlock()
r.queue = q
r.ownerID = uuid.NewString()
}
func jobFromRecord(c store.JobRecord) *Job {
return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested}
}
func recordFromJob(j *Job) store.JobRecord {
j.mu.Lock()
defer j.mu.Unlock()
return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested}
} }
type idempoKey struct { type idempoKey struct {
@@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int {
return 8 return 8
} }
func jobRegistryPendingLimit() int {
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 {
return n
}
return 10000
}
func (r *Registry) ActiveCountUnsafe() int {
n := 0
for _, j := range r.byID {
st := j.statusLocked()
if st == StatusQueued || st == StatusRunning {
n++
}
}
return n
}
// pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs. // pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
func (r *Registry) pruneTerminalIfOver(maxJobs int) { func (r *Registry) pruneTerminalIfOver(maxJobs int) {
if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs { if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
@@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
// Enqueue creates a job or returns an existing one for the same idempotency key. // Enqueue creates a job or returns an existing one for the same idempotency key.
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) { func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta)
}
// EnqueueContext commits a durable job before returning it to an HTTP caller.
func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil)
}
// EnqueueRequestContext preserves idempotency of dynamically resolved selectors.
func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) {
if r.queue != nil {
r.durableMu.Lock()
stopped := r.durableStop
r.durableMu.Unlock()
if stopped {
return nil, false, fmt.Errorf("jobs: shutting down")
}
c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
if err != nil {
return nil, false, err
}
return jobFromRecord(c), created, nil
}
hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
if err != nil {
return nil, false, err
}
r.mu.Lock() r.mu.Lock()
maxJobs := registryMaxJobsFromEnv() maxJobs := registryMaxJobsFromEnv()
r.pruneTerminalIfOver(maxJobs) r.pruneTerminalIfOver(maxJobs)
@@ -359,13 +434,16 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
if idempotencyKey != nil && *idempotencyKey != "" { if idempotencyKey != nil && *idempotencyKey != "" {
k := idempoKey{tenant: tenantID, key: *idempotencyKey} k := idempoKey{tenant: tenantID, key: *idempotencyKey}
if existing, ok := r.byIdempo[k]; ok { if existing, ok := r.byIdempo[k]; ok {
st := existing.statusLocked()
if st == StatusQueued || st == StatusRunning {
r.mu.Unlock() r.mu.Unlock()
if existing.requestHash != hash {
return nil, false, store.ErrJobConflict
}
return existing, false, nil return existing, false, nil
} }
delete(r.byIdempo, k)
} }
if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() {
r.mu.Unlock()
return nil, false, store.ErrJobQueueFull
} }
j := &Job{ j := &Job{
@@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
ModuleID: moduleID, ModuleID: moduleID,
CreatedAt: time.Now().UTC(), CreatedAt: time.Now().UTC(),
Meta: cloneMeta(meta), Meta: cloneMeta(meta),
requestHash: hash,
} }
if idempotencyKey != nil && *idempotencyKey != "" { if idempotencyKey != nil && *idempotencyKey != "" {
r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
@@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int {
if r == nil { if r == nil {
return 0 return 0
} }
if r.queue != nil {
r.durableMu.Lock()
r.durableStop = true
r.durableMu.Unlock()
}
r.mu.RLock() r.mu.RLock()
defer r.mu.RUnlock() defer r.mu.RUnlock()
n := 0 n := 0
@@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int {
j.RequestCancel() j.RequestCancel()
n++ n++
} }
if r.queue != nil && len(r.workerSem) > n {
n = len(r.workerSem)
}
return n return n
} }
@@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error {
if r == nil { if r == nil {
return nil return nil
} }
if r.queue != nil {
done := make(chan struct{})
go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }()
select {
case <-done:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
ticker := time.NewTicker(50 * time.Millisecond) ticker := time.NewTicker(50 * time.Millisecond)
defer ticker.Stop() defer ticker.Stop()
for { for {
+115
View File
@@ -0,0 +1,115 @@
package jobs
import (
"context"
"errors"
"evobgp/internal/store"
)
// FindIdempotentContext looks up a retained logical operation without inserting.
func (r *Registry) FindIdempotentContext(ctx context.Context, tenant, key string) (*Job, error) {
if r.queue != nil {
c, err := r.queue.FindJobByIdempotencyKey(ctx, tenant, key)
if err != nil {
return nil, err
}
j := jobFromRecord(c)
j.requestHash = c.RequestHash
return j, nil
}
r.mu.RLock()
defer r.mu.RUnlock()
j := r.byIdempo[idempoKey{tenant: tenant, key: key}]
if j == nil {
return nil, store.ErrJobNotFound
}
return j, nil
}
// ReplayContext validates an immutable request before expensive live resolution.
func (r *Registry) ReplayContext(ctx context.Context, in store.JobEnqueue) (*Job, bool, error) {
if in.IdempotencyKey == nil || *in.IdempotencyKey == "" {
return nil, false, nil
}
j, err := r.FindIdempotentContext(ctx, in.TenantID, *in.IdempotencyKey)
if errors.Is(err, store.ErrJobNotFound) {
return nil, false, nil
}
if err != nil {
return nil, false, err
}
h, err := store.JobRequestHash(in)
if err != nil {
return nil, false, err
}
if j.requestHash != h {
return nil, false, store.ErrJobConflict
}
return j, true, nil
}
// GetContext reads a job from the shared store when durability is enabled.
func (r *Registry) GetContext(ctx context.Context, tenant, id string) (*Job, error) {
if r.queue == nil {
return r.Get(tenant, id)
}
c, err := r.queue.GetJob(ctx, tenant, id)
if errors.Is(err, store.ErrJobNotFound) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
return jobFromRecord(c), nil
}
// ListContext returns a bounded tenant page from the authoritative queue.
func (r *Registry) ListContext(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]*Job, string, bool, error) {
if r.queue == nil {
j, n, m := r.List(tenant, status, kind, cursor, limit)
return j, n, m, nil
}
c, next, more, err := r.queue.ListJobs(ctx, tenant, status, kind, cursor, limit)
if err != nil {
return nil, "", false, err
}
out := make([]*Job, 0, len(c))
for _, v := range c {
out = append(out, jobFromRecord(v))
}
return out, next, more, nil
}
// RequestCancelContext persists a cancel request visible to any worker replica.
func (r *Registry) RequestCancelContext(ctx context.Context, tenant, id string) (*Job, error) {
if r.queue == nil {
return r.RequestCancel(tenant, id)
}
c, err := r.queue.CancelJob(ctx, tenant, id)
if errors.Is(err, store.ErrJobNotFound) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
r.mu.RLock()
local := r.byID[id]
r.mu.RUnlock()
if local != nil {
local.RequestCancel()
}
return jobFromRecord(c), nil
}
// RetryContext schedules an explicit retry of a failed shared operation.
func (r *Registry) RetryContext(ctx context.Context, tenant, id string) (*Job, error) {
if r.queue == nil {
return nil, store.ErrJobConflict
}
c, err := r.queue.RetryJob(ctx, tenant, id)
if err != nil {
return nil, err
}
return jobFromRecord(c), nil
}
+570 -122
View File
@@ -2,6 +2,7 @@ package jobs
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"net/http" "net/http"
"net/netip" "net/netip"
@@ -69,10 +70,15 @@ type Worker struct {
Store store.Backend Store store.Backend
PgPool *pgxpool.Pool PgPool *pgxpool.Pool
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout). HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback. // Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
Registry *Registry Registry *Registry
// refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess). // refreshGate decides which module refresh renders for a tenant.
refreshGate sync.Map // map[string]*sync.Mutex // The mutex covers only that decision; the render itself runs outside the lock.
refreshGate sync.Map // map[string]*tenantRefreshGate
// dispatchGate serializes publish and deployment state for one speaker in this process.
// Agent wake runs outside the mutex.
dispatchGate sync.Map // map[string]*sync.Mutex
} }
type revisionLogEntry struct { type revisionLogEntry struct {
@@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) {
j.MarkCancelled() j.MarkCancelled()
return return
} }
// A committed child is the durable checkpoint for a parent that crashed
// before its own terminal write. Reuse it before rendering another revision.
if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
if err == nil {
j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
if isRefreshKind(j.Kind) {
w.Registry.finalizeRefreshCoalesce(j.TenantID)
}
j.Succeed()
return
}
if !errors.Is(err, store.ErrJobNotFound) {
j.Fail(err.Error())
return
}
}
switch j.Kind { switch j.Kind {
case KindModuleRefresh: case KindModuleRefresh:
@@ -185,17 +208,17 @@ func (w *Worker) runPeerReconcile(j *Job) {
const peerJobTitle = "Обновление BGP пиров" const peerJobTitle = "Обновление BGP пиров"
j.mergeMeta(map[string]any{"job_title": peerJobTitle}) j.mergeMeta(map[string]any{"job_title": peerJobTitle})
var revID string
latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1) latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest) triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
if err != nil { if err != nil {
j.Fail(err.Error()) j.Fail(err.Error())
return return
} }
if len(latest) == 0 { // Peer edits bump render generation. Render from module snapshots so the
// new revision includes current prefixes without paging the previous one.
ctx, cancel := j.workContext() ctx, cancel := j.workContext()
defer cancel() defer cancel()
rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
if err != nil { if err != nil {
if ctx.Err() != nil { if ctx.Err() != nil {
j.MarkCancelled() j.MarkCancelled()
@@ -204,32 +227,6 @@ func (w *Worker) runPeerReconcile(j *Job) {
j.Fail(err.Error()) j.Fail(err.Error())
return return
} }
revID = rid
} else {
baseRevID := latest[0].ID
rows := make([]store.PrefixRow, 0, 1024)
cursor := ""
for {
page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000)
rows = append(rows, page...)
if !more || strings.TrimSpace(next) == "" {
break
}
cursor = next
}
ctx, cancel := j.workContext()
defer cancel()
rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
if err != nil {
if ctx.Err() != nil {
j.MarkCancelled()
return
}
j.Fail(err.Error())
return
}
revID = rid
}
j.mergeMeta(map[string]any{"revision_id": revID}) j.mergeMeta(map[string]any{"revision_id": revID})
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil { if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
j.mergeMeta(map[string]any{ j.mergeMeta(map[string]any{
@@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) {
} else { } else {
j.mergeMeta(map[string]any{"log_build_error": err.Error()}) j.mergeMeta(map[string]any{"log_build_error": err.Error()})
} }
w.enqueueDeployAllSpeakers(j, j.TenantID, revID) if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
j.Fail(err.Error())
return
}
j.Succeed() j.Succeed()
} }
@@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision)
return "", fmt.Errorf("missing module_id for peer reconcile") return "", fmt.Errorf("missing module_id for peer reconcile")
} }
type tenantRefreshGate struct {
mu sync.Mutex
rendering bool
dirty bool
}
func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
return v.(*tenantRefreshGate)
}
func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex { func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{}) return &w.tenantRefreshGate(tenantID).mu
}
func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
return v.(*sync.Mutex) return v.(*sync.Mutex)
} }
@@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) {
moduleIDs := moduleIDsFromJobMeta(j.metaCopy()) moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
if len(moduleIDs) == 0 { if len(moduleIDs) == 0 {
j.Fail("missing module_ids in job meta") j.mergeMeta(map[string]any{"modules_refreshed": 0})
j.Succeed()
return return
} }
ctx, cancel := j.workContext() ctx, cancel := j.workContext()
@@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
j.Succeed() j.Succeed()
return return
} }
mu := w.tenantRefreshMu(j.TenantID) gate := w.tenantRefreshGate(j.TenantID)
mu.Lock() gate.mu.Lock()
defer mu.Unlock() // The counter is incremented in Enqueue and decremented here, so the "last one"
ctx, cancel := j.workContext() // decision does not depend on job status polling. Render runs after the unlock.
defer cancel()
// Determine whether this is the last outstanding refresh for the tenant. The counter is
// incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is
// deterministic regardless of ingest timing — unlike the previous status-polling approach
// (CountOtherActiveRefresh) which could race under -race.
isLastRefresh := true isLastRefresh := true
if w.Registry != nil { if w.Registry != nil {
isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID) isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
} }
if !isLastRefresh { if !isLastRefresh || gate.rendering {
if gate.rendering {
gate.dirty = true
}
gate.mu.Unlock()
j.mergeMeta(map[string]any{ j.mergeMeta(map[string]any{
"deploy_apply_deferred": true, "deploy_apply_deferred": true,
"deploy_apply_defer_reason": "parallel_module_refresh", "deploy_apply_defer_reason": "parallel_module_refresh",
@@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
j.Succeed() j.Succeed()
return return
} }
gate.rendering = true
gate.mu.Unlock()
defer func() {
gate.mu.Lock()
gate.rendering = false
gate.mu.Unlock()
}()
rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID) ctx, cancel := j.workContext()
defer cancel()
rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
if err != nil { if err != nil {
if ctx.Err() != nil { if errors.Is(err, errRenderSuperseded) {
j.Fail(err.Error())
return
}
if ctx.Err() != nil || j.IsCancelRequested() {
j.MarkCancelled() j.MarkCancelled()
return return
} }
j.Fail(err.Error()) j.Fail(err.Error())
return return
} }
if ctx.Err() != nil || j.IsCancelRequested() {
j.MarkCancelled()
return
}
j.mergeMeta(map[string]any{"revision_id": rev}) j.mergeMeta(map[string]any{"revision_id": rev})
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil { if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
j.mergeMeta(map[string]any{ j.mergeMeta(map[string]any{
@@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
} else { } else {
j.mergeMeta(map[string]any{"log_build_error": err.Error()}) j.mergeMeta(map[string]any{"log_build_error": err.Error()})
} }
w.enqueueDeployAllSpeakers(j, j.TenantID, rev) if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
j.Fail(err.Error())
return
}
j.Succeed() j.Succeed()
} }
const (
// renderQuietPassLimit stops a render loop that never observes a quiet input set.
// Hitting the cap fails the job; the last revision must not be published.
renderQuietPassLimit = 32
// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
speakerWakeLimit = 4
)
// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
// landed, the job was cancelled while inputs were still changing, or the quiet
// pass cap was exhausted.
var errRenderSuperseded = errors.New("newer snapshot still pending")
// renderTenantUntilQuiet builds a revision outside the tenant gate.
// It keeps rendering while another ingest marks the gate dirty or the snapshot
// generation moves, and returns a revision only after a fresh check shows the
// inputs are still the ones just rendered. Cancellation or the safety cap while
// inputs are still changing fails the render instead of publishing a stale revision.
func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
gate := w.tenantRefreshGate(j.TenantID)
var rev string
for pass := 0; pass < renderQuietPassLimit; pass++ {
if abort := renderJobAborted(ctx, j); abort != nil {
if takeRefreshDirty(gate) {
return "", errRenderWhileDirty(abort)
}
return "", abort
}
before, err := w.captureRenderInputs(ctx, j.TenantID)
if err != nil {
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
return "", errRenderWhileDirty(abort)
}
return "", err
}
rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
if err != nil {
dirty := takeRefreshDirty(gate)
if abort := renderJobAborted(ctx, j); abort != nil && dirty {
return "", errRenderWhileDirty(abort)
}
if abort := renderJobAborted(ctx, j); abort != nil {
return "", abort
}
if dirty {
continue
}
return "", err
}
after, err := w.captureRenderInputs(ctx, j.TenantID)
if err != nil {
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
return "", errRenderWhileDirty(abort)
}
return "", err
}
dirty := takeRefreshDirty(gate)
if dirty || renderInputsChanged(before, after) {
if abort := renderJobAborted(ctx, j); abort != nil {
return "", errRenderWhileDirty(abort)
}
continue
}
confirm, err := w.captureRenderInputs(ctx, j.TenantID)
if err != nil {
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
return "", errRenderWhileDirty(abort)
}
return "", err
}
if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
if abort := renderJobAborted(ctx, j); abort != nil {
return "", errRenderWhileDirty(abort)
}
continue
}
if abort := renderJobAborted(ctx, j); abort != nil {
return "", abort
}
return rev, nil
}
if abort := renderJobAborted(ctx, j); abort != nil {
return "", errRenderWhileDirty(abort)
}
return "", errRenderWhileDirty(nil)
}
func errRenderWhileDirty(cause error) error {
if cause == nil {
return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
}
return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
}
func renderJobAborted(ctx context.Context, j *Job) error {
if err := ctx.Err(); err != nil {
return err
}
if j != nil && j.IsCancelRequested() {
return context.Canceled
}
return nil
}
func takeRefreshDirty(gate *tenantRefreshGate) bool {
if gate == nil {
return false
}
gate.mu.Lock()
defer gate.mu.Unlock()
dirty := gate.dirty
gate.dirty = false
return dirty
}
type renderInputView struct {
generation int64
snapshots []store.ModuleSnapshotUse
}
func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
if w == nil || w.Store == nil {
return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
}
gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
if err != nil {
if !errors.Is(err, store.ErrNotFound) {
return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
}
gen = 0
}
headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
if err != nil {
return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
}
return renderInputView{generation: gen, snapshots: headers}, nil
}
func renderInputsChanged(before, after renderInputView) bool {
if before.generation != after.generation {
return true
}
return !sameSnapshotHeaders(before.snapshots, after.snapshots)
}
func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
if len(a) != len(b) {
return false
}
aa := append([]store.ModuleSnapshotUse(nil), a...)
bb := append([]store.ModuleSnapshotUse(nil), b...)
sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
for i := range aa {
if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
return false
}
}
return true
}
// enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id). // enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) { func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
if w == nil || w.Registry == nil { if w == nil || w.Registry == nil {
return return fmt.Errorf("jobs: deploy queue unavailable")
} }
revID = strings.TrimSpace(revID) revID = strings.TrimSpace(revID)
if revID == "" { if revID == "" {
return return fmt.Errorf("jobs: missing revision for deploy")
} }
meta := map[string]any{"revision_id": revID} meta := map[string]any{"revision_id": revID}
if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" { if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
meta["trigger"] = tr meta["trigger"] = tr
} }
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta) key := "deploy_from:" + j.ID
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
if err != nil { if err != nil {
j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()}) j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
return return fmt.Errorf("jobs: enqueue deploy: %w", err)
} }
if applyJob != nil { if applyJob != nil {
j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID}) j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
} }
return nil
} }
func (w *Worker) runDeployApply(j *Job) { func (w *Worker) runDeployApply(j *Job) {
revID := strings.TrimSpace(j.metaString("revision_id")) revID := strings.TrimSpace(j.metaString("revision_id"))
spk := strings.TrimSpace(j.metaString("speaker_id")) spk := strings.TrimSpace(j.metaString("speaker_id"))
hasSpeaker := spk != ""
if revID == "" { if revID == "" {
j.Fail("missing revision_id in job meta") j.Fail("missing revision_id in job meta")
return return
} }
if j.OwnerID == "" {
if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
j.Fail("deployment requires a current durable job lease")
return
}
}
ctx, cancel := j.workContext() ctx, cancel := j.workContext()
defer cancel() defer cancel()
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")) cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
if activeDir != "" { var speakers []*store.Speaker
revObj, err := w.Store.GetRevision(j.TenantID, revID) if spk != "" {
sp, err := w.Store.GetSpeaker(j.TenantID, spk)
if err != nil { if err != nil {
j.Fail(err.Error()) j.Fail(err.Error())
return return
} }
speakers = []*store.Speaker{sp}
} else {
speakers = w.Store.ListSpeakersForTenant(j.TenantID)
}
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
localAttempted := false
var localErr error
applyLocal := func() error {
if localAttempted {
return localErr
}
localAttempted = true
revObj, err := w.Store.GetRevision(j.TenantID, revID)
if err != nil {
localErr = err
return err
}
staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR")) staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
if staging == "" { if staging == "" {
staging = os.TempDir() + "/evobgp-bird-staging" staging = os.TempDir() + "/evobgp-bird-staging"
@@ -480,39 +702,292 @@ func (w *Worker) runDeployApply(j *Job) {
} }
ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket} ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil { if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
if ctx.Err() != nil { localErr = err
return err
}
return nil
}
tally := &deployTally{applied: make([]string, 0, len(speakers))}
slots := make(chan struct{}, speakerWakeLimit)
var wakeGroup sync.WaitGroup
acquireWake := func() bool {
select {
case slots <- struct{}{}:
case <-ctx.Done():
return false
}
if cancelled() {
<-slots
return false
}
return true
}
for _, sp := range speakers {
if cancelled() {
break
}
if sp == nil {
continue
}
held := false
if speakerRemoteWake(sp) {
if !acquireWake() {
break
}
held = true
}
staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
if staged.wake && !staged.stop {
wakeGroup.Add(1)
go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
defer wakeGroup.Done()
defer func() { <-slots }()
wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
cancelWake()
out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
}(sp, staged.deployment)
} else if held {
<-slots
}
tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
if staged.stop {
break
}
}
wakeGroup.Wait()
results, applied, published, failed, unconfirmed := tally.snapshot()
j.mergeMeta(map[string]any{
"apply_summary": map[string]any{
"revision_id": revID,
"speakers_count": len(speakers),
"published_count": published,
"applied_count": len(applied),
"unconfirmed_count": unconfirmed,
"failed_count": failed,
"speaker_ids": applied,
"message": fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
},
"node_dispatch": map[string]any{"revision_id": revID, "results": results},
})
if cancelled() {
j.MarkCancelled() j.MarkCancelled()
return return
} }
j.Fail(err.Error()) mergeBirdPostApplyMeta(j)
if cancelled() {
j.MarkCancelled()
return return
} }
if failed > 0 {
j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
return
} }
applied := make([]string, 0, 8) j.Succeed()
var dispatchResults []nodedispatch.Result
applyOne := func(speakerID string) error {
if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil {
return err
} }
// Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy.
if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil { type speakerStage struct {
return err stop bool
wake bool
deployment store.DeploymentSpeaker
result nodedispatch.Result
hasResult bool
published int
failed int
unconfirmed int
appliedID string
} }
applied = append(applied, speakerID)
type deployTally struct {
mu sync.Mutex
results []nodedispatch.Result
applied []string
published int
failed int
unconfirmed int
}
func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
t.mu.Lock()
defer t.mu.Unlock()
if include {
t.results = append(t.results, res)
}
t.published += published
t.failed += failed
t.unconfirmed += unconfirmed
if appliedID != "" {
t.applied = append(t.applied, appliedID)
}
}
func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
t.mu.Lock()
defer t.mu.Unlock()
results = append([]nodedispatch.Result(nil), t.results...)
applied = append([]string(nil), t.applied...)
if applied == nil {
applied = []string{}
}
sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
sort.Strings(applied)
return results, applied, t.published, t.failed, t.unconfirmed
}
func speakerRemoteWake(sp *store.Speaker) bool {
if sp == nil || !nodedispatch.Enabled() {
return false
}
return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
}
func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
fence := deploymentFenceForJob(j)
record := func(state, confirmation, message string) error {
if j.OwnerID == "" {
return nil return nil
} }
dispatchSpeaker := func(sp *store.Speaker) { return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
if !nodedispatch.Enabled() || sp == nil { }
return markApplied := func(confirmation string) error {
if j.OwnerID == "" {
return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
}
return record("applied", confirmation, "")
}
return record, markApplied
}
// stageSpeakerDeploy publishes one speaker and records terminal states.
// Remote wakes are left to the caller so the speaker lock is not held across the network.
func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
var staged speakerStage
mu := w.speakerDispatchMu(sp.ID)
mu.Lock()
defer mu.Unlock()
if cancelled() {
staged.stop = true
return staged
}
var deployment store.DeploymentSpeaker
var publishErr error
if j.OwnerID != "" {
deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
} else {
publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
}
staged.deployment = deployment
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
if publishErr != nil {
staged.failed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
return staged
}
staged.published = 1
if deployment.State == "applied" {
staged.appliedID = sp.ID
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
return staged
}
if cancelled() {
staged.stop = true
return staged
}
if !speakerRemoteWake(sp) {
if activeDir != "" && strings.EqualFold(sp.Role, "master") {
err := applyLocal()
if cancelled() {
staged.stop = true
return staged
}
if err != nil {
_ = record("failed", "unconfirmed", err.Error())
staged.failed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
return staged
}
if err := markApplied("local_applied"); err != nil {
staged.failed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
return staged
}
staged.appliedID = sp.ID
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
return staged
}
if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
staged.failed = 1
}
staged.unconfirmed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
return staged
}
if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
staged.failed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
return staged
}
if deployment.DispatchAttempts >= 3 {
_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
staged.failed = 1
staged.hasResult = true
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
return staged
}
staged.wake = true
return staged
}
// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
// state update and is not held while WakeSpeaker is in flight.
func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
mu := w.speakerDispatchMu(sp.ID)
mu.Lock()
defer mu.Unlock()
if cancelled != nil && cancelled() {
return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
}
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
appliedID := ""
if res.Status == "ok" && res.AppliedRevisionID != "" {
if res.AppliedRevisionID != revID {
res.Status = "error"
res.Confirmation = "unconfirmed"
res.Error = "agent reported a different applied revision"
} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
res.Status = "error"
res.Confirmation = "unconfirmed"
res.Error = err.Error()
} else if latest != revID {
res.Status = "error"
res.Confirmation = "unconfirmed"
res.Error = "stale agent acknowledgement: desired revision changed"
} else if err := markApplied("agent_reported"); err != nil {
res.Status = "error"
res.Confirmation = "unconfirmed"
res.Error = err.Error()
} else {
appliedID = sp.ID
}
}
if res.Status != "ok" || res.Confirmation != "agent_reported" {
state := "unconfirmed"
if res.Status != "ok" {
state = "unknown"
}
if err := record(state, "unconfirmed", res.Error); err != nil {
res.Status = "error"
res.Error = err.Error()
} }
meta := store.ParseSpeakerMeta(sp.MetaJSON)
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
return
} }
ctx2, cancel := context.WithTimeout(ctx, 35*time.Second)
defer cancel()
res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID})
dispatchResults = append(dispatchResults, res)
patch := store.SpeakerMeta{ patch := store.SpeakerMeta{
LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano), LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano),
LastDispatchStatus: res.Status, LastDispatchStatus: res.Status,
@@ -520,57 +995,27 @@ func (w *Worker) runDeployApply(j *Job) {
if res.Error != "" { if res.Error != "" {
patch.LastDispatchError = res.Error patch.LastDispatchError = res.Error
patch.SyncStatus = "error" patch.SyncStatus = "error"
} else if res.Status == "ok" { } else if res.Status == "ok" && res.Confirmation == "agent_reported" {
patch.LastDispatchError = "" patch.LastDispatchError = ""
patch.SyncStatus = "synced" patch.SyncStatus = "synced"
} else if res.Status == "ok" {
patch.SyncStatus = "unconfirmed"
} }
merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch) merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
_, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}) if j.OwnerID == "" {
} if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
if hasSpeaker && spk != "" { res.Status = "error"
if err := applyOne(spk); err != nil { res.Error = "dispatch metadata update: " + err.Error()
j.Fail(err.Error())
return
}
if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil {
dispatchSpeaker(sp)
}
if len(dispatchResults) > 0 {
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
"revision_id": revID,
"results": dispatchResults,
}})
}
mergeBirdPostApplyMeta(j)
j.Succeed()
return
}
speakers := w.Store.ListSpeakersForTenant(j.TenantID)
for _, sp := range speakers {
if err := applyOne(sp.ID); err != nil {
j.Fail(err.Error())
return
} }
} }
for _, sp := range speakers { failed := 0
dispatchSpeaker(sp) unconfirmed := 0
if res.Status == "ok" && res.Confirmation == "unconfirmed" {
unconfirmed = 1
} else if res.Status != "ok" {
failed = 1
} }
j.mergeMeta(map[string]any{ return res, true, failed, unconfirmed, appliedID
"apply_summary": map[string]any{
"revision_id": revID,
"speakers_count": len(applied),
"speaker_ids": applied,
"message": fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)),
},
})
if len(dispatchResults) > 0 {
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
"revision_id": revID,
"results": dispatchResults,
}})
}
mergeBirdPostApplyMeta(j)
j.Succeed()
} }
func (w *Worker) runRollback(j *Job) { func (w *Worker) runRollback(j *Job) {
@@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) {
"message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)), "message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
}, },
}) })
w.enqueueDeployAllSpeakers(j, j.TenantID, newID) if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
j.Fail(err.Error())
return
}
j.Succeed() j.Succeed()
} }
+187
View File
@@ -1,6 +1,9 @@
package jobs package jobs
import ( import (
"context"
"fmt"
"strings"
"testing" "testing"
"time" "time"
@@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) {
t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta) t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
} }
} }
type renderHookStore struct {
*store.Memory
onCreate func(prefixes []store.PrefixRow)
}
func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error {
if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil {
return err
}
if s.onCreate != nil {
copied := append([]store.PrefixRow(nil), prefixes...)
s.onCreate(copied)
}
return nil
}
func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) {
t.Helper()
t.Setenv("EVOBGP_ASN_RESOLVE", "0")
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
mem = store.NewMemory()
mem.SeedDemo()
var modIP string
tenant, _, modIP, _, _ = mem.DemoIDs()
module = modIP
for _, mod := range mem.ListModules(tenant) {
if mod == nil || mod.ID == module {
continue
}
disabled := false
if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
t.Fatal(err)
}
}
return tenant, module, mem
}
func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) {
t.Helper()
err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{
Prefix: prefix,
Source: "ip_range",
}})
if err != nil {
t.Fatalf("snapshot %s: %v", prefix, err)
}
}
func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool {
t.Helper()
rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100)
for _, row := range rows {
if row.Prefix == prefix {
return true
}
}
return false
}
func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) {
t.Run("latest snapshot", func(t *testing.T) {
tenant, module, mem := newIPRenderFixture(t)
prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"}
installModuleSnapshot(t, mem, tenant, module, prefixes[0])
hook := &renderHookStore{Memory: mem}
var wk *Worker
step := 0
hook.onCreate = func([]store.PrefixRow) {
step++
if step >= len(prefixes) {
return
}
installModuleSnapshot(t, mem, tenant, module, prefixes[step])
gate := wk.tenantRefreshGate(tenant)
gate.mu.Lock()
gate.dirty = true
gate.mu.Unlock()
}
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
wk.finishModuleRefreshSuccess(j, module)
if j.Status != StatusSucceeded {
errText := ""
if j.Error != nil {
errText = *j.Error
}
t.Fatalf("status=%s err=%s", j.Status, errText)
}
if step != len(prefixes) {
t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes))
}
rev := j.metaString("revision_id")
if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) {
t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1])
}
if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) {
t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2])
}
deployID := j.metaString("deploy_apply_job_id")
deploy, err := wk.Registry.Get(tenant, deployID)
if err != nil {
t.Fatal(err)
}
if deploy.metaString("revision_id") != rev {
t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev)
}
})
t.Run("cancel does not deploy", func(t *testing.T) {
tenant, module, mem := newIPRenderFixture(t)
installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24")
hook := &renderHookStore{Memory: mem}
var wk *Worker
var j *Job
commits := 0
hook.onCreate = func([]store.PrefixRow) {
commits++
installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24")
gate := wk.tenantRefreshGate(tenant)
gate.mu.Lock()
gate.dirty = true
gate.mu.Unlock()
j.RequestCancel()
}
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
wk.finishModuleRefreshSuccess(j, module)
if j.Status != StatusFailed {
t.Fatalf("status=%s, want failed", j.Status)
}
if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") {
t.Fatalf("error=%v, want superseded render", j.Error)
}
if commits != 1 {
t.Fatalf("render commits=%d, want 1", commits)
}
if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" {
t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy())
}
deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10)
if len(deploys) != 0 {
t.Fatalf("deploy jobs=%d, want 0", len(deploys))
}
revs, _, _ := mem.ListRevisions(tenant, "", "", 50)
for _, rev := range revs {
if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") {
t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID)
}
}
})
t.Run("cap does not publish", func(t *testing.T) {
tenant, module, mem := newIPRenderFixture(t)
installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24")
hook := &renderHookStore{Memory: mem}
var wk *Worker
commits := 0
hook.onCreate = func([]store.PrefixRow) {
commits++
installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits))
gate := wk.tenantRefreshGate(tenant)
gate.mu.Lock()
gate.dirty = true
gate.mu.Unlock()
}
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
wk.finishModuleRefreshSuccess(j, module)
if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") {
errText := ""
if j.Error != nil {
errText = *j.Error
}
t.Fatalf("status=%s err=%s", j.Status, errText)
}
if commits != renderQuietPassLimit {
t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit)
}
if j.metaString("deploy_apply_job_id") != "" {
t.Fatal("safety cap published a deploy")
}
})
}
+10 -8
View File
@@ -107,14 +107,14 @@ func LookupWithResolver(
if addr, err := netip.ParseAddr(raw); err == nil { if addr, err := netip.ParseAddr(raw); err == nil {
out.QueryKind = KindIP out.QueryKind = KindIP
out.Normalized = addr.String() out.Normalized = addr.String()
if err := lookupIP(st, tenantID, addr, out, commByID, ""); err != nil { if err := lookupIP(ctx, st, tenantID, addr, out, commByID, ""); err != nil {
return nil, err return nil, err
} }
} else if pfx, err := netip.ParsePrefix(raw); err == nil { } else if pfx, err := netip.ParsePrefix(raw); err == nil {
masked := pfx.Masked() masked := pfx.Masked()
out.QueryKind = KindCIDR out.QueryKind = KindCIDR
out.Normalized = masked.String() out.Normalized = masked.String()
if err := lookupCIDR(st, tenantID, masked, out, commByID); err != nil { if err := lookupCIDR(ctx, st, tenantID, masked, out, commByID); err != nil {
return nil, err return nil, err
} }
} else { } else {
@@ -124,7 +124,7 @@ func LookupWithResolver(
} }
out.QueryKind = KindDomain out.QueryKind = KindDomain
out.Normalized = fqdn out.Normalized = fqdn
if err := lookupDomain(st, tenantID, fqdn, out, commByID); err != nil { if err := lookupDomain(ctx, st, tenantID, fqdn, out, commByID); err != nil {
return nil, err return nil, err
} }
if resolve == nil { if resolve == nil {
@@ -178,7 +178,7 @@ func lookupResolvedIPs(
out.ResolvedIPs = make([]string, 0, len(ips)) out.ResolvedIPs = make([]string, 0, len(ips))
for _, ip := range ips { for _, ip := range ips {
out.ResolvedIPs = append(out.ResolvedIPs, ip.String()) out.ResolvedIPs = append(out.ResolvedIPs, ip.String())
if err := lookupIP(st, tenantID, ip, out, commByID, ip.String()); err != nil { if err := lookupIP(ctx, st, tenantID, ip, out, commByID, ip.String()); err != nil {
return err return err
} }
} }
@@ -186,6 +186,7 @@ func lookupResolvedIPs(
} }
func lookupIP( func lookupIP(
ctx context.Context,
st store.Backend, st store.Backend,
tenantID string, tenantID string,
addr netip.Addr, addr netip.Addr,
@@ -227,7 +228,7 @@ func lookupIP(
} }
} }
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
if err != nil { if err != nil {
return err return err
} }
@@ -269,6 +270,7 @@ func prefixCoversQuery(listPfx, query netip.Prefix) bool {
} }
func lookupCIDR( func lookupCIDR(
ctx context.Context,
st store.Backend, st store.Backend,
tenantID string, tenantID string,
query netip.Prefix, query netip.Prefix,
@@ -308,7 +310,7 @@ func lookupCIDR(
} }
} }
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID) snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
if err != nil { if err != nil {
return err return err
} }
@@ -338,7 +340,7 @@ func lookupCIDR(
return nil return nil
} }
func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error { func lookupDomain(ctx context.Context, st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error {
matchedModuleIDs := make(map[string]*store.Module) matchedModuleIDs := make(map[string]*store.Module)
for _, mod := range st.ListModules(tenantID) { for _, mod := range st.ListModules(tenantID) {
@@ -372,7 +374,7 @@ func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID
} }
for mid, mod := range matchedModuleIDs { for mid, mod := range matchedModuleIDs {
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mid) snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mid)
if err != nil { if err != nil {
return err return err
} }
+4 -4
View File
@@ -35,7 +35,7 @@ func TestLookupIPEntryAndSnapshot(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash1", []store.PrefixRow{ if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash1", []store.PrefixRow{
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -129,7 +129,7 @@ func TestLookupDomainEntryAndSnapshot(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := m.SetModulePrefixSnapshot(tenant, mod.ID, "hash-d", []store.PrefixRow{ if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash-d", []store.PrefixRow{
{Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"}, {Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"},
{Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"}, {Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"},
}); err != nil { }); err != nil {
@@ -183,7 +183,7 @@ func TestLookupDomainResolvedIPAgainstRanges(t *testing.T) {
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-r", []store.PrefixRow{ if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-r", []store.PrefixRow{
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
@@ -234,7 +234,7 @@ func TestLookupCIDREntryAndSnapshot(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-cidr", []store.PrefixRow{ if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-cidr", []store.PrefixRow{
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"}, {Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
}); err != nil { }); err != nil {
t.Fatal(err) t.Fatal(err)
+51 -8
View File
@@ -61,13 +61,17 @@ func nodeHTTPClient() *http.Client {
} }
func fetchLatestRevision(base, token, speaker string) (string, error) { func fetchLatestRevision(base, token, speaker string) (string, error) {
return fetchLatestRevisionContext(context.Background(), base, token, speaker)
}
func fetchLatestRevisionContext(parent context.Context, base, token, speaker string) (string, error) {
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest" u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
req, err := http.NewRequest(http.MethodGet, u, nil) req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil { if err != nil {
return "", err return "", err
} }
req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Authorization", "Bearer "+token)
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) ctx, cancel := context.WithTimeout(parent, 60*time.Second)
defer cancel() defer cancel()
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3) resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
if err != nil { if err != nil {
@@ -75,13 +79,17 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body) b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b))) return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b)))
} }
var out struct { var out struct {
RevisionID string `json:"revision_id"` RevisionID string `json:"revision_id"`
} }
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { body, err := readHTTPBytes(resp, 64<<10)
if err != nil {
return "", err
}
if err := json.Unmarshal(body, &out); err != nil {
return "", err return "", err
} }
if out.RevisionID == "" { if out.RevisionID == "" {
@@ -91,13 +99,21 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
} }
func fetchBundle(base, token, speaker, revision string) ([]byte, error) { func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
return fetchBundleContext(context.Background(), base, token, speaker, revision, bundle.Limits{})
}
func fetchBundleContext(parent context.Context, base, token, speaker, revision string, limits bundle.Limits) ([]byte, error) {
l, err := limits.Normalized()
if err != nil {
return nil, err
}
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision
req, err := http.NewRequest(http.MethodGet, u, nil) req, err := http.NewRequest(http.MethodGet, u, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Authorization", "Bearer "+token)
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) ctx, cancel := context.WithTimeout(parent, 120*time.Second)
defer cancel() defer cancel()
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3) resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
if err != nil { if err != nil {
@@ -105,10 +121,37 @@ func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
if resp.StatusCode != http.StatusOK { if resp.StatusCode != http.StatusOK {
b, _ := io.ReadAll(resp.Body) b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b))) return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b)))
} }
return io.ReadAll(resp.Body) return readHTTPBytes(resp, l.MaxCompressedBytes)
}
func readHTTPBytes(resp *http.Response, limit int64) ([]byte, error) {
if resp.ContentLength > limit {
return nil, bundle.ErrLimit
}
return bundle.ReadCompressed(resp.Body, bundle.Limits{MaxCompressedBytes: limit})
}
func readBundleFile(path string, limits bundle.Limits) ([]byte, error) {
l, err := limits.Normalized()
if err != nil {
return nil, err
}
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer func() { _ = f.Close() }()
info, err := f.Stat()
if err != nil {
return nil, err
}
if info.Size() > l.MaxCompressedBytes {
return nil, bundle.ErrLimit
}
return bundle.ReadCompressed(f, l)
} }
func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) { func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) {
@@ -146,7 +189,7 @@ func CmdVerifyBundle(args []string) int {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
return 2 return 2
} }
raw, err := os.ReadFile(*path) raw, err := readBundleFile(*path, bundle.Limits{})
if err != nil { if err != nil {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
return 1 return 1
@@ -182,7 +225,7 @@ func CmdApplyBundle(args []string) int {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
return 2 return 2
} }
raw, err := os.ReadFile(*path) raw, err := readBundleFile(*path, bundle.Limits{})
if err != nil { if err != nil {
fmt.Fprintln(os.Stderr, err) fmt.Fprintln(os.Stderr, err)
return 1 return 1
+81
View File
@@ -0,0 +1,81 @@
package nodecli
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"evobgp/internal/bundle"
)
type countingBody struct {
io.Reader
read int
}
func (b *countingBody) Read(p []byte) (int, error) {
n, e := b.Reader.Read(p)
b.read += n
return n, e
}
func (b *countingBody) Close() error { return nil }
func TestHTTPBundleBoundKnownAndChunked(t *testing.T) {
for _, length := range []int64{-1, 2048} {
body := &countingBody{Reader: bytes.NewReader(bytes.Repeat([]byte{'x'}, 2048))}
_, err := readHTTPBytes(&http.Response{Body: body, ContentLength: length}, 1024)
if !errors.Is(err, bundle.ErrLimit) {
t.Fatalf("length=%d limit=%v", length, err)
}
if body.read > 1025 || (length > 1024 && body.read != 0) {
t.Fatalf("unbounded read %d for length %d", body.read, length)
}
}
}
func TestLocalBundleSizeBound(t *testing.T) {
path := filepath.Join(t.TempDir(), "bundle.tar.gz")
if err := os.WriteFile(path, bytes.Repeat([]byte{'x'}, 2048), 0o600); err != nil {
t.Fatal(err)
}
if _, err := readBundleFile(path, bundle.Limits{MaxCompressedBytes: 1024}); !errors.Is(err, bundle.ErrLimit) {
t.Fatalf("local input cap: %v", err)
}
}
func TestBundleHTTPMetadataAndDiagnosticsBound(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.Contains(r.URL.Path, "/revisions/latest"):
_, _ = io.WriteString(w, strings.Repeat(" ", 65<<10))
default:
w.WriteHeader(http.StatusBadRequest)
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
}
}))
defer server.Close()
if _, err := fetchLatestRevision(server.URL, "key", "speaker"); !errors.Is(err, bundle.ErrLimit) {
t.Fatalf("metadata unbounded: %v", err)
}
if _, err := fetchBundle(server.URL, "key", "speaker", "revision"); err == nil || len(err.Error()) > 5<<10 {
t.Fatalf("diagnostic unbounded: %v", err)
}
}
func TestSyncFetchRespectsCallerCancellation(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := fetchBundleContext(ctx, "http://127.0.0.1:1", "key", "speaker", "revision", bundle.Limits{}); !errors.Is(err, context.Canceled) {
t.Fatalf("download ignored cancel: %v", err)
}
if _, err := fetchLatestRevisionContext(ctx, "http://127.0.0.1:1", "key", "speaker"); !errors.Is(err, context.Canceled) {
t.Fatalf("metadata ignored cancel: %v", err)
}
}
+5 -5
View File
@@ -12,11 +12,11 @@ import (
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/bundle" "evobgp/internal/bundle"
"evobgp/internal/signing"
) )
// SyncConfig drives pull → verify → apply on a replica node. // SyncConfig drives pull → verify → apply on a replica node.
type SyncConfig struct { type SyncConfig struct {
BundleLimits bundle.Limits // zero selects bounded large-config defaults
BaseURL string BaseURL string
Token string Token string
SpeakerID string SpeakerID string
@@ -59,12 +59,12 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
rev := strings.TrimSpace(cfg.RevisionID) rev := strings.TrimSpace(cfg.RevisionID)
if rev == "" { if rev == "" {
var err error var err error
rev, err = fetchLatestRevision(cfg.BaseURL, cfg.Token, cfg.SpeakerID) rev, err = fetchLatestRevisionContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID)
if err != nil { if err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
} }
raw, err := fetchBundle(cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev) raw, err := fetchBundleContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev, cfg.BundleLimits)
if err != nil { if err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
@@ -75,7 +75,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
if err := os.WriteFile(bundlePath, raw, 0o644); err != nil { if err := os.WriteFile(bundlePath, raw, 0o644); err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
v, err := signing.VerifyGzippedTar(raw, pub) v, err := bundle.VerifyGzippedTarWithLimits(raw, pub, cfg.BundleLimits)
if err != nil { if err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
@@ -83,7 +83,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
if err := os.MkdirAll(root, 0o755); err != nil { if err := os.MkdirAll(root, 0o755); err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
if err := bundle.WriteExtractedFiles(root, v); err != nil { if err := bundle.WriteExtractedFilesWithLimits(root, v, cfg.BundleLimits); err != nil {
return SyncResult{}, err return SyncResult{}, err
} }
mainRel := v.FindMainBirdConf() mainRel := v.FindMainBirdConf()
+7 -1
View File
@@ -9,6 +9,7 @@ import (
"strings" "strings"
"evobgp/internal/birdfmt" "evobgp/internal/birdfmt"
"evobgp/internal/httpclient"
"evobgp/internal/store" "evobgp/internal/store"
) )
@@ -52,11 +53,16 @@ func FetchBirdProtocols(ctx context.Context, sp *store.Speaker, opts Options) Bi
return res return res
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 { if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
return res return res
} }
b, err := httpclient.ReadResponseBody(resp, 8<<20)
if err != nil {
res.Error = err.Error()
return res
}
var out struct { var out struct {
Sessions []birdfmt.BGPSession `json:"sessions"` Sessions []birdfmt.BGPSession `json:"sessions"`
} }
+24 -4
View File
@@ -20,6 +20,7 @@ type Result struct {
SpeakerID string `json:"speaker_id"` SpeakerID string `json:"speaker_id"`
Endpoint string `json:"endpoint,omitempty"` Endpoint string `json:"endpoint,omitempty"`
Status string `json:"status"` Status string `json:"status"`
Confirmation string `json:"confirmation,omitempty"`
AppliedRevisionID string `json:"applied_revision_id,omitempty"` AppliedRevisionID string `json:"applied_revision_id,omitempty"`
Error string `json:"error,omitempty"` Error string `json:"error,omitempty"`
} }
@@ -62,12 +63,13 @@ func Enabled() bool {
// WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik). // WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik).
func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result { func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
res := Result{SpeakerID: sp.ID} res := Result{}
if sp == nil { if sp == nil {
res.Status = "error" res.Status = "error"
res.Error = "nil speaker" res.Error = "nil speaker"
return res return res
} }
res.SpeakerID = sp.ID
meta := store.ParseSpeakerMeta(sp.MetaJSON) meta := store.ParseSpeakerMeta(sp.MetaJSON)
url := store.AgentSyncURL(meta) url := store.AgentSyncURL(meta)
if url == "" { if url == "" {
@@ -117,13 +119,31 @@ func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
_ = resp.Body.Close() _ = resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 { if resp.StatusCode >= 200 && resp.StatusCode < 300 {
var out struct { var out struct {
OK *bool `json:"ok"`
AppliedRevisionID string `json:"applied_revision_id"` AppliedRevisionID string `json:"applied_revision_id"`
Error string `json:"error"`
}
if len(bytes.TrimSpace(b)) > 0 {
if err := json.Unmarshal(b, &out); err != nil {
res.Status = "error"
res.Error = "invalid agent sync response"
return res
}
}
if (out.OK != nil && !*out.OK) || strings.TrimSpace(out.Error) != "" {
res.Status = "error"
res.Error = strings.TrimSpace(out.Error)
if res.Error == "" {
res.Error = "agent sync reported failure"
}
return res
} }
_ = json.Unmarshal(b, &out)
res.Status = "ok" res.Status = "ok"
res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID) res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID)
if res.AppliedRevisionID == "" { if res.AppliedRevisionID != "" {
res.AppliedRevisionID = strings.TrimSpace(opts.RevisionID) res.Confirmation = "agent_reported"
} else {
res.Confirmation = "unconfirmed"
} }
return res return res
} }
+6 -1
View File
@@ -8,6 +8,7 @@ import (
"net/http" "net/http"
"strings" "strings"
"evobgp/internal/httpclient"
"evobgp/internal/store" "evobgp/internal/store"
) )
@@ -43,12 +44,16 @@ func FetchAgentHealth(ctx context.Context, sp *store.Speaker, opts Options) Agen
return AgentHealthResult{Error: err.Error()} return AgentHealthResult{Error: err.Error()}
} }
defer func() { _ = resp.Body.Close() }() defer func() { _ = resp.Body.Close() }()
b, _ := io.ReadAll(resp.Body)
if resp.StatusCode < 200 || resp.StatusCode >= 300 { if resp.StatusCode < 200 || resp.StatusCode >= 300 {
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
return AgentHealthResult{ return AgentHealthResult{
Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))), Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))),
} }
} }
b, err := httpclient.ReadResponseBody(resp, 64<<10)
if err != nil {
return AgentHealthResult{Error: err.Error()}
}
var out struct { var out struct {
OK bool `json:"ok"` OK bool `json:"ok"`
LastAppliedRevisionID string `json:"last_applied_revision_id"` LastAppliedRevisionID string `json:"last_applied_revision_id"`
@@ -0,0 +1,223 @@
package nodedispatch_test
import (
"context"
"crypto/ed25519"
"crypto/rand"
"encoding/base64"
"fmt"
"net"
"net/http"
"net/http/httptest"
"os"
"os/exec"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"evobgp/internal/bundle"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
func TestMain(m *testing.M) {
if os.Getenv("EVOBGP_TEST_LEGACY_BIRD_HELPER") != "1" {
os.Exit(m.Run())
}
args := os.Args[1:]
log, err := os.OpenFile(os.Getenv("EVOBGP_TEST_BIRD_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
if err != nil {
os.Exit(2)
}
_, _ = fmt.Fprintln(log, strings.Join(args, " "))
_ = log.Close()
switch {
case len(args) == 3 && args[0] == "-c" && args[2] == "-p":
if _, err := os.Stat(args[1]); err == nil {
os.Exit(0)
}
case len(args) == 3 && args[0] == "-s" && args[2] == "configure":
os.Exit(0)
case len(args) == 5 && args[0] == "-s" && strings.Join(args[2:], " ") == "show protocols all":
fmt.Println("peer_test BGP master up 2026-01-01 Established")
os.Exit(0)
}
os.Exit(2)
}
// TestLegacyAgentBinaryCurrentDispatch runs the historical agent as a separate
// process. All network traffic stays on loopback and BIRD executables are fakes.
func TestLegacyAgentBinaryCurrentDispatch(t *testing.T) {
binary := os.Getenv("EVOBGP_TEST_LEGACY_AGENT_BIN")
if binary == "" {
t.Skip("EVOBGP_TEST_LEGACY_AGENT_BIN not set")
}
binary, err := filepath.Abs(binary)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(binary); err != nil {
t.Fatal(err)
}
const speakerID = "legacy-speaker"
const revisionID = "legacy-revision"
const nodeToken = "local-node-token"
const agentSecret = "local-agent-secret"
pub, priv, err := ed25519.GenerateKey(rand.Reader)
if err != nil {
t.Fatal(err)
}
archive, err := bundle.BuildGzippedTar(revisionID, speakerID, map[string]string{
"bird.conf": "router id 192.0.2.1;\n",
}, priv)
if err != nil {
t.Fatal(err)
}
var bundleGets atomic.Int32
controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer "+nodeToken {
http.Error(w, "unauthorized", http.StatusUnauthorized)
return
}
switch r.URL.Path {
case "/v1/speakers/" + speakerID + "/bundle/" + revisionID:
bundleGets.Add(1)
w.Header().Set("Content-Type", "application/gzip")
_, _ = w.Write(archive)
default:
http.NotFound(w, r)
}
}))
defer controlPlane.Close()
root := t.TempDir()
logPath := filepath.Join(root, "bird-invocations")
helper, err := os.Executable()
if err != nil {
t.Fatal(err)
}
listener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
agentAddr := listener.Addr().String()
_ = listener.Close()
outputPath := filepath.Join(root, "agent.log")
output, err := os.Create(outputPath)
if err != nil {
t.Fatal(err)
}
defer func() { _ = output.Close() }()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cmd := exec.CommandContext(ctx, binary, "-listen", agentAddr, "serve")
cmd.Stdout, cmd.Stderr = output, output
cmd.Env = append(os.Environ(),
"EVOBGP_AGENT_SECRET="+agentSecret,
"EVOBGP_CONTROL_PLANE_URL="+controlPlane.URL,
"EVOBGP_NODE_TOKEN="+nodeToken,
"EVOBGP_SPEAKER_ID="+speakerID,
"EVOBGP_BUNDLE_PUBKEY_BASE64="+base64.StdEncoding.EncodeToString(pub),
"EVOBGP_BIRD_EXTRACT_DIR="+filepath.Join(root, "extract"),
"EVOBGP_BIRD_BIN="+helper,
"EVOBGP_BIRDC_BIN="+helper,
"EVOBGP_BIRDC_SOCKET="+filepath.Join(root, "fake.ctl"),
"EVOBGP_TEST_LEGACY_BIRD_HELPER=1",
"EVOBGP_TEST_BIRD_LOG="+logPath,
"TMPDIR="+root,
"TMP="+root,
"TEMP="+root,
)
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() {
cancel()
_ = cmd.Wait()
}()
transport := http.DefaultTransport.(*http.Transport).Clone()
// Under -race each helper process can spend a second flushing race output at
// exit. Sync executes parse-check and configure before returning its ACK.
client := &http.Client{Timeout: 10 * time.Second, Transport: legacyAgentTransport{addr: agentAddr, base: transport}}
defer transport.CloseIdleConnections()
sp := &store.Speaker{ID: speakerID, Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
AgentDomain: "legacy-agent.example.test", AgentSecret: agentSecret,
})}
deadline := time.Now().Add(10 * time.Second)
for {
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
if health.OK {
if health.LastAppliedRevisionID != "" {
t.Fatalf("initial health has applied revision: %+v", health)
}
break
}
if time.Now().After(deadline) {
log, _ := os.ReadFile(outputPath)
t.Fatalf("legacy agent did not become healthy: %+v; output: %s", health, log)
}
time.Sleep(50 * time.Millisecond)
}
// The historical agent must enforce the bearer token before reaching sync.
badReq, err := http.NewRequest(http.MethodPost, "https://legacy-agent.example.test/v1/agent/sync", strings.NewReader(`{"revision_id":"legacy-revision"}`))
if err != nil {
t.Fatal(err)
}
badReq.Header.Set("Authorization", "Bearer wrong-secret")
badResp, err := client.Do(badReq)
if err != nil {
t.Fatal(err)
}
_ = badResp.Body.Close()
if badResp.StatusCode != http.StatusUnauthorized || bundleGets.Load() != 0 {
t.Fatalf("unauthorized sync: status=%d bundle_gets=%d", badResp.StatusCode, bundleGets.Load())
}
result := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{
HTTPClient: client, MaxRetries: 1, RevisionID: revisionID,
})
if result.Status != "ok" || result.Confirmation != "agent_reported" || result.AppliedRevisionID != revisionID {
log, _ := os.ReadFile(outputPath)
t.Fatalf("current dispatch to legacy agent: %+v; output: %s", result, log)
}
if bundleGets.Load() != 1 {
t.Fatalf("signed bundle gets = %d, want 1", bundleGets.Load())
}
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
if !health.OK || health.LastAppliedRevisionID != revisionID || health.LastSyncAt == "" {
t.Fatalf("post-sync legacy health: %+v", health)
}
protocols := nodedispatch.FetchBirdProtocols(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
if protocols.Error != "" || len(protocols.Sessions) != 1 || protocols.Sessions[0].State != "Established" {
t.Fatalf("legacy bird protocols: %+v", protocols)
}
mainConfig, err := os.ReadFile(filepath.Join(root, "extract", "bird.conf"))
if err != nil || string(mainConfig) != "router id 192.0.2.1;\n" {
t.Fatalf("extracted signed bundle: content=%q err=%v", mainConfig, err)
}
invocations, err := os.ReadFile(logPath)
if err != nil {
t.Fatal(err)
}
wantParse := fmt.Sprintf("-c %s -p", filepath.Join(root, "extract", "bird.conf"))
if !strings.Contains(string(invocations), wantParse) || !strings.Contains(string(invocations), "-s "+filepath.Join(root, "fake.ctl")+" configure") {
t.Fatalf("legacy agent did not parse-check and configure via stand-ins: %s", invocations)
}
}
type legacyAgentTransport struct {
addr string
base http.RoundTripper
}
func (tr legacyAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
copyReq := req.Clone(req.Context())
copyReq.URL.Scheme = "http"
copyReq.URL.Host = tr.addr
copyReq.Host = tr.addr
return tr.base.RoundTrip(copyReq)
}
@@ -0,0 +1,106 @@
package nodedispatch_test
import (
"context"
"encoding/json"
"io"
"net/http"
"reflect"
"strings"
"testing"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
type legacyRoundTripFunc func(*http.Request) (*http.Response, error)
func (f legacyRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
func TestLegacyCompatibilityAgentSync(t *testing.T) {
for _, tc := range []struct {
name, response, wantApplied string
status int
}{
{"legacy bare success", `{"ok":true}`, "", http.StatusOK},
{"legacy empty no-content", "", "", http.StatusNoContent},
{"legacy explicit revision", `{"ok":true,"applied_revision_id":"revision-1"}`, "revision-1", http.StatusOK},
} {
t.Run(tc.name, func(t *testing.T) {
var gotAuth, gotType string
var gotBody map[string]json.RawMessage
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" {
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
}
gotAuth, gotType = r.Header.Get("Authorization"), r.Header.Get("Content-Type")
body, _ := io.ReadAll(r.Body)
if err := json.Unmarshal(body, &gotBody); err != nil {
t.Errorf("sync JSON: %v", err)
}
return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.response)), Header: make(http.Header)}, nil
})}
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
})}
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
if got.Status != "ok" || got.AppliedRevisionID != tc.wantApplied {
t.Fatalf("sync result: %+v", got)
}
wantConfirmation := "unconfirmed"
if tc.wantApplied != "" {
wantConfirmation = "agent_reported"
}
if got.Confirmation != wantConfirmation {
t.Fatalf("confirmation = %q, want %q", got.Confirmation, wantConfirmation)
}
if gotAuth != "Bearer legacy-secret" || gotType != "application/json" {
t.Fatalf("legacy headers: authorization=%q content-type=%q", gotAuth, gotType)
}
if !reflect.DeepEqual(gotBody, map[string]json.RawMessage{"revision_id": json.RawMessage(`"revision-1"`)}) {
t.Fatalf("unexpected fields sent to strict legacy decoder: %s", gotBody)
}
})
}
}
func TestAgentSyncRejectsMalformedSuccess(t *testing.T) {
for _, response := range []string{`{"ok":false}`, `{"ok":true`, `{"ok":false,"applied_revision_id":"revision-1"}`} {
t.Run(response, func(t *testing.T) {
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
})}
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
if got.Status != "error" || got.AppliedRevisionID != "" {
t.Fatalf("invalid 2xx response accepted: %+v", got)
}
})
}
}
func TestLegacyCompatibilityAgentHealthMissingNewFields(t *testing.T) {
for _, tc := range []struct {
name, body, wantRevision, wantSync string
}{
{"bare legacy health", `{"ok":true}`, "", ""},
{"revision without time", `{"ok":true,"last_applied_revision_id":"revision-1"}`, "revision-1", ""},
{"revision with time", `{"ok":true,"last_applied_revision_id":"revision-1","last_sync_at":"2026-10-26T00:00:00Z"}`, "revision-1", "2026-10-26T00:00:00Z"},
} {
t.Run(tc.name, func(t *testing.T) {
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
if r.Method != http.MethodGet || r.URL.Path != "/v1/agent/health" || r.Header.Get("Authorization") != "Bearer legacy-secret" {
t.Errorf("unexpected health request: %s %s", r.Method, r.URL)
}
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(tc.body)), Header: make(http.Header)}, nil
})}
sp := &store.Speaker{ID: "speaker-1", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
})}
got := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
if !got.OK || got.Error != "" || got.LastAppliedRevisionID != tc.wantRevision || got.LastSyncAt != tc.wantSync {
t.Fatalf("legacy health: %+v", got)
}
})
}
}
@@ -0,0 +1,80 @@
package nodedispatch_test
import (
"context"
"errors"
"io"
"net/http"
"strings"
"testing"
"evobgp/internal/nodedispatch"
"evobgp/internal/store"
)
type telemetryTestBody struct {
io.Reader
read int
closed bool
}
func (b *telemetryTestBody) Read(p []byte) (int, error) {
n, err := b.Reader.Read(p)
b.read += n
return n, err
}
func (b *telemetryTestBody) Close() error { b.closed = true; return nil }
type telemetryErrorReader struct{}
func (telemetryErrorReader) Read([]byte) (int, error) { return 0, errors.New("injected body failure") }
func TestAgentTelemetryCompleteBodyLimits(t *testing.T) {
sp := &store.Speaker{ID: "legacy", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
for _, tc := range []struct {
name, prefix string
limit int
call func(nodedispatch.Options) string
}{
{"health", `{"ok":true}`, 64 << 10, func(opts nodedispatch.Options) string {
return nodedispatch.FetchAgentHealth(context.Background(), sp, opts).Error
}},
{"protocols", `{"sessions":[]}`, 8 << 20, func(opts nodedispatch.Options) string {
return nodedispatch.FetchBirdProtocols(context.Background(), sp, opts).Error
}},
} {
t.Run(tc.name, func(t *testing.T) {
for _, known := range []bool{true, false} {
for _, extra := range []int{0, 1} {
data := tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra)
body := &telemetryTestBody{Reader: strings.NewReader(data)}
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
length := int64(-1)
if known {
length = int64(len(data))
}
return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil
})}
err := tc.call(nodedispatch.Options{HTTPClient: client})
if (extra == 0 && err != "") || (extra == 1 && err == "") || body.read > tc.limit+1 || !body.closed {
t.Fatalf("known=%v extra=%d read=%d closed=%v error=%s", known, extra, body.read, body.closed, err)
}
}
}
for _, status := range []int{200, 400} {
reader := io.MultiReader(strings.NewReader(tc.prefix), telemetryErrorReader{})
if status == 400 {
reader = io.MultiReader(strings.NewReader(strings.Repeat("x", 16000)))
}
body := &telemetryTestBody{Reader: reader}
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
return &http.Response{StatusCode: status, ContentLength: -1, Body: body, Header: make(http.Header)}, nil
})}
err := tc.call(nodedispatch.Options{HTTPClient: client})
if err == "" || !body.closed || (status == 400 && body.read > 4<<10) {
t.Fatalf("status=%d read=%d closed=%v error=%s", status, body.read, body.closed, err)
}
}
})
}
}
+71 -12
View File
@@ -20,10 +20,12 @@ import (
const namespace = "evobgp" const namespace = "evobgp"
var ( var (
metricsStore atomic.Value // store.Backend metricsStore atomic.Value // storeBackendHolder
registerCollectorOnce sync.Once registerCollectorOnce sync.Once
) )
type storeBackendHolder struct{ backend store.Backend }
var ( var (
httpRequests = promauto.NewCounterVec( httpRequests = promauto.NewCounterVec(
prometheus.CounterOpts{ prometheus.CounterOpts{
@@ -117,7 +119,7 @@ var (
upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{ upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace, Namespace: namespace,
Name: "upstream_breaker_open", Name: "upstream_breaker_open",
Help: "1 if per-host HTTP circuit breaker is open.", Help: "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.",
}, []string{"host"}) }, []string{"host"})
) )
@@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) {
// RecordJobAuditDepth updates durable job_audit depth gauges. // RecordJobAuditDepth updates durable job_audit depth gauges.
func RecordJobAuditDepth(counts map[string]int64) { func RecordJobAuditDepth(counts map[string]int64) {
for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} {
jobAuditDepth.WithLabelValues(status).Set(float64(counts[status]))
}
for status, n := range counts { for status, n := range counts {
if status == "" { if status == "" {
status = "unknown" status = "unknown"
@@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) {
upstreamBreakerOpen.WithLabelValues(host).Set(v) upstreamBreakerOpen.WithLabelValues(host).Set(v)
} }
// RemoveUpstreamBreaker removes an expired host series together with its cache entry.
// Callers must retain active entries until their final metric update completes.
func RemoveUpstreamBreaker(host string) {
if host == "" {
host = "_"
}
upstreamBreakerOpen.DeleteLabelValues(host)
}
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels). // SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
func SetBuildInfo(version, gitSHA string) { func SetBuildInfo(version, gitSHA string) {
if version == "" { if version == "" {
@@ -209,30 +223,45 @@ type memoryStoreCollector struct {
prefixSumDesc *prometheus.Desc prefixSumDesc *prometheus.Desc
peersDesc *prometheus.Desc peersDesc *prometheus.Desc
peerStateDesc *prometheus.Desc peerStateDesc *prometheus.Desc
storedMaxDesc *prometheus.Desc
storedSumDesc *prometheus.Desc
inventoryDesc *prometheus.Desc
storedStateDesc *prometheus.Desc
collectionSuccessDesc *prometheus.Desc
} }
func newMemoryStoreCollector() *memoryStoreCollector { func newMemoryStoreCollector() *memoryStoreCollector {
return &memoryStoreCollector{ return &memoryStoreCollector{
prefixMaxDesc: prometheus.NewDesc( prefixMaxDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"), prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
"Maximum materialized_prefix_count among all revisions in the store.", "Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.",
nil, nil, nil, nil,
), ),
prefixSumDesc: prometheus.NewDesc( prefixSumDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"), prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
"Sum of materialized_prefix_count over revisions (development aggregate).", "Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.",
nil, nil, nil, nil,
), ),
peersDesc: prometheus.NewDesc( peersDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"), prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
"BGP peers configured in the control-plane store.", "Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.",
nil, nil, nil, nil,
), ),
peerStateDesc: prometheus.NewDesc( peerStateDesc: prometheus.NewDesc(
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"), prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).", "Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.",
[]string{"state"}, nil, []string{"state"}, nil,
), ),
storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"),
"Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil),
storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"),
"Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil),
inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"),
"Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil),
storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"),
"BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil),
collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"),
"1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil),
} }
} }
@@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.prefixSumDesc ch <- c.prefixSumDesc
ch <- c.peersDesc ch <- c.peersDesc
ch <- c.peerStateDesc ch <- c.peerStateDesc
ch <- c.storedMaxDesc
ch <- c.storedSumDesc
ch <- c.inventoryDesc
ch <- c.storedStateDesc
ch <- c.collectionSuccessDesc
} }
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) { func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
@@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
if v == nil { if v == nil {
return return
} }
b, ok := v.(store.Backend) holder, ok := v.(storeBackendHolder)
if !ok || b == nil { if !ok || holder.backend == nil {
return return
} }
maxN, sumN := b.MaterializedPrefixStats() b := holder.backend
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx)
cancel()
if err != nil {
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0)
return
}
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1)
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN)) ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN)) ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount())) ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN))
for state, n := range b.PeerSessionCountsByState() { ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN))
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers))
ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers))
canonicalStates := make(map[string]int)
for state, n := range states {
if state == "" { if state == "" {
state = "unknown" state = "unknown"
} }
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state) ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
canonicalStates[canonicalPeerState(state)] += n
}
for state, n := range canonicalStates {
ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state)
}
}
func canonicalPeerState(state string) string {
switch strings.ToLower(strings.TrimSpace(state)) {
case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown":
return strings.ToLower(strings.TrimSpace(state))
default:
return "other"
} }
} }
@@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) {
if b == nil { if b == nil {
return return
} }
metricsStore.Store(b) metricsStore.Store(storeBackendHolder{backend: b})
registerCollectorOnce.Do(func() { registerCollectorOnce.Do(func() {
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector()) prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
}) })
@@ -0,0 +1,116 @@
package observability
import (
"context"
"errors"
"testing"
"github.com/prometheus/client_golang/prometheus"
"evobgp/internal/store"
)
type failingMetricsBackend struct{ store.Backend }
func (failingMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
return 0, 0, 0, nil, errors.New("database unavailable")
}
type storedMetricsBackend struct{ store.Backend }
func (storedMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
return 12, 24, 1, map[string]int{"established": 1}, nil
}
type legacyStateBackend struct{ store.Backend }
func (legacyStateBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
return 0, 0, 2, map[string]int{" Established ": 1, "established": 1}, nil
}
func TestStoreMetricsCollectionErrorOmitsAggregates(t *testing.T) {
metricsStore.Store(storeBackendHolder{backend: failingMetricsBackend{}})
registry := prometheus.NewRegistry()
if err := registry.Register(newMemoryStoreCollector()); err != nil {
t.Fatal(err)
}
families, err := registry.Gather()
if err != nil {
t.Fatal(err)
}
if len(families) != 1 || families[0].GetName() != "evobgp_store_metrics_collection_success" {
t.Fatalf("expected only collection status on error, got %v", families)
}
if got := families[0].Metric[0].Gauge.GetValue(); got != 0 {
t.Fatalf("collection status = %v, want 0", got)
}
}
func TestStoredRevisionMetricsDoNotClaimCurrentRoutes(t *testing.T) {
metricsStore.Store(storeBackendHolder{backend: storedMetricsBackend{}})
registry := prometheus.NewRegistry()
if err := registry.Register(newMemoryStoreCollector()); err != nil {
t.Fatal(err)
}
families, err := registry.Gather()
if err != nil {
t.Fatal(err)
}
values := make(map[string]float64)
for _, family := range families {
if len(family.Metric) > 0 && family.Metric[0].Gauge != nil {
values[family.GetName()] = family.Metric[0].Gauge.GetValue()
}
}
if values["evobgp_stored_revision_materialized_prefixes_sum"] != 24 || values["evobgp_materialized_prefixes_sum"] != 24 {
t.Fatalf("stored revision and legacy sums must match: %v", values)
}
if values["evobgp_store_metrics_collection_success"] != 1 {
t.Fatalf("successful collection not reported: %v", values)
}
if _, ok := values["evobgp_current_routes"]; ok {
t.Fatal("historical revision sum must not be published as current routes")
}
}
func TestCanonicalPeerStateBoundsNewLabelValues(t *testing.T) {
if got := canonicalPeerState("Established"); got != "established" {
t.Fatalf("state = %q", got)
}
if got := canonicalPeerState("unexpected peer text"); got != "other" {
t.Fatalf("state = %q", got)
}
}
func TestLegacyPeerStateLabelPreservedWhileNewLabelIsCanonical(t *testing.T) {
metricsStore.Store(storeBackendHolder{backend: legacyStateBackend{}})
registry := prometheus.NewRegistry()
if err := registry.Register(newMemoryStoreCollector()); err != nil {
t.Fatal(err)
}
families, err := registry.Gather()
if err != nil {
t.Fatal(err)
}
legacyRaw, newCanonical := false, false
for _, family := range families {
for _, metric := range family.Metric {
if len(metric.Label) != 1 {
continue
}
switch family.GetName() {
case "evobgp_bgp_peer_sessions":
if metric.Label[0].GetValue() == " Established " && metric.Gauge.GetValue() == 1 {
legacyRaw = true
}
case "evobgp_stored_bgp_peer_session_state":
if metric.Label[0].GetValue() == "established" && metric.Gauge.GetValue() == 2 {
newCanonical = true
}
}
}
}
if !legacyRaw || !newCanonical {
t.Fatalf("legacy raw state preserved=%v, new canonical state aggregated=%v", legacyRaw, newCanonical)
}
}
+17 -11
View File
@@ -86,20 +86,26 @@ func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *htt
} }
func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) { func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
if rows, ok, err := moduleRowsFromSnapshot(st, tenantID, mod); err != nil { snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
return nil, err
} else if ok {
return rows, nil
}
var prior []store.PrefixRow
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
prior = snap.Prefixes
}
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil { if ok && snap != nil {
if rows, matched, err := snapshotRowsIfCurrent(ctx, st, tenantID, mod, snap); err != nil {
return nil, err
} else if matched {
return rows, nil
}
}
var priorRows []store.PrefixRow
if ok && snap != nil {
priorRows = snap.Prefixes
}
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
if err != nil {
return nil, err
}
if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
return nil, err return nil, err
} }
return rows, nil return rows, nil
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff. Show more