feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18499
-1260
No files matched your search
@@ -322,6 +322,66 @@ jobs:
|
|||||||
go build -o "$out/$name" "./$d"
|
go build -o "$out/$name" "./$d"
|
||||||
done
|
done
|
||||||
|
|
||||||
|
postgres-integration:
|
||||||
|
needs: [changes]
|
||||||
|
if: needs.changes.outputs.go == 'true'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
services:
|
||||||
|
postgres:
|
||||||
|
image: postgres:16
|
||||||
|
env:
|
||||||
|
POSTGRES_USER: evobgp
|
||||||
|
POSTGRES_PASSWORD: evobgp_ci
|
||||||
|
POSTGRES_DB: evobgp_test
|
||||||
|
ports:
|
||||||
|
- 5432:5432
|
||||||
|
options: >-
|
||||||
|
--health-cmd "pg_isready -U evobgp -d evobgp_test"
|
||||||
|
--health-interval 5s
|
||||||
|
--health-timeout 5s
|
||||||
|
--health-retries 12
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||||
|
with:
|
||||||
|
go-version: "1.24"
|
||||||
|
cache: false
|
||||||
|
- name: Export cache paths
|
||||||
|
run: sh scripts/ci/export-cache-env.sh
|
||||||
|
- id: go-hash
|
||||||
|
run: echo "key=$(sha256sum go.sum | awk '{print $1}')" >> "$GITHUB_OUTPUT"
|
||||||
|
- uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
|
||||||
|
with:
|
||||||
|
path: |
|
||||||
|
${{ env.GOMODCACHE }}
|
||||||
|
${{ env.GOCACHE }}
|
||||||
|
key: go-${{ runner.os }}-1.24-postgres-${{ steps.go-hash.outputs.key }}
|
||||||
|
restore-keys: |
|
||||||
|
go-${{ runner.os }}-1.24-postgres-
|
||||||
|
- name: Run migrations and verify PostgreSQL integration tests executed
|
||||||
|
env:
|
||||||
|
GOFLAGS: -mod=readonly
|
||||||
|
EVOBGP_TEST_DATABASE_URL: postgres://evobgp:[email protected]:5432/evobgp_test?sslmode=disable
|
||||||
|
run: bash scripts/ci/postgres-integration.sh
|
||||||
|
|
||||||
|
legacy-compatibility:
|
||||||
|
needs: [changes]
|
||||||
|
if: needs.changes.outputs.go == 'true'
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
legacy_version: ["2.56.2", "1.0.0"]
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||||
|
with:
|
||||||
|
go-version: "1.24"
|
||||||
|
cache: false
|
||||||
|
- name: Build pinned legacy executables and test current control plane
|
||||||
|
env:
|
||||||
|
EVOBGP_LEGACY_VERSION: ${{ matrix.legacy_version }}
|
||||||
|
run: bash scripts/ci/legacy-node-compatibility.sh
|
||||||
|
|
||||||
bird2:
|
bird2:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: [changes, go]
|
needs: [changes, go]
|
||||||
@@ -335,6 +395,10 @@ jobs:
|
|||||||
needs.changes.outputs.docker_go == 'true')
|
needs.changes.outputs.docker_go == 'true')
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
- uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
|
||||||
|
with:
|
||||||
|
go-version: "1.24"
|
||||||
|
cache: false
|
||||||
- name: Install bird2 (репозиторий Ubuntu runner, как в образе evobgp-bird2)
|
- name: Install bird2 (репозиторий Ubuntu runner, как в образе evobgp-bird2)
|
||||||
run: |
|
run: |
|
||||||
set -euxo pipefail
|
set -euxo pipefail
|
||||||
@@ -358,6 +422,11 @@ jobs:
|
|||||||
echo "==> $conf"
|
echo "==> $conf"
|
||||||
bird -c "$WS/$conf" -p
|
bird -c "$WS/$conf" -p
|
||||||
done
|
done
|
||||||
|
- name: Parse staged birddeploy generation with real BIRD
|
||||||
|
env:
|
||||||
|
GOFLAGS: -mod=readonly
|
||||||
|
EVOBGP_TEST_BIRD_BIN: bird
|
||||||
|
run: go test ./internal/birddeploy -run '^TestRealBirdParseGeneration$' -count=1 -v
|
||||||
|
|
||||||
commitlint:
|
commitlint:
|
||||||
if: inputs.is_pull_request
|
if: inputs.is_pull_request
|
||||||
@@ -405,6 +474,8 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
||||||
|
- name: Offline mirror behavior tests
|
||||||
|
run: bash scripts/tests/mirror-base-images.test.sh
|
||||||
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
- uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
|
||||||
with:
|
with:
|
||||||
name: evobgp
|
name: evobgp
|
||||||
|
|||||||
@@ -32,3 +32,6 @@ deploy/compose/runtime-logs/
|
|||||||
.cursor/mcp.json
|
.cursor/mcp.json
|
||||||
.mcp.json
|
.mcp.json
|
||||||
.codegraph/daemon.pid
|
.codegraph/daemon.pid
|
||||||
|
|
||||||
|
# Local backend resource snapshots (host metadata, not source artifacts)
|
||||||
|
artifacts/backend-baseline/
|
||||||
+79
-18
@@ -6,10 +6,14 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/agentserver"
|
"evobgp/internal/agentserver"
|
||||||
|
"evobgp/internal/agentwatch"
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -18,19 +22,22 @@ func main() {
|
|||||||
birdc := flag.String("birdc", "", "path to birdc binary (default: birdc from PATH)")
|
birdc := flag.String("birdc", "", "path to birdc binary (default: birdc from PATH)")
|
||||||
socket := flag.String("socket", "", "optional birdc control socket (-s)")
|
socket := flag.String("socket", "", "optional birdc control socket (-s)")
|
||||||
timeout := flag.Duration("timeout", 30*time.Second, "timeout for bird/birdc")
|
timeout := flag.Duration("timeout", 30*time.Second, "timeout for bird/birdc")
|
||||||
watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between birdc configure")
|
watchEvery := flag.Duration("watch-interval", 30*time.Second, "for watch: interval between configuration checks")
|
||||||
listen := flag.String("listen", "", "for serve: listen address (default :8443 or EVOBGP_AGENT_LISTEN)")
|
listen := flag.String("listen", "", "for serve: listen address (default :8443 or EVOBGP_AGENT_LISTEN)")
|
||||||
flag.Usage = func() {
|
flag.Usage = func() {
|
||||||
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <command>\n", os.Args[0])
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] <command>\n", os.Args[0])
|
||||||
fmt.Fprintf(os.Stderr, "Commands:\n")
|
fmt.Fprintf(os.Stderr, "Commands:\n")
|
||||||
fmt.Fprintf(os.Stderr, " parse-check <path/to/bird.conf> run bird -c <path> -p (syntax check)\n")
|
fmt.Fprintf(os.Stderr, " parse-check <path/to/bird.conf> run bird -c <path> -p (syntax check)\n")
|
||||||
fmt.Fprintf(os.Stderr, " configure run birdc configure (reload running BIRD)\n")
|
fmt.Fprintf(os.Stderr, " configure run birdc configure (reload running BIRD)\n")
|
||||||
fmt.Fprintf(os.Stderr, " watch periodically run birdc configure (compose sidecar)\n")
|
fmt.Fprintf(os.Stderr, " watch reload changed BIRD configuration and periodically reconcile\n")
|
||||||
fmt.Fprintf(os.Stderr, " serve Panel→Node HTTP API (POST /v1/agent/sync)\n")
|
fmt.Fprintf(os.Stderr, " serve Panel→Node HTTP API (POST /v1/agent/sync)\n")
|
||||||
flag.PrintDefaults()
|
flag.PrintDefaults()
|
||||||
}
|
}
|
||||||
flag.Parse()
|
args, err := parseAgentArgs(flag.CommandLine, os.Args[1:])
|
||||||
args := flag.Args()
|
if err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
if len(args) < 1 {
|
if len(args) < 1 {
|
||||||
flag.Usage()
|
flag.Usage()
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
@@ -47,10 +54,12 @@ func main() {
|
|||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "serve":
|
case "serve":
|
||||||
runServe(*listen, *timeout)
|
runServe(*listen, *timeout)
|
||||||
case "parse-check", "configure", "watch":
|
case "watch":
|
||||||
|
runWatch(ctl, *watchEvery, *timeout)
|
||||||
|
case "parse-check", "configure":
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
runBirdCommand(ctx, args, ctl, *watchEvery, *timeout)
|
runBirdCommand(ctx, args, ctl)
|
||||||
default:
|
default:
|
||||||
fmt.Fprintf(os.Stderr, "unknown command: %s\n", args[0])
|
fmt.Fprintf(os.Stderr, "unknown command: %s\n", args[0])
|
||||||
flag.Usage()
|
flag.Usage()
|
||||||
@@ -58,7 +67,47 @@ func main() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, watchEvery, timeout time.Duration) {
|
// parseAgentArgs accepts flags before or immediately after the subcommand.
|
||||||
|
// A second parse preserves previously supplied flag values and rejects unknown
|
||||||
|
// flags instead of silently passing them to watch or serve as positional args.
|
||||||
|
func parseAgentArgs(fs *flag.FlagSet, argv []string) ([]string, error) {
|
||||||
|
if err := fs.Parse(argv); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
args := fs.Args()
|
||||||
|
if len(args) == 0 {
|
||||||
|
return args, nil
|
||||||
|
}
|
||||||
|
command := args[0]
|
||||||
|
globalStop := false
|
||||||
|
for i, consumed := 0, len(argv)-len(args); i < consumed; i++ {
|
||||||
|
if argv[i] == "--" {
|
||||||
|
globalStop = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
name, _, hasValue := strings.Cut(strings.TrimLeft(argv[i], "-"), "=")
|
||||||
|
if !hasValue {
|
||||||
|
if f := fs.Lookup(name); f != nil {
|
||||||
|
boolean, ok := f.Value.(interface{ IsBoolFlag() bool })
|
||||||
|
if !ok || !boolean.IsBoolFlag() {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !globalStop {
|
||||||
|
if err := fs.Parse(args[1:]); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
args = append([]string{command}, fs.Args()...)
|
||||||
|
}
|
||||||
|
if (command == "watch" || command == "serve" || command == "configure") && len(args) != 1 {
|
||||||
|
return nil, fmt.Errorf("%s takes no positional arguments", command)
|
||||||
|
}
|
||||||
|
return args, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl) {
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
case "parse-check":
|
case "parse-check":
|
||||||
if len(args) != 2 {
|
if len(args) != 2 {
|
||||||
@@ -78,21 +127,33 @@ func runBirdCommand(ctx context.Context, args []string, ctl *birdfmt.BirdCtl, wa
|
|||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
os.Exit(1)
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
case "watch":
|
}
|
||||||
if watchEvery <= 0 {
|
}
|
||||||
fmt.Fprintln(os.Stderr, "watch-interval must be > 0")
|
|
||||||
|
func runWatch(ctl *birdfmt.BirdCtl, every, timeout time.Duration) {
|
||||||
|
if every <= 0 || timeout <= 0 {
|
||||||
|
fmt.Fprintln(os.Stderr, "watch-interval and timeout must be > 0")
|
||||||
os.Exit(2)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
log.Printf("evobgp-agent watch: birdc configure every %s (socket=%q)", watchEvery, ctl.Socket)
|
reconcile := 5 * time.Minute
|
||||||
for {
|
if raw := os.Getenv("EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL"); raw != "" {
|
||||||
cctx, cancel := context.WithTimeout(context.Background(), timeout)
|
var err error
|
||||||
err := ctl.Configure(cctx)
|
reconcile, err = time.ParseDuration(raw)
|
||||||
cancel()
|
if err != nil || reconcile <= 0 {
|
||||||
if err != nil {
|
fmt.Fprintln(os.Stderr, "EVOBGP_AGENT_WATCH_RECONCILE_INTERVAL must be a positive duration")
|
||||||
log.Printf("evobgp-agent watch: configure: %v", err)
|
os.Exit(2)
|
||||||
}
|
}
|
||||||
time.Sleep(watchEvery)
|
|
||||||
}
|
}
|
||||||
|
active := os.Getenv("EVOBGP_BIRD_ACTIVE_DIR")
|
||||||
|
if active == "" {
|
||||||
|
active = "/etc/bird"
|
||||||
|
}
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
log.Printf("evobgp-agent watch: checking %s every %s, forced reconciliation every %s (socket=%q)", active, every, reconcile, ctl.Socket)
|
||||||
|
if err := agentwatch.Run(ctx, ctl, agentwatch.Config{ActiveDir: active, Every: every, ReconcileEvery: reconcile, CommandTimeout: timeout}); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, err)
|
||||||
|
os.Exit(1)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"io"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAgentFlagsBeforeAndAfterCommand(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
args []string
|
||||||
|
want []string
|
||||||
|
socket, listen string
|
||||||
|
every time.Duration
|
||||||
|
}{
|
||||||
|
{"legacy flags first", []string{"-socket=/custom/bird.ctl", "-watch-interval=45s", "watch"}, []string{"watch"}, "/custom/bird.ctl", "", 45 * time.Second},
|
||||||
|
{"compose command first", []string{"watch", "-socket=/run/bird/bird.ctl", "-watch-interval=30s"}, []string{"watch"}, "/run/bird/bird.ctl", "", 30 * time.Second},
|
||||||
|
{"serve", []string{"serve", "-listen=127.0.0.1:9999"}, []string{"serve"}, "", "127.0.0.1:9999", time.Minute},
|
||||||
|
{"mixed", []string{"-socket=/first", "watch", "-watch-interval=5s"}, []string{"watch"}, "/first", "", 5 * time.Second},
|
||||||
|
{"parse check", []string{"parse-check", "bird.conf"}, []string{"parse-check", "bird.conf"}, "", "", time.Minute},
|
||||||
|
{"global delimiter", []string{"--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "", "", time.Minute},
|
||||||
|
{"flags then delimiter", []string{"-socket=/first", "--", "parse-check", "-bird.conf"}, []string{"parse-check", "-bird.conf"}, "/first", "", time.Minute},
|
||||||
|
{"delimiter as value", []string{"-socket", "--", "watch", "-watch-interval=5s"}, []string{"watch"}, "--", "", 5 * time.Second},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
fs := flag.NewFlagSet("agent", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
socket := fs.String("socket", "", "")
|
||||||
|
listen := fs.String("listen", "", "")
|
||||||
|
every := fs.Duration("watch-interval", time.Minute, "")
|
||||||
|
args, err := parseAgentArgs(fs, tc.args)
|
||||||
|
if err != nil || !reflect.DeepEqual(args, tc.want) || *socket != tc.socket || *listen != tc.listen || *every != tc.every {
|
||||||
|
t.Fatalf("args=%v socket=%s listen=%s every=%s err=%v", args, *socket, *listen, *every, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
func TestAgentRejectsIgnoredFlagsAndUnexpectedArguments(t *testing.T) {
|
||||||
|
for _, argv := range [][]string{{"watch", "-unknown=1"}, {"serve", "-unknown=1"}, {"watch", "unused"}, {"serve", "unused"}, {"configure", "unused"}} {
|
||||||
|
fs := flag.NewFlagSet("agent", flag.ContinueOnError)
|
||||||
|
fs.SetOutput(io.Discard)
|
||||||
|
if _, err := parseAgentArgs(fs, argv); err == nil {
|
||||||
|
t.Fatalf("ignored argv: %v", argv)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -41,23 +41,41 @@ func main() {
|
|||||||
|
|
||||||
apiBase := strings.TrimSpace(os.Getenv("EVOBGP_CONTROL_PLANE_URL"))
|
apiBase := strings.TrimSpace(os.Getenv("EVOBGP_CONTROL_PLANE_URL"))
|
||||||
apiTok := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_BEARER"))
|
apiTok := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_BEARER"))
|
||||||
if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" && apiBase == "" {
|
mode := strings.ToLower(strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_MODE")))
|
||||||
log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_STANDALONE=1 requires EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER (split deploy must not use in-process jobs.Registry)")
|
if mode == "" {
|
||||||
|
switch {
|
||||||
|
case apiBase != "" || apiTok != "":
|
||||||
|
mode = "http"
|
||||||
|
case pool != nil:
|
||||||
|
mode = "db"
|
||||||
|
default:
|
||||||
|
mode = "local"
|
||||||
}
|
}
|
||||||
deps := &scheduler.Deps{
|
|
||||||
Store: st,
|
|
||||||
HTTP: &http.Client{Timeout: 45 * time.Second},
|
|
||||||
}
|
}
|
||||||
if (apiBase != "") != (apiTok != "") {
|
deps := &scheduler.Deps{Store: st, HTTP: &http.Client{Timeout: 45 * time.Second}}
|
||||||
log.Fatal("evobgp-scheduler: set both EVOBGP_CONTROL_PLANE_URL and EVOBGP_SCHEDULER_BEARER for HTTP mode, or leave both empty for in-process job registry")
|
switch mode {
|
||||||
|
case "db":
|
||||||
|
if pool == nil {
|
||||||
|
log.Fatal("evobgp-scheduler: db mode requires EVOBGP_DATABASE_URL")
|
||||||
}
|
}
|
||||||
if apiBase != "" {
|
|
||||||
deps.APIBase = apiBase
|
|
||||||
deps.APIToken = apiTok
|
|
||||||
log.Printf("evobgp-scheduler: control plane HTTP mode (%s)", apiBase)
|
|
||||||
} else {
|
|
||||||
deps.Jobs = reg
|
deps.Jobs = reg
|
||||||
log.Printf("evobgp-scheduler: in-process job registry mode (use EVOBGP_CONTROL_PLANE_URL + EVOBGP_SCHEDULER_BEARER for split containers)")
|
log.Print("evobgp-scheduler: trusted DB enqueue mode; API workers consume the shared durable queue")
|
||||||
|
case "http":
|
||||||
|
tenant := strings.TrimSpace(os.Getenv("EVOBGP_SCHEDULER_TENANT_ID"))
|
||||||
|
if apiBase == "" || apiTok == "" || tenant == "" {
|
||||||
|
log.Fatal("evobgp-scheduler: http mode requires EVOBGP_CONTROL_PLANE_URL, EVOBGP_SCHEDULER_BEARER and EVOBGP_SCHEDULER_TENANT_ID")
|
||||||
|
}
|
||||||
|
deps.APIBase, deps.APIToken, deps.APITenantID = apiBase, apiTok, tenant
|
||||||
|
log.Print("evobgp-scheduler: single-tenant HTTP enqueue mode")
|
||||||
|
case "local":
|
||||||
|
if os.Getenv("EVOBGP_SCHEDULER_STANDALONE") == "1" || pool != nil {
|
||||||
|
log.Fatal("evobgp-scheduler: local mode is for in-process development only; use db for a shared PostgreSQL queue")
|
||||||
|
}
|
||||||
|
deps.Jobs = reg
|
||||||
|
defer func() { reg.RequestCancelAll(); _ = reg.Drain(context.Background()) }()
|
||||||
|
log.Print("evobgp-scheduler: local development mode")
|
||||||
|
default:
|
||||||
|
log.Fatal("evobgp-scheduler: EVOBGP_SCHEDULER_MODE must be db, http or local")
|
||||||
}
|
}
|
||||||
scheduler.Run(ctx, deps)
|
scheduler.Run(ctx, deps)
|
||||||
}
|
}
|
||||||
+213
-9
@@ -10,6 +10,20 @@ info:
|
|||||||
**Соглашения:** префикс `/v1`; идентификаторы - UUID v7 или ULID (строки); время - ISO 8601 UTC.
|
**Соглашения:** префикс `/v1`; идентификаторы - UUID v7 или ULID (строки); время - ISO 8601 UTC.
|
||||||
Ошибки - `application/problem+json` ([RFC 9457](https://www.rfc-editor.org/rfc/rfc9457)).
|
Ошибки - `application/problem+json` ([RFC 9457](https://www.rfc-editor.org/rfc/rfc9457)).
|
||||||
Пагинация списков - `cursor` + `limit`; ответ содержит `items`, `next_cursor`, `has_more`.
|
Пагинация списков - `cursor` + `limit`; ответ содержит `items`, `next_cursor`, `has_more`.
|
||||||
|
**Общий предел request body:** после аутентификации каждый непубличный `/v1/...`
|
||||||
|
запрос проверяется целиком до выполнения handler: максимум 8 MiB (8388608 bytes),
|
||||||
|
независимо от HTTP-метода, Content-Type и наличия Content-Length. Превышение,
|
||||||
|
включая whitespace после JSON, возвращает `413 application/problem+json`
|
||||||
|
до мутаций. Ошибка чтения тела возвращает `400`, отмена/таймаут чтения — `408`.
|
||||||
|
Пустые node GET/pull и streaming SSE responses не меняются. CSV дополнительно
|
||||||
|
сохраняет ограничения числа строк и полей своей операции.
|
||||||
|
Запросы с телом занимают ограниченный слот API instance до окончания handler:
|
||||||
|
по умолчанию 4 одновременно, `EVOBGP_API_BODY_MAX_CONCURRENT` задаёт 1..64.
|
||||||
|
Свободных слотов нет — немедленный `429` с `Retry-After: 1`, без очереди ожидания.
|
||||||
|
Пустые запросы (`http.NoBody`) слота не занимают.
|
||||||
|
Если CRUD-изменение сохранено, но автоматическая refresh/reconcile задача не поставлена,
|
||||||
|
успешный ответ содержит `X-EvoBGP-Job-Status: enqueue-failed`; ошибка также попадает в аудит.
|
||||||
|
Изменение уже сохранено: следует явно запустить refresh/reconcile, а не повторять создание объекта.
|
||||||
|
|
||||||
**Аутентификация (dual):**
|
**Аутентификация (dual):**
|
||||||
- **API key** — `Authorization: Bearer <token>` из `EVOBGP_API_KEYS` / таблицы `api_key` (роли `viewer`/`editor`/`operator`/`node`/`firewall`).
|
- **API key** — `Authorization: Bearer <token>` из `EVOBGP_API_KEYS` / таблицы `api_key` (роли `viewer`/`editor`/`operator`/`node`/`firewall`).
|
||||||
@@ -226,6 +240,16 @@ components:
|
|||||||
Основные значения: `module_refresh`, `peer_reconcile`, `deploy_apply`, `revision_rollback`, `bird_reload`.
|
Основные значения: `module_refresh`, `peer_reconcile`, `deploy_apply`, `revision_rollback`, `bird_reload`.
|
||||||
|
|
||||||
responses:
|
responses:
|
||||||
|
ExpensiveOperationLimited:
|
||||||
|
description: Локальная квота процесса API — по умолчанию 4 активных preview/lookup, 2 на tenant; 120 lookup и по 30 CDN/revision preview в минуту на tenant. Квоты умножаются на число реплик. Повторить после Retry-After, без очереди ожидания.
|
||||||
|
headers:
|
||||||
|
Retry-After:
|
||||||
|
description: Минимальная задержка повторной попытки в секундах; не гарантия свободного места.
|
||||||
|
schema: {type: integer, minimum: 1}
|
||||||
|
content:
|
||||||
|
application/problem+json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Problem"
|
||||||
Unauthorized:
|
Unauthorized:
|
||||||
description: Нет или невалидный Bearer-токен.
|
description: Нет или невалидный Bearer-токен.
|
||||||
content:
|
content:
|
||||||
@@ -263,7 +287,10 @@ components:
|
|||||||
schema:
|
schema:
|
||||||
$ref: "#/components/schemas/Problem"
|
$ref: "#/components/schemas/Problem"
|
||||||
DefaultProblem:
|
DefaultProblem:
|
||||||
description: Ошибка (см. тело Problem).
|
description: >
|
||||||
|
Ошибка (см. тело Problem). Общие для authenticated `/v1` ответы middleware:
|
||||||
|
413 при request body свыше 8 MiB, 400 при ошибке чтения тела, 408 при отмене/таймауте чтения.
|
||||||
|
429 с Retry-After: 1 при занятых слотах обработки request body.
|
||||||
content:
|
content:
|
||||||
application/problem+json:
|
application/problem+json:
|
||||||
schema:
|
schema:
|
||||||
@@ -903,6 +930,21 @@ components:
|
|||||||
connected_speaker_label:
|
connected_speaker_label:
|
||||||
type: string
|
type: string
|
||||||
description: Человекочитаемая метка ноды из live-опроса.
|
description: Человекочитаемая метка ноды из live-опроса.
|
||||||
|
last_known_session_state:
|
||||||
|
type: ["string", "null"]
|
||||||
|
description: Последнее сохранённое состояние; не является текущим наблюдением.
|
||||||
|
session_state:
|
||||||
|
type: ["string", "null"]
|
||||||
|
description: >
|
||||||
|
При live-ответе текущее наблюдение: BGP state, absent после успешного
|
||||||
|
пустого опроса или null, если актуальное состояние неизвестно.
|
||||||
|
live_observation:
|
||||||
|
$ref: "#/components/schemas/LiveObservation"
|
||||||
|
description: >
|
||||||
|
Качество BGP-наблюдений при GET /v1/peers?live=1. Для назначенного
|
||||||
|
пира учитывается только bgp_speaker_id; для общего — все speakers.
|
||||||
|
Смешанный успех и сбой не дают quality=ok или session_state=absent.
|
||||||
|
Timestamps относятся только к ожидаемым speakers.
|
||||||
session_on_speakers:
|
session_on_speakers:
|
||||||
type: array
|
type: array
|
||||||
description: Состояние протокола пира на каждой опрошенной ноде.
|
description: Состояние протокола пира на каждой опрошенной ноде.
|
||||||
@@ -915,6 +957,10 @@ components:
|
|||||||
type: string
|
type: string
|
||||||
state:
|
state:
|
||||||
type: string
|
type: string
|
||||||
|
poll_error:
|
||||||
|
type: string
|
||||||
|
observation:
|
||||||
|
$ref: "#/components/schemas/LiveObservation"
|
||||||
established_on_speakers:
|
established_on_speakers:
|
||||||
type: array
|
type: array
|
||||||
description: Ноды, где сессия в состоянии Established (один пир может быть на нескольких).
|
description: Ноды, где сессия в состоянии Established (один пир может быть на нескольких).
|
||||||
@@ -1001,7 +1047,11 @@ components:
|
|||||||
description: Человекочитаемая метка ноды (agent domain или CP master).
|
description: Человекочитаемая метка ноды (agent domain или CP master).
|
||||||
agent_ok:
|
agent_ok:
|
||||||
type: boolean
|
type: boolean
|
||||||
description: true если agent /v1/agent/health успешен (master — local birdc poll).
|
description: true только после успешного agent /v1/agent/health (master — local birdc poll).
|
||||||
|
agent_observation:
|
||||||
|
$ref: "#/components/schemas/LiveObservation"
|
||||||
|
bgp_observation:
|
||||||
|
$ref: "#/components/schemas/LiveObservation"
|
||||||
agent_error:
|
agent_error:
|
||||||
type: string
|
type: string
|
||||||
agent_last_sync_at:
|
agent_last_sync_at:
|
||||||
@@ -1015,15 +1065,49 @@ components:
|
|||||||
bgp_poll_error:
|
bgp_poll_error:
|
||||||
type: string
|
type: string
|
||||||
bgp_sessions_total:
|
bgp_sessions_total:
|
||||||
type: integer
|
type: ["integer", "null"]
|
||||||
|
description: null при отсутствии успешного опроса.
|
||||||
bgp_established:
|
bgp_established:
|
||||||
type: integer
|
type: ["integer", "null"]
|
||||||
|
description: null при отсутствии успешного опроса.
|
||||||
sessions:
|
sessions:
|
||||||
type: array
|
type: array
|
||||||
items:
|
items:
|
||||||
$ref: "#/components/schemas/BgpSessionLive"
|
$ref: "#/components/schemas/BgpSessionLive"
|
||||||
additionalProperties: true
|
additionalProperties: true
|
||||||
|
|
||||||
|
LiveObservation:
|
||||||
|
type: object
|
||||||
|
description: >
|
||||||
|
Метаданные конкретного опроса. observed_at — время завершения опроса;
|
||||||
|
received_at — время получения результата CP; last_success_at — время
|
||||||
|
последнего успешного опроса. Старые агенты не обязаны передавать эти поля:
|
||||||
|
CP формирует их по собственным часам. При отсутствии опроса timestamps null.
|
||||||
|
required: [observed_at, received_at, last_success_at, source, quality]
|
||||||
|
properties:
|
||||||
|
observed_at:
|
||||||
|
type: ["string", "null"]
|
||||||
|
format: date-time
|
||||||
|
received_at:
|
||||||
|
type: ["string", "null"]
|
||||||
|
format: date-time
|
||||||
|
last_success_at:
|
||||||
|
type: ["string", "null"]
|
||||||
|
format: date-time
|
||||||
|
expires_at:
|
||||||
|
type: ["string", "null"]
|
||||||
|
format: date-time
|
||||||
|
description: Срок актуальности snapshot по часам CP и настроенному интервалу collector; клиент не должен считать cached quality=ok актуальным после этого времени. Для агрегата — самый ранний срок источников.
|
||||||
|
source:
|
||||||
|
type: string
|
||||||
|
enum: [none, cp_birdc, agent_bird_protocols, agent_health, mixed]
|
||||||
|
quality:
|
||||||
|
type: string
|
||||||
|
enum: [ok, stale, unreachable, unknown, disabled]
|
||||||
|
error_code:
|
||||||
|
type: string
|
||||||
|
enum: [not_configured, not_polled, poll_failed, reported_unhealthy, snapshot_expired, configuration_changed, ambiguous_authority]
|
||||||
|
|
||||||
BgpSessionLive:
|
BgpSessionLive:
|
||||||
type: object
|
type: object
|
||||||
properties:
|
properties:
|
||||||
@@ -1052,9 +1136,12 @@ components:
|
|||||||
ok:
|
ok:
|
||||||
type: boolean
|
type: boolean
|
||||||
session_count:
|
session_count:
|
||||||
type: integer
|
type: ["integer", "null"]
|
||||||
|
description: null, если успешного опроса не было.
|
||||||
poll_error:
|
poll_error:
|
||||||
type: string
|
type: string
|
||||||
|
observation:
|
||||||
|
$ref: "#/components/schemas/LiveObservation"
|
||||||
additionalProperties: true
|
additionalProperties: true
|
||||||
|
|
||||||
PostgresOverview:
|
PostgresOverview:
|
||||||
@@ -2108,6 +2195,8 @@ paths:
|
|||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: "#/components/schemas/LookupResponse"
|
$ref: "#/components/schemas/LookupResponse"
|
||||||
|
"429":
|
||||||
|
$ref: "#/components/responses/ExpensiveOperationLimited"
|
||||||
"400":
|
"400":
|
||||||
$ref: "#/components/responses/BadRequest"
|
$ref: "#/components/responses/BadRequest"
|
||||||
"401":
|
"401":
|
||||||
@@ -2115,6 +2204,29 @@ paths:
|
|||||||
default:
|
default:
|
||||||
$ref: "#/components/responses/DefaultProblem"
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
|
/v1/telemetry/refresh:
|
||||||
|
parameters:
|
||||||
|
- $ref: "#/components/parameters/TenantId"
|
||||||
|
post:
|
||||||
|
tags: [Speakers]
|
||||||
|
summary: Запросить фоновое обновление телеметрии tenant
|
||||||
|
description: Требует bgp:network:read. Объединяет повторные запросы в пределах 10 секунд; HTTP handler не опрашивает агентов или BIRD. GET speakers/peers/discoveries читает сохранённые snapshots, включая live=1. Запрос во время выполняющегося poll сохраняется для следующего цикла. Не изменяет конфигурацию speaker и не требует нового протокола агента.
|
||||||
|
operationId: requestTelemetryRefresh
|
||||||
|
responses:
|
||||||
|
"202":
|
||||||
|
description: Запрос принят либо объединён с уже принятым.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
required: [requested]
|
||||||
|
properties:
|
||||||
|
requested:
|
||||||
|
type: boolean
|
||||||
|
description: true при постановке нового запроса; false при объединении или отсутствии целей.
|
||||||
|
default:
|
||||||
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
/v1/router-lists/catalog:
|
/v1/router-lists/catalog:
|
||||||
get:
|
get:
|
||||||
tags: [Modules]
|
tags: [Modules]
|
||||||
@@ -2789,6 +2901,9 @@ paths:
|
|||||||
- `ipRange,community`
|
- `ipRange,community`
|
||||||
|
|
||||||
В поле `community` можно передавать либо ID community, либо её значение.
|
В поле `community` можно передавать либо ID community, либо её значение.
|
||||||
|
Предел запроса — 8 MiB (8388608 байт), 10000 строк данных без заголовка и 64 поля в записи.
|
||||||
|
Превышение возвращает 413 до создания записей; большой импорт следует разбить на части.
|
||||||
|
Ошибка в данных или БД после начала создания записей может оставить частичный импорт.
|
||||||
operationId: importModuleEntriesCsv
|
operationId: importModuleEntriesCsv
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: "#/components/parameters/IdempotencyKey"
|
- $ref: "#/components/parameters/IdempotencyKey"
|
||||||
@@ -2812,6 +2927,12 @@ paths:
|
|||||||
minimum: 0
|
minimum: 0
|
||||||
module_type:
|
module_type:
|
||||||
type: string
|
type: string
|
||||||
|
"413":
|
||||||
|
description: Превышен лимит байт, строк или полей CSV; записи не созданы.
|
||||||
|
content:
|
||||||
|
application/problem+json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Problem"
|
||||||
"404":
|
"404":
|
||||||
$ref: "#/components/responses/NotFound"
|
$ref: "#/components/responses/NotFound"
|
||||||
"422":
|
"422":
|
||||||
@@ -2898,6 +3019,8 @@ paths:
|
|||||||
application/json:
|
application/json:
|
||||||
schema:
|
schema:
|
||||||
$ref: "#/components/schemas/CdnPreviewResponse"
|
$ref: "#/components/schemas/CdnPreviewResponse"
|
||||||
|
"429":
|
||||||
|
$ref: "#/components/responses/ExpensiveOperationLimited"
|
||||||
"401":
|
"401":
|
||||||
$ref: "#/components/responses/Unauthorized"
|
$ref: "#/components/responses/Unauthorized"
|
||||||
"403":
|
"403":
|
||||||
@@ -3146,7 +3269,7 @@ paths:
|
|||||||
type: string
|
type: string
|
||||||
enum: ["1"]
|
enum: ["1"]
|
||||||
description: >
|
description: >
|
||||||
Опрос birdc на CP и GET /v1/agent/bird/protocols на репликах; обогащает session_state и connected_speaker_*.
|
Сохранённые результаты фонового опроса birdc на CP и agent protocols на репликах; обогащает session_state и connected_speaker_*. GET не запускает опрос; live=1 сохранён для совместимости.
|
||||||
responses:
|
responses:
|
||||||
"200":
|
"200":
|
||||||
description: Успешно.
|
description: Успешно.
|
||||||
@@ -3167,7 +3290,7 @@ paths:
|
|||||||
live_speaker_poll:
|
live_speaker_poll:
|
||||||
type: array
|
type: array
|
||||||
description: >
|
description: >
|
||||||
При live=1 — результат опроса каждой ноды (CP birdc + agent protocols).
|
Сохранённый результат фонового опроса каждой ноды (CP birdc + agent protocols); live=1 не запускает новый опрос.
|
||||||
items:
|
items:
|
||||||
$ref: "#/components/schemas/LiveSpeakerPoll"
|
$ref: "#/components/schemas/LiveSpeakerPoll"
|
||||||
default:
|
default:
|
||||||
@@ -3204,7 +3327,7 @@ paths:
|
|||||||
summary: Список обнаруженных (неодобренных) пиров
|
summary: Список обнаруженных (неодобренных) пиров
|
||||||
description: >
|
description: >
|
||||||
Dynamic BGP-сессии (`evobgp_dyn_*`), которых ещё нет в `bgp_peer`.
|
Dynamic BGP-сессии (`evobgp_dyn_*`), которых ещё нет в `bgp_peer`.
|
||||||
По умолчанию возвращает `status=pending`. При листинге выполняет live-опрос birdc/agent и upsert pending.
|
По умолчанию возвращает `status=pending`. Читает сохранённые discoveries. Фоновый collector выполняет опрос и fenced upsert; GET не опрашивает ноды и не изменяет данные.
|
||||||
operationId: listDiscoveredPeers
|
operationId: listDiscoveredPeers
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: "#/components/parameters/TenantId"
|
- $ref: "#/components/parameters/TenantId"
|
||||||
@@ -3380,7 +3503,7 @@ paths:
|
|||||||
type: string
|
type: string
|
||||||
enum: ["1"]
|
enum: ["1"]
|
||||||
description: >
|
description: >
|
||||||
Live-опрос agent /v1/agent/health и BGP protocols на репликах; CP — local birdc.
|
Сохранённые результаты фоновых опросов agent health/BGP и local birdc. GET не запускает опрос.
|
||||||
Обогащает каждый item полем `live`.
|
Обогащает каждый item полем `live`.
|
||||||
responses:
|
responses:
|
||||||
"200":
|
"200":
|
||||||
@@ -3640,6 +3763,8 @@ paths:
|
|||||||
description: Только чтение, без apply.
|
description: Только чтение, без apply.
|
||||||
operationId: getRevisionPreview
|
operationId: getRevisionPreview
|
||||||
responses:
|
responses:
|
||||||
|
"429":
|
||||||
|
$ref: "#/components/responses/ExpensiveOperationLimited"
|
||||||
"200":
|
"200":
|
||||||
description: Текст или структурированное представление - формат задаётся реализацией.
|
description: Текст или структурированное представление - формат задаётся реализацией.
|
||||||
content:
|
content:
|
||||||
@@ -3881,6 +4006,58 @@ paths:
|
|||||||
default:
|
default:
|
||||||
$ref: "#/components/responses/DefaultProblem"
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
|
/v1/jobs/{job_id}/report:
|
||||||
|
parameters:
|
||||||
|
- $ref: "#/components/parameters/TenantId"
|
||||||
|
- $ref: "#/components/parameters/JobId"
|
||||||
|
get:
|
||||||
|
tags: [Jobs]
|
||||||
|
summary: Отчёт задачи и сохранённый прогресс доставки
|
||||||
|
description: Требует bgp:operations:read. Прогресс хранится независимо от meta задачи. Для reconciliation без собственного прогресса возвращается прогресс исходной задачи в том же tenant; после начала восстановления приоритет имеет собственный прогресс. agent_reported не означает проверку состояния BIRD.
|
||||||
|
operationId: getJobReport
|
||||||
|
responses:
|
||||||
|
"200":
|
||||||
|
description: Успешно.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
type: object
|
||||||
|
required: [job_id, kind, status, deployment_progress]
|
||||||
|
properties:
|
||||||
|
job_id: {type: string}
|
||||||
|
kind: {type: string}
|
||||||
|
status: {type: string}
|
||||||
|
meta: {type: [object, "null"], additionalProperties: true}
|
||||||
|
error: {type: [string, "null"]}
|
||||||
|
created_at: {type: string, format: date-time}
|
||||||
|
revision_id: {type: string}
|
||||||
|
log_entries: {type: array, items: {type: object, additionalProperties: true}}
|
||||||
|
log_total: {type: integer}
|
||||||
|
source_deployment_job_id: {type: string}
|
||||||
|
deployment_progress:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: object
|
||||||
|
required: [job_id, speaker_id, revision_id, generation, state, confirmation, dispatch_attempts, retry_round]
|
||||||
|
properties:
|
||||||
|
job_id: {type: string}
|
||||||
|
speaker_id: {type: string}
|
||||||
|
revision_id: {type: string}
|
||||||
|
generation: {type: integer, format: int64}
|
||||||
|
state: {type: string}
|
||||||
|
confirmation: {type: string}
|
||||||
|
error: {type: string}
|
||||||
|
dispatch_attempts: {type: integer}
|
||||||
|
retry_round: {type: integer}
|
||||||
|
health_received_at: {type: [string, "null"], format: date-time}
|
||||||
|
health_revision_id: {type: string}
|
||||||
|
health_source_at: {type: string}
|
||||||
|
health_error: {type: string}
|
||||||
|
"404":
|
||||||
|
$ref: "#/components/responses/NotFound"
|
||||||
|
default:
|
||||||
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
/v1/jobs/{job_id}/cancel:
|
/v1/jobs/{job_id}/cancel:
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: "#/components/parameters/TenantId"
|
- $ref: "#/components/parameters/TenantId"
|
||||||
@@ -3904,6 +4081,33 @@ paths:
|
|||||||
default:
|
default:
|
||||||
$ref: "#/components/responses/DefaultProblem"
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
|
/v1/jobs/{job_id}/retry:
|
||||||
|
parameters:
|
||||||
|
- $ref: "#/components/parameters/TenantId"
|
||||||
|
- $ref: "#/components/parameters/JobId"
|
||||||
|
post:
|
||||||
|
tags: [Jobs]
|
||||||
|
summary: Повторить завершившуюся ошибкой задачу
|
||||||
|
description: Операторский повтор сохраняет ID и исходный запрос, сбрасывая бюджет попыток.
|
||||||
|
operationId: postJobRetry
|
||||||
|
responses:
|
||||||
|
"202":
|
||||||
|
description: Повтор поставлен в очередь.
|
||||||
|
content:
|
||||||
|
application/json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Job"
|
||||||
|
"409":
|
||||||
|
$ref: "#/components/responses/Conflict"
|
||||||
|
"429":
|
||||||
|
description: Очередь задач заполнена.
|
||||||
|
content:
|
||||||
|
application/problem+json:
|
||||||
|
schema:
|
||||||
|
$ref: "#/components/schemas/Problem"
|
||||||
|
default:
|
||||||
|
$ref: "#/components/responses/DefaultProblem"
|
||||||
|
|
||||||
/v1/speakers/{speaker_id}/revisions/latest:
|
/v1/speakers/{speaker_id}/revisions/latest:
|
||||||
parameters:
|
parameters:
|
||||||
- $ref: "#/components/parameters/SpeakerId"
|
- $ref: "#/components/parameters/SpeakerId"
|
||||||
|
|||||||
@@ -0,0 +1,312 @@
|
|||||||
|
package agentwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/binary"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"hash"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxConfigFiles = 512
|
||||||
|
maxConfigBytes = 64 << 20
|
||||||
|
maxFileBytes = 16 << 20
|
||||||
|
maxIncludeDepth = 16
|
||||||
|
maxDirectoryEntries = 4096
|
||||||
|
)
|
||||||
|
|
||||||
|
var errMainNotReady = errors.New("main bird.conf is not ready")
|
||||||
|
|
||||||
|
// Only standalone literal include statements are hashable. Other valid BIRD
|
||||||
|
// grammar falls back to the legacy configure-on-every-tick path.
|
||||||
|
var includeLine = regexp.MustCompile(`(?m)^[\t ]*include[\t ]+"([^"\r\n]+)"[\t ]*;[\t ]*(?:#[^\r\n]*)?\r?$`)
|
||||||
|
|
||||||
|
type configScanner struct {
|
||||||
|
ctx context.Context
|
||||||
|
root string
|
||||||
|
h hash.Hash
|
||||||
|
seen map[string]bool
|
||||||
|
files int
|
||||||
|
bytes int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashConfigTree(ctx context.Context, activeDir string) ([sha256.Size]byte, error) {
|
||||||
|
var zero [sha256.Size]byte
|
||||||
|
root, err := filepath.Abs(activeDir)
|
||||||
|
if err != nil {
|
||||||
|
return zero, fmt.Errorf("agentwatch: active directory: %w", err)
|
||||||
|
}
|
||||||
|
root, err = filepath.EvalSymlinks(root)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return zero, fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||||
|
}
|
||||||
|
return zero, fmt.Errorf("agentwatch: resolve active directory: %w", err)
|
||||||
|
}
|
||||||
|
s := &configScanner{ctx: ctx, root: filepath.Clean(root), h: sha256.New(), seen: make(map[string]bool)}
|
||||||
|
if err := s.visit(filepath.Join(s.root, "bird.conf"), 0); err != nil {
|
||||||
|
return zero, err
|
||||||
|
}
|
||||||
|
var digest [sha256.Size]byte
|
||||||
|
copy(digest[:], s.h.Sum(nil))
|
||||||
|
return digest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *configScanner) visit(path string, depth int) error {
|
||||||
|
if err := s.ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if depth > maxIncludeDepth {
|
||||||
|
return fmt.Errorf("agentwatch: include depth exceeds %d", maxIncludeDepth)
|
||||||
|
}
|
||||||
|
path = filepath.Clean(path)
|
||||||
|
rel, err := filepath.Rel(s.root, path)
|
||||||
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||||
|
return fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||||
|
}
|
||||||
|
if s.seen[path] {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(path)
|
||||||
|
if err != nil {
|
||||||
|
if depth == 0 && errors.Is(err, os.ErrNotExist) {
|
||||||
|
return fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("agentwatch: inspect %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("agentwatch: include %q is not a regular file", path)
|
||||||
|
}
|
||||||
|
resolved, err := filepath.EvalSymlinks(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: resolve %q: %w", path, err)
|
||||||
|
}
|
||||||
|
if _, err := s.withinRoot(resolved); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if info.Size() > maxFileBytes {
|
||||||
|
return fmt.Errorf("agentwatch: include %q exceeds %d bytes", path, maxFileBytes)
|
||||||
|
}
|
||||||
|
if s.files >= maxConfigFiles || s.bytes+info.Size() > maxConfigBytes {
|
||||||
|
return fmt.Errorf("agentwatch: configuration tree exceeds %d files or %d bytes", maxConfigFiles, maxConfigBytes)
|
||||||
|
}
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: open %q: %w", path, err)
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
limit := int64(maxFileBytes)
|
||||||
|
if remaining := int64(maxConfigBytes) - s.bytes; remaining < limit {
|
||||||
|
limit = remaining
|
||||||
|
}
|
||||||
|
content, err := readBounded(f, limit)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: read %q: %w", path, err)
|
||||||
|
}
|
||||||
|
s.seen[path] = true
|
||||||
|
s.files++
|
||||||
|
s.bytes += int64(len(content))
|
||||||
|
writeHash(s.h, filepath.ToSlash(rel), content)
|
||||||
|
patterns, err := includePatterns(string(content))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
for _, pattern := range patterns {
|
||||||
|
if strings.ContainsAny(pattern, `[\`) {
|
||||||
|
return fmt.Errorf("agentwatch: unsupported escaped include %q", pattern)
|
||||||
|
}
|
||||||
|
candidate := pattern
|
||||||
|
if !filepath.IsAbs(candidate) {
|
||||||
|
candidate = filepath.Join(filepath.Dir(path), filepath.FromSlash(candidate))
|
||||||
|
}
|
||||||
|
candidate = filepath.Clean(candidate)
|
||||||
|
// The pattern itself changes the digest even when its glob currently matches nothing.
|
||||||
|
writeHash(s.h, "include", []byte(filepath.ToSlash(candidate)))
|
||||||
|
if _, err := s.withinRoot(candidate); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if strings.ContainsAny(candidate, "*?") {
|
||||||
|
if strings.ContainsAny(filepath.Dir(candidate), "*?") {
|
||||||
|
return fmt.Errorf("agentwatch: unsupported include directory glob %q", pattern)
|
||||||
|
}
|
||||||
|
resolvedDir, err := filepath.EvalSymlinks(filepath.Dir(candidate))
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: resolve include directory %q: %w", pattern, err)
|
||||||
|
}
|
||||||
|
if _, err := s.withinRoot(resolvedDir); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
matches, err := boundedGlob(candidate)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("agentwatch: include glob %q: %w", pattern, err)
|
||||||
|
}
|
||||||
|
for _, match := range matches {
|
||||||
|
if err := s.visit(match, depth+1); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := s.visit(candidate, depth+1); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func readBounded(r io.Reader, limit int64) ([]byte, error) {
|
||||||
|
content, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(content)) > limit {
|
||||||
|
return nil, fmt.Errorf("configuration tree changed or exceeded size limit")
|
||||||
|
}
|
||||||
|
return content, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// boundedGlob only handles * and ? in a filename. Broader BIRD glob grammar
|
||||||
|
// falls back to every-tick configure instead of risking a false unchanged hash.
|
||||||
|
func boundedGlob(pattern string) ([]string, error) {
|
||||||
|
dir, base := filepath.Dir(pattern), filepath.Base(pattern)
|
||||||
|
f, err := os.Open(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
names, err := f.Readdirnames(maxDirectoryEntries + 1)
|
||||||
|
if err != nil && !errors.Is(err, io.EOF) {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(names) > maxDirectoryEntries {
|
||||||
|
return nil, fmt.Errorf("directory exceeds %d entries", maxDirectoryEntries)
|
||||||
|
}
|
||||||
|
matches := make([]string, 0)
|
||||||
|
for _, name := range names {
|
||||||
|
match, err := filepath.Match(base, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if match {
|
||||||
|
matches = append(matches, filepath.Join(dir, name))
|
||||||
|
if len(matches) > maxConfigFiles {
|
||||||
|
return nil, fmt.Errorf("glob exceeds %d matches", maxConfigFiles)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(matches)
|
||||||
|
return matches, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *configScanner) withinRoot(path string) (string, error) {
|
||||||
|
rel, err := filepath.Rel(s.root, path)
|
||||||
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||||
|
return "", fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||||
|
}
|
||||||
|
return rel, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeHash(h hash.Hash, name string, content []byte) {
|
||||||
|
var size [8]byte
|
||||||
|
binary.LittleEndian.PutUint64(size[:], uint64(len(name)))
|
||||||
|
_, _ = h.Write(size[:])
|
||||||
|
_, _ = h.Write([]byte(name))
|
||||||
|
binary.LittleEndian.PutUint64(size[:], uint64(len(content)))
|
||||||
|
_, _ = h.Write(size[:])
|
||||||
|
_, _ = h.Write(content)
|
||||||
|
}
|
||||||
|
|
||||||
|
func includePatterns(content string) ([]string, error) {
|
||||||
|
tokens, unsupportedQuotedText := includeTokenOffsets(content)
|
||||||
|
if unsupportedQuotedText {
|
||||||
|
return nil, fmt.Errorf("unsupported quoted text or escape")
|
||||||
|
}
|
||||||
|
tokenSet := make(map[int]bool, len(tokens))
|
||||||
|
for _, token := range tokens {
|
||||||
|
tokenSet[token] = true
|
||||||
|
}
|
||||||
|
var patterns []string
|
||||||
|
for _, match := range includeLine.FindAllStringSubmatchIndex(content, -1) {
|
||||||
|
keyword := match[0] + strings.Index(content[match[0]:match[1]], "include")
|
||||||
|
if !tokenSet[keyword] {
|
||||||
|
continue
|
||||||
|
} // text inside a block comment
|
||||||
|
delete(tokenSet, keyword)
|
||||||
|
patterns = append(patterns, content[match[2]:match[3]])
|
||||||
|
}
|
||||||
|
if len(tokenSet) != 0 {
|
||||||
|
return nil, fmt.Errorf("unsupported include syntax")
|
||||||
|
}
|
||||||
|
return patterns, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Recognize include keywords outside BIRD comments and quoted strings.
|
||||||
|
func includeTokenOffsets(config string) ([]int, bool) {
|
||||||
|
var offsets []int
|
||||||
|
quote := byte(0)
|
||||||
|
lineComment, blockComment := false, false
|
||||||
|
for i := 0; i < len(config); {
|
||||||
|
c := config[i]
|
||||||
|
if lineComment {
|
||||||
|
if c == '\n' {
|
||||||
|
lineComment = false
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if blockComment {
|
||||||
|
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
|
||||||
|
blockComment = false
|
||||||
|
i += 2
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if quote != 0 {
|
||||||
|
if c == '\\' {
|
||||||
|
return offsets, true
|
||||||
|
}
|
||||||
|
if c == quote {
|
||||||
|
quote = 0
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '#' || (c == '/' && i+1 < len(config) && config[i+1] == '/') {
|
||||||
|
lineComment = true
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
|
||||||
|
blockComment = true
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '"' || c == '\'' {
|
||||||
|
quote = c
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
|
||||||
|
start := i
|
||||||
|
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
if config[start:i] == "include" {
|
||||||
|
offsets = append(offsets, start)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
return offsets, quote != 0 || blockComment
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Package agentwatch reloads BIRD when its active configuration tree changes.
|
||||||
|
package agentwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"path/filepath"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Control is the subset of BIRD operations required by the watch loop.
|
||||||
|
type Control interface {
|
||||||
|
ParseCheck(context.Context, string) error
|
||||||
|
Configure(context.Context) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Config sets bounded polling and reconciliation behavior.
|
||||||
|
type Config struct {
|
||||||
|
ActiveDir string
|
||||||
|
Every time.Duration
|
||||||
|
ReconcileEvery time.Duration
|
||||||
|
CommandTimeout time.Duration
|
||||||
|
Logf func(string, ...any)
|
||||||
|
}
|
||||||
|
|
||||||
|
type watcher struct {
|
||||||
|
ctl Control
|
||||||
|
cfg Config
|
||||||
|
hash [sha256.Size]byte
|
||||||
|
haveHash bool
|
||||||
|
lastConfigured time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run starts with one check, then polls until ctx is cancelled. A successful
|
||||||
|
// birdc exit advances the remembered hash; it is not proof of BIRD ACK.
|
||||||
|
func Run(ctx context.Context, ctl Control, cfg Config) error {
|
||||||
|
if ctl == nil || cfg.Every <= 0 || cfg.ReconcileEvery <= 0 || cfg.CommandTimeout <= 0 {
|
||||||
|
return fmt.Errorf("agentwatch: control and positive intervals/timeouts are required")
|
||||||
|
}
|
||||||
|
if cfg.ActiveDir == "" {
|
||||||
|
cfg.ActiveDir = "/etc/bird"
|
||||||
|
}
|
||||||
|
if cfg.Logf == nil {
|
||||||
|
cfg.Logf = log.Printf
|
||||||
|
}
|
||||||
|
w := &watcher{ctl: ctl, cfg: cfg}
|
||||||
|
if err := w.step(ctx, time.Now()); err != nil && ctx.Err() == nil {
|
||||||
|
cfg.Logf("evobgp-agent watch: %v", err)
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(cfg.Every)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case now := <-ticker.C:
|
||||||
|
if err := w.step(ctx, now); err != nil && ctx.Err() == nil {
|
||||||
|
cfg.Logf("evobgp-agent watch: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watcher) step(ctx context.Context, now time.Time) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
digest, scanErr := hashConfigTree(ctx, w.cfg.ActiveDir)
|
||||||
|
if scanErr != nil {
|
||||||
|
if errors.Is(scanErr, errMainNotReady) {
|
||||||
|
return fmt.Errorf("configuration not ready: %w", scanErr)
|
||||||
|
}
|
||||||
|
// Unsupported/unhashable trees retain the old every-tick configure cadence.
|
||||||
|
w.cfg.Logf("evobgp-agent watch: cannot hash complete config tree (%v); using every-tick parse/configure", scanErr)
|
||||||
|
} else if w.haveHash && digest == w.hash && now.Sub(w.lastConfigured) < w.cfg.ReconcileEvery {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
main := filepath.Join(w.cfg.ActiveDir, "bird.conf")
|
||||||
|
if err := w.command(ctx, func(cctx context.Context) error { return w.ctl.ParseCheck(cctx, main) }); err != nil {
|
||||||
|
return fmt.Errorf("parse check: %w", err)
|
||||||
|
}
|
||||||
|
if scanErr == nil {
|
||||||
|
stable, err := hashConfigTree(ctx, w.cfg.ActiveDir)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("configuration changed during parse check: %w", err)
|
||||||
|
}
|
||||||
|
if stable != digest {
|
||||||
|
return fmt.Errorf("configuration changed during parse check; retrying")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := w.command(ctx, w.ctl.Configure); err != nil {
|
||||||
|
return fmt.Errorf("configure: %w", err)
|
||||||
|
}
|
||||||
|
w.haveHash = scanErr == nil
|
||||||
|
if w.haveHash {
|
||||||
|
w.hash = digest
|
||||||
|
}
|
||||||
|
w.lastConfigured = now
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *watcher) command(parent context.Context, run func(context.Context) error) error {
|
||||||
|
ctx, cancel := context.WithTimeout(parent, w.cfg.CommandTimeout)
|
||||||
|
defer cancel()
|
||||||
|
return run(ctx)
|
||||||
|
}
|
||||||
@@ -0,0 +1,311 @@
|
|||||||
|
package agentwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type fakeControl struct {
|
||||||
|
parsed int
|
||||||
|
configured int
|
||||||
|
failConfigure int
|
||||||
|
onParse func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeControl) ParseCheck(ctx context.Context, _ string) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.parsed++
|
||||||
|
if f.onParse != nil {
|
||||||
|
f.onParse()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeControl) Configure(ctx context.Context) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f.configured++
|
||||||
|
if f.failConfigure > 0 {
|
||||||
|
f.failConfigure--
|
||||||
|
return errors.New("birdc unavailable")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func testConfig(t *testing.T) (string, string) {
|
||||||
|
t.Helper()
|
||||||
|
root := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(root, "bird.d"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main := filepath.Join(root, "bird.conf")
|
||||||
|
if err := os.WriteFile(main, []byte("router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fragment := filepath.Join(root, "bird.d", "peer.conf")
|
||||||
|
if err := os.WriteFile(fragment, []byte("# peer v1\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return root, fragment
|
||||||
|
}
|
||||||
|
|
||||||
|
func testWatcher(root string, ctl Control) *watcher {
|
||||||
|
return &watcher{ctl: ctl, cfg: Config{ActiveDir: root, Every: 30 * time.Second, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashConfigTreeTracksMainIncludesAndGlobMembership(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
first, err := hashConfigTree(context.Background(), root)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fragment, []byte("# peer v2\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
second, err := hashConfigTree(context.Background(), root)
|
||||||
|
if err != nil || second == first {
|
||||||
|
t.Fatalf("included edit not detected: digest=%x err=%v", second, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.d", "manual.conf"), []byte("# manual\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
third, err := hashConfigTree(context.Background(), root)
|
||||||
|
if err != nil || third == second {
|
||||||
|
t.Fatalf("new glob member not detected: digest=%x err=%v", third, err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fourth, err := hashConfigTree(context.Background(), root)
|
||||||
|
if err != nil || fourth == third {
|
||||||
|
t.Fatalf("main edit not detected: digest=%x err=%v", fourth, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWatchSkipsUnchangedRetriesErrorsAndForcesReconciliation(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
ctl := &fakeControl{}
|
||||||
|
w := testWatcher(root, ctl)
|
||||||
|
now := time.Now()
|
||||||
|
if err := w.step(context.Background(), now); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.step(context.Background(), now.Add(30*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 1 || ctl.parsed != 1 {
|
||||||
|
t.Fatalf("unchanged tree configured again: %+v", ctl)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fragment, []byte("# changed\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctl.failConfigure = 1
|
||||||
|
if err := w.step(context.Background(), now.Add(time.Minute)); err == nil {
|
||||||
|
t.Fatal("expected configure failure")
|
||||||
|
}
|
||||||
|
if !w.haveHash || ctl.configured != 2 {
|
||||||
|
t.Fatalf("failure changed remembered state: %+v", w)
|
||||||
|
}
|
||||||
|
if err := w.step(context.Background(), now.Add(90*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 3 {
|
||||||
|
t.Fatalf("failed change was not retried: %+v", ctl)
|
||||||
|
}
|
||||||
|
if err := w.step(context.Background(), now.Add(7*time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 4 {
|
||||||
|
t.Fatalf("forced reconciliation missing: %+v", ctl)
|
||||||
|
}
|
||||||
|
// A fresh watcher must reconcile once even when another process already loaded the same bytes.
|
||||||
|
restarted := testWatcher(root, ctl)
|
||||||
|
if err := restarted.step(context.Background(), now.Add(8*time.Minute)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 5 {
|
||||||
|
t.Fatalf("restart did not reconcile: %+v", ctl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWatchRejectsChangingTreeBeforeConfigure(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
ctl := &fakeControl{onParse: func() { _ = os.WriteFile(fragment, []byte("# raced\n"), 0o644) }}
|
||||||
|
w := testWatcher(root, ctl)
|
||||||
|
if err := w.step(context.Background(), time.Now()); err == nil || !strings.Contains(err.Error(), "changed during parse check") {
|
||||||
|
t.Fatalf("want race retry, got %v", err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 0 {
|
||||||
|
t.Fatal("configured a tree that changed after parse check")
|
||||||
|
}
|
||||||
|
ctl.onParse = nil
|
||||||
|
if err := w.step(context.Background(), time.Now()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 1 {
|
||||||
|
t.Fatal("stable tree was not configured")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnsupportedIncludeUsesLegacyEveryTickCadence(t *testing.T) {
|
||||||
|
root, _ := testConfig(t)
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include variable;\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctl := &fakeControl{}
|
||||||
|
w := testWatcher(root, ctl)
|
||||||
|
now := time.Now()
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctl.parsed != 3 || ctl.configured != 3 || w.haveHash {
|
||||||
|
t.Fatalf("unsupported tree was skipped: %+v", ctl)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMissingIncludedFileFallsBackButMissingMainWaits(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/peer.conf\";\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(fragment); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctl := &fakeControl{}
|
||||||
|
w := testWatcher(root, ctl)
|
||||||
|
now := time.Now()
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if ctl.configured != 2 {
|
||||||
|
t.Fatalf("missing include skipped legacy cadence: %+v", ctl)
|
||||||
|
}
|
||||||
|
if err := os.Remove(filepath.Join(root, "bird.conf")); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := w.step(context.Background(), now.Add(time.Minute)); !errors.Is(err, errMainNotReady) {
|
||||||
|
t.Fatalf("missing main should wait, got %v", err)
|
||||||
|
}
|
||||||
|
if ctl.configured != 2 {
|
||||||
|
t.Fatal("configured without main bird.conf")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashConfigTreeBoundsAndEscapes(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
if err := os.WriteFile(fragment, make([]byte, maxFileBytes+1), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("want size limit, got %v", err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(fragment, []byte("# small\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outside := filepath.Join(t.TempDir(), "outside.conf")
|
||||||
|
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \""+filepath.ToSlash(outside)+"\";\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "escapes active directory") {
|
||||||
|
t.Fatalf("want outside include rejection, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHashConfigTreeRejectsSymlinkInclude(t *testing.T) {
|
||||||
|
root, fragment := testConfig(t)
|
||||||
|
if err := os.Remove(fragment); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
outside := filepath.Join(t.TempDir(), "outside.conf")
|
||||||
|
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(outside, fragment); err != nil {
|
||||||
|
t.Skipf("symlinks unavailable: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not a regular file") {
|
||||||
|
t.Fatalf("want symlink rejection, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncludeCommentsAndUnsupportedForms(t *testing.T) {
|
||||||
|
patterns, err := includePatterns("# include \"ignored.conf\";\n/*\ninclude \"also-ignored.conf\";\n*/\ninclude \"bird.d/*.conf\";\n")
|
||||||
|
if err != nil || len(patterns) != 1 || patterns[0] != "bird.d/*.conf" {
|
||||||
|
t.Fatalf("comment handling failed: patterns=%v err=%v", patterns, err)
|
||||||
|
}
|
||||||
|
if _, err := includePatterns("if true then include \"dynamic.conf\";\n"); err == nil {
|
||||||
|
t.Fatal("inline include should fall back to legacy cadence")
|
||||||
|
}
|
||||||
|
if _, err := includePatterns("include \"bird.d/[!a].conf\";\n"); err != nil {
|
||||||
|
// Bracket glob is rejected by the scanner after lexical parsing.
|
||||||
|
t.Fatalf("literal include statement should parse before glob validation: %v", err)
|
||||||
|
}
|
||||||
|
root, _ := testConfig(t)
|
||||||
|
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/[!a].conf\";\n"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||||
|
t.Fatalf("bracket glob should fall back, got %v", err)
|
||||||
|
}
|
||||||
|
if _, err := includePatterns("define note = \"single\\\";\ninclude \"bird.d/real.conf\";\n"); err == nil {
|
||||||
|
t.Fatal("backslash in quoted text must force conservative fallback")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBoundedGlobLimitsMatches(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
for i := 0; i <= maxConfigFiles; i++ {
|
||||||
|
name := filepath.Join(root, fmt.Sprintf("peer-%03d.conf", i))
|
||||||
|
if err := os.WriteFile(name, nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := boundedGlob(filepath.Join(root, "*.conf")); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||||
|
t.Fatalf("want bounded glob rejection, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestReadBoundedStopsAfterLimit(t *testing.T) {
|
||||||
|
r := strings.NewReader("abcdefghij")
|
||||||
|
if _, err := readBounded(r, 3); err == nil {
|
||||||
|
t.Fatal("growing file exceeded limit")
|
||||||
|
}
|
||||||
|
if got := r.Len(); got != 6 {
|
||||||
|
t.Fatalf("reader consumed beyond limit+1: %d bytes remain", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunStopsOnCancellation(t *testing.T) {
|
||||||
|
root, _ := testConfig(t)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
ctl := &fakeControl{}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
done <- Run(ctx, ctl, Config{ActiveDir: root, Every: time.Hour, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}})
|
||||||
|
}()
|
||||||
|
cancel()
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(2 * time.Second):
|
||||||
|
t.Fatal("watch did not stop after cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,6 +23,8 @@ const DefaultASOverviewURL = "https://stat.ripe.net/data/as-overview/data.json"
|
|||||||
|
|
||||||
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
|
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
|
||||||
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
|
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
|
||||||
|
ctx, cancel := ripeRequestContext(ctx)
|
||||||
|
defer cancel()
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = httpclient.New(httpclient.DefaultTimeout)
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
@@ -43,13 +45,14 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
|||||||
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
|
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
|
||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
||||||
}
|
}
|
||||||
|
body, err := httpclient.ReadResponseBody(resp, 32<<20)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("ripestat AS%d response: %w", asn, err)
|
||||||
|
}
|
||||||
|
|
||||||
var wrap struct {
|
var wrap struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
@@ -85,6 +88,8 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
|||||||
|
|
||||||
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
|
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
|
||||||
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
|
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
|
||||||
|
ctx, cancel := ripeRequestContext(ctx)
|
||||||
|
defer cancel()
|
||||||
if hc == nil {
|
if hc == nil {
|
||||||
hc = httpclient.New(httpclient.DefaultTimeout)
|
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||||
}
|
}
|
||||||
@@ -105,13 +110,14 @@ func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, erro
|
|||||||
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
|
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
|
||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
||||||
}
|
}
|
||||||
|
body, err := httpclient.ReadResponseBody(resp, 4<<20)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("ripestat as-overview AS%d response: %w", asn, err)
|
||||||
|
}
|
||||||
|
|
||||||
var wrap struct {
|
var wrap struct {
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
@@ -135,3 +141,10 @@ func truncateForErr(b []byte, n int) string {
|
|||||||
}
|
}
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ripeRequestContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||||
|
if _, ok := ctx.Deadline(); ok {
|
||||||
|
return context.WithCancel(ctx)
|
||||||
|
}
|
||||||
|
return context.WithTimeout(ctx, httpclient.DefaultTimeout)
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package asnresolve
|
||||||
|
|
||||||
|
import (
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
|
)
|
||||||
|
|
||||||
|
type ripeRoundTrip func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f ripeRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
type ripeCountingBody struct {
|
||||||
|
io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *ripeCountingBody) Read(p []byte) (int, error) {
|
||||||
|
n, e := b.Reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, e
|
||||||
|
}
|
||||||
|
func (b *ripeCountingBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
func TestRIPECompleteResponseLimits(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
limit int
|
||||||
|
prefix string
|
||||||
|
call func(context.Context, *http.Client) error
|
||||||
|
}{
|
||||||
|
{"announced", 32 << 20, `{"status":"ok","data":{"prefixes":[]}}`, func(ctx context.Context, hc *http.Client) error { _, e := AnnouncedPrefixes(ctx, hc, 64496); return e }},
|
||||||
|
{"holder", 4 << 20, `{"status":"ok","data":{"holder":"Example"}}`, func(ctx context.Context, hc *http.Client) error { _, e := ASHolderName(ctx, hc, 64496); return e }},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
for _, known := range []bool{true, false} {
|
||||||
|
for _, extra := range []int{0, 1} {
|
||||||
|
body := &ripeCountingBody{Reader: strings.NewReader(tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra))}
|
||||||
|
hc := &http.Client{Transport: ripeRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||||
|
if _, ok := r.Context().Deadline(); !ok {
|
||||||
|
t.Error("missing request deadline")
|
||||||
|
}
|
||||||
|
length := int64(-1)
|
||||||
|
if known {
|
||||||
|
length = int64(tc.limit + extra)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Body: body, ContentLength: length, Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
err := tc.call(context.Background(), hc)
|
||||||
|
if extra == 0 && err != nil {
|
||||||
|
t.Fatalf("exact limit rejected known=%v: %v", known, err)
|
||||||
|
}
|
||||||
|
if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) {
|
||||||
|
t.Fatalf("oversized valid prefix accepted known=%v: %v", known, err)
|
||||||
|
}
|
||||||
|
if body.read > tc.limit+1 || !body.closed {
|
||||||
|
t.Fatalf("read=%d closed=%v", body.read, body.closed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRIPEErrorBodySmallAndContextPreserved(t *testing.T) {
|
||||||
|
body := &ripeCountingBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))}
|
||||||
|
hc := &http.Client{Transport: ripeRoundTrip(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
if _, err := AnnouncedPrefixes(context.Background(), hc, 1); err == nil || body.read > 4<<10 || !body.closed {
|
||||||
|
t.Fatalf("non200 unbounded read=%d close=%v err=%v", body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
hc.Transport = ripeRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() })
|
||||||
|
if _, err := ASHolderName(ctx, hc, 1); !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("caller deadline lost: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRIPEGzipExpandedLimit(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Content-Encoding", "gzip")
|
||||||
|
gz := gzip.NewWriter(w)
|
||||||
|
_, _ = io.WriteString(gz, `{"status":"ok","data":{"holder":"Example"}}`+strings.Repeat(" ", 4<<20))
|
||||||
|
_ = gz.Close()
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", server.URL)
|
||||||
|
if _, err := ASHolderName(context.Background(), server.Client(), 1); !errors.Is(err, httpclient.ErrResponseTooLarge) {
|
||||||
|
t.Fatalf("gzip bypassed decoded limit: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+370
-89
@@ -1,152 +1,433 @@
|
|||||||
// Package birddeploy applies revision preview fragments to a shared BIRD volume with parse-check and LKG (plan §13).
|
|
||||||
package birddeploy
|
package birddeploy
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io/fs"
|
"io/fs"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config drives two-phase deploy from a revision in the store.
|
const recoveryTimeout = 30 * time.Second
|
||||||
|
|
||||||
|
var includeLine = regexp.MustCompile(`(?m)^([ \t]*include[ \t]+")([^"]+)("[ \t]*;[ \t]*(?:#[^\n]*)?$)`)
|
||||||
|
|
||||||
|
// Config identifies the BIRD instance. StagingDir remains for caller compatibility;
|
||||||
|
// generations live inside ActiveDir so referenced files survive process restart.
|
||||||
type Config struct {
|
type Config struct {
|
||||||
ActiveDir string // e.g. /etc/bird (live config tree)
|
ActiveDir string
|
||||||
StagingDir string // e.g. /tmp/evobgp-staging (same host/volume as ActiveDir)
|
StagingDir string
|
||||||
BirdBin string
|
BirdBin string
|
||||||
BirdcBin string
|
BirdcBin string
|
||||||
Socket string // birdc -s
|
Socket string
|
||||||
|
writeFile func(string, []byte) error // test-only fault injection; nil uses synced writes
|
||||||
}
|
}
|
||||||
|
|
||||||
// ApplyRevision writes preview fragments to staging, runs bird -p, swaps into active, runs birdc configure; restores LKG on failure.
|
// ApplyRevision installs a checked generation through one atomic bird.conf rename.
|
||||||
func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error {
|
func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error {
|
||||||
if rev == nil || len(rev.PreviewFragments) == 0 {
|
if rev == nil || len(rev.PreviewFragments) == 0 {
|
||||||
return fmt.Errorf("birddeploy: no preview fragments")
|
return fmt.Errorf("birddeploy: no preview fragments")
|
||||||
}
|
}
|
||||||
active := strings.TrimSpace(cfg.ActiveDir)
|
if strings.TrimSpace(cfg.ActiveDir) == "" {
|
||||||
staging := strings.TrimSpace(cfg.StagingDir)
|
return fmt.Errorf("birddeploy: ActiveDir required")
|
||||||
if active == "" || staging == "" {
|
|
||||||
return fmt.Errorf("birddeploy: ActiveDir and StagingDir required")
|
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(staging, 0o755); err != nil {
|
active, err := filepath.Abs(cfg.ActiveDir)
|
||||||
return err
|
if err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: resolve active directory: %w", err)
|
||||||
}
|
}
|
||||||
if err := os.MkdirAll(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
if err := os.MkdirAll(active, 0o755); err != nil {
|
||||||
return err
|
return fmt.Errorf("birddeploy: create active directory: %w", err)
|
||||||
}
|
}
|
||||||
lkg := filepath.Join(active, ".evobgp_lkg")
|
lock, err := lockTarget(ctx, filepath.Join(active, ".evobgp-apply.lock"))
|
||||||
|
if err != nil {
|
||||||
// Phase 1: write staging tree
|
return fmt.Errorf("birddeploy: acquire apply lock: %w", err)
|
||||||
for rel, content := range rev.PreviewFragments {
|
|
||||||
rel = strings.TrimPrefix(rel, "/")
|
|
||||||
if !isDeployableBirdFragment(rel) {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
dst := filepath.Join(staging, rel)
|
defer func() {
|
||||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
if err := lock.Close(); err != nil {
|
||||||
return err
|
fmt.Fprintf(os.Stderr, "birddeploy: release apply lock: %v\n", err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(dst, []byte(content), 0o644); err != nil {
|
}()
|
||||||
return err
|
if err := ctx.Err(); err != nil {
|
||||||
}
|
return fmt.Errorf("birddeploy: apply cancelled: %w", err)
|
||||||
}
|
|
||||||
mainConf := filepath.Join(staging, "bird.conf")
|
|
||||||
if _, err := os.Stat(mainConf); err != nil {
|
|
||||||
return fmt.Errorf("birddeploy: staging missing bird.conf: %w", err)
|
|
||||||
}
|
}
|
||||||
bc := ctl
|
bc := ctl
|
||||||
if bc == nil {
|
if bc == nil {
|
||||||
bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
||||||
}
|
}
|
||||||
if err := bc.ParseCheck(ctx, mainConf); err != nil {
|
writeFile := cfg.writeFile
|
||||||
return fmt.Errorf("birddeploy: bird -p failed: %w", err)
|
if writeFile == nil {
|
||||||
|
writeFile = writeSynced
|
||||||
}
|
}
|
||||||
|
mainPath := filepath.Join(active, "bird.conf")
|
||||||
// Snapshot LKG (best-effort copy of current active bird.conf + bird.d)
|
mainInfo, err := os.Lstat(mainPath)
|
||||||
_ = os.RemoveAll(lkg)
|
if err != nil {
|
||||||
_ = snapshotBirdTree(active, lkg)
|
return fmt.Errorf("birddeploy: inspect current bird.conf: %w", err)
|
||||||
|
|
||||||
// Phase 2: atomic swap into active
|
|
||||||
for rel, content := range rev.PreviewFragments {
|
|
||||||
rel = strings.TrimPrefix(rel, "/")
|
|
||||||
if !isDeployableBirdFragment(rel) {
|
|
||||||
continue
|
|
||||||
}
|
}
|
||||||
dst := filepath.Join(active, rel)
|
if !mainInfo.Mode().IsRegular() {
|
||||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
return fmt.Errorf("birddeploy: current bird.conf must be a regular file; refusing to replace %s", mainInfo.Mode())
|
||||||
_ = restoreLKG(lkg, active)
|
}
|
||||||
|
oldMain, err := os.ReadFile(mainPath)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: read current bird.conf: %w", err)
|
||||||
|
}
|
||||||
|
if err := bc.ParseCheck(ctx, mainPath); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: current configuration is not a usable LKG: %w", err)
|
||||||
|
}
|
||||||
|
gen, err := os.MkdirTemp(active, ".evobgp-generation-")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: create generation: %w", err)
|
||||||
|
}
|
||||||
|
activated := false
|
||||||
|
defer func() {
|
||||||
|
if !activated {
|
||||||
|
if err := os.RemoveAll(gen); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "birddeploy: remove unactivated generation %s: %v\n", gen, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if err := writeGenerationWith(active, gen, rev.PreviewFragments, writeFile); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
tmp := dst + ".tmp"
|
newMain, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||||
if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
|
if err != nil {
|
||||||
_ = restoreLKG(lkg, active)
|
return fmt.Errorf("birddeploy: read generated bird.conf: %w", err)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
if err := os.Rename(tmp, dst); err != nil {
|
if err := bc.ParseCheck(ctx, filepath.Join(gen, "bird.conf")); err != nil {
|
||||||
_ = restoreLKG(lkg, active)
|
return fmt.Errorf("birddeploy: staged bird -p failed: %w", err)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
if err := writeFile(filepath.Join(gen, "previous-bird.conf"), oldMain); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: save LKG: %w", err)
|
||||||
|
}
|
||||||
|
if err := syncDir(gen); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: sync LKG generation: %w", err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: apply cancelled: %w", err)
|
||||||
|
}
|
||||||
|
swapped, err := replaceMain(active, newMain)
|
||||||
|
activated = swapped
|
||||||
|
if err != nil {
|
||||||
|
if swapped {
|
||||||
|
return recoverPrevious(bc, active, gen, oldMain, fmt.Errorf("activate generation: %w", err))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("birddeploy: activate generation: %w", err)
|
||||||
}
|
}
|
||||||
if err := bc.Configure(ctx); err != nil {
|
if err := bc.Configure(ctx); err != nil {
|
||||||
_ = restoreLKG(lkg, active)
|
return recoverPrevious(bc, active, gen, oldMain, err)
|
||||||
_ = bc.Configure(ctx)
|
}
|
||||||
return fmt.Errorf("birddeploy: birdc configure failed, restored LKG: %w", err)
|
if err := pruneGenerations(active, gen, oldMain); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "birddeploy: configuration applied; generation cleanup failed: %v\n", err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// isDeployableBirdFragment skips UI-only preview keys (e.g. flattened full config) not referenced from bird.conf.
|
func recoverPrevious(bc *birdfmt.BirdCtl, active, gen string, oldMain []byte, cause error) error {
|
||||||
func isDeployableBirdFragment(rel string) bool {
|
ctx, cancel := context.WithTimeout(context.Background(), recoveryTimeout)
|
||||||
return rel != "" && !strings.HasPrefix(rel, "_")
|
defer cancel()
|
||||||
|
if _, err := replaceMain(active, oldMain); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: configure failed: %w; restore failed: %v", cause, err)
|
||||||
|
}
|
||||||
|
if err := bc.Configure(ctx); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored on disk but recovery configure failed: %v", cause, err)
|
||||||
|
}
|
||||||
|
if err := os.RemoveAll(gen); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "birddeploy: LKG restored; remove failed generation %s: %v\n", gen, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored and recovery configure succeeded", cause)
|
||||||
}
|
}
|
||||||
|
|
||||||
func snapshotBirdTree(activeRoot, dstRoot string) error {
|
func writeGeneration(active, gen string, fragments map[string]string) error {
|
||||||
if err := os.MkdirAll(dstRoot, 0o755); err != nil {
|
return writeGenerationWith(active, gen, fragments, writeSynced)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeGenerationWith(active, gen string, fragments map[string]string, writeFile func(string, []byte) error) error {
|
||||||
|
main, ok := fragments["bird.conf"]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("birddeploy: preview missing bird.conf")
|
||||||
|
}
|
||||||
|
oldBirdD := filepath.Join(active, birdfmt.DirBirdD)
|
||||||
|
newBirdD := filepath.Join(gen, birdfmt.DirBirdD)
|
||||||
|
if err := os.MkdirAll(newBirdD, 0o755); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: create generation bird.d: %w", err)
|
||||||
|
}
|
||||||
|
if err := filepath.WalkDir(oldBirdD, func(path string, d fs.DirEntry, walkErr error) error {
|
||||||
|
if errors.Is(walkErr, fs.ErrNotExist) && path == oldBirdD {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(oldBirdD, path)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
mc := filepath.Join(activeRoot, "bird.conf")
|
if d.Type()&os.ModeSymlink != 0 {
|
||||||
if b, err := os.ReadFile(mc); err == nil {
|
return fmt.Errorf("operator snippet %q is a symlink", rel)
|
||||||
_ = os.WriteFile(filepath.Join(dstRoot, "bird.conf"), b, 0o644)
|
|
||||||
}
|
}
|
||||||
bd := filepath.Join(activeRoot, "bird.d")
|
if rel == "." {
|
||||||
if st, err := os.Stat(bd); err == nil && st.IsDir() {
|
if !d.IsDir() {
|
||||||
_ = filepath.WalkDir(bd, func(path string, d fs.DirEntry, err error) error {
|
return fmt.Errorf("birddeploy: bird.d is not a directory")
|
||||||
if err != nil || d.IsDir() {
|
}
|
||||||
return err
|
return nil
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return os.MkdirAll(filepath.Join(newBirdD, rel), 0o755)
|
||||||
|
}
|
||||||
|
if !d.Type().IsRegular() {
|
||||||
|
return fmt.Errorf("operator snippet %q is not a regular file", rel)
|
||||||
|
}
|
||||||
|
_, replaced := fragments[filepath.ToSlash(filepath.Join(birdfmt.DirBirdD, rel))]
|
||||||
|
if strings.HasPrefix(filepath.Base(rel), "evobgp_") || replaced {
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
suffix, _ := filepath.Rel(activeRoot, path)
|
|
||||||
out := filepath.Join(dstRoot, suffix)
|
|
||||||
_ = os.MkdirAll(filepath.Dir(out), 0o755)
|
|
||||||
b, err := os.ReadFile(path)
|
b, err := os.ReadFile(path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return err
|
||||||
}
|
}
|
||||||
return os.WriteFile(out, b, 0o644)
|
if countIncludeTokens(string(b)) > 0 {
|
||||||
|
return fmt.Errorf("operator snippet %q has a nested include; cannot pin the complete tree", rel)
|
||||||
|
}
|
||||||
|
return writeFile(filepath.Join(newBirdD, rel), b)
|
||||||
|
}); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: copy operator snippets: %w", err)
|
||||||
|
}
|
||||||
|
for rel, content := range fragments {
|
||||||
|
if rel == "bird.conf" || strings.HasPrefix(rel, "_") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if countIncludeTokens(content) > 0 {
|
||||||
|
return fmt.Errorf("birddeploy: nested include in preview fragment %q is unsupported", rel)
|
||||||
|
}
|
||||||
|
clean := filepath.Clean(filepath.FromSlash(rel))
|
||||||
|
if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) || filepath.Ext(clean) != ".conf" {
|
||||||
|
return fmt.Errorf("birddeploy: unsafe preview path %q", rel)
|
||||||
|
}
|
||||||
|
dst := filepath.Join(gen, clean)
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: create fragment directory: %w", err)
|
||||||
|
}
|
||||||
|
if err := writeFile(dst, []byte(content)); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: write %s: %w", rel, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var rewriteErr error
|
||||||
|
if !standaloneIncludesOnly(main) {
|
||||||
|
return fmt.Errorf("birddeploy: unsupported include syntax in bird.conf")
|
||||||
|
}
|
||||||
|
rewritten := includeLine.ReplaceAllStringFunc(main, func(line string) string {
|
||||||
|
m := includeLine.FindStringSubmatch(line)
|
||||||
|
inc := m[2]
|
||||||
|
if filepath.IsAbs(inc) {
|
||||||
|
rel, err := filepath.Rel(active, inc)
|
||||||
|
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||||
|
rewriteErr = fmt.Errorf("birddeploy: external absolute include %q is unsupported", inc)
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
inc = rel
|
||||||
|
}
|
||||||
|
clean := filepath.Clean(filepath.FromSlash(inc))
|
||||||
|
if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) {
|
||||||
|
rewriteErr = fmt.Errorf("birddeploy: unsupported relative include %q", inc)
|
||||||
|
return line
|
||||||
|
}
|
||||||
|
return m[1] + filepath.ToSlash(filepath.Join(gen, clean)) + m[3]
|
||||||
})
|
})
|
||||||
|
if rewriteErr != nil {
|
||||||
|
return rewriteErr
|
||||||
|
}
|
||||||
|
if err := writeFile(filepath.Join(gen, ".evobgp-owned"), []byte("evobgp-generation-v1\n")); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: mark generation: %w", err)
|
||||||
|
}
|
||||||
|
if err := writeFile(filepath.Join(gen, "bird.conf"), []byte(rewritten)); err != nil {
|
||||||
|
return fmt.Errorf("birddeploy: write generated bird.conf: %w", err)
|
||||||
|
}
|
||||||
|
return syncTree(gen)
|
||||||
|
}
|
||||||
|
|
||||||
|
func standaloneIncludesOnly(config string) bool {
|
||||||
|
tokens := includeTokenOffsets(config)
|
||||||
|
matches := includeLine.FindAllStringIndex(config, -1)
|
||||||
|
if len(tokens) != len(matches) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i, match := range matches {
|
||||||
|
word := strings.Index(config[match[0]:match[1]], "include")
|
||||||
|
if word < 0 || tokens[i] != match[0]+word {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func countIncludeTokens(config string) int { return len(includeTokenOffsets(config)) }
|
||||||
|
|
||||||
|
// includeTokenOffsets skips BIRD comments and strings, retaining exact offsets
|
||||||
|
// so a commented standalone include cannot mask a live inline include.
|
||||||
|
func includeTokenOffsets(config string) []int {
|
||||||
|
var offsets []int
|
||||||
|
quote := byte(0)
|
||||||
|
lineComment, blockComment := false, false
|
||||||
|
for i := 0; i < len(config); {
|
||||||
|
c := config[i]
|
||||||
|
if lineComment {
|
||||||
|
if c == '\n' {
|
||||||
|
lineComment = false
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if blockComment {
|
||||||
|
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
|
||||||
|
blockComment = false
|
||||||
|
i += 2
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if quote != 0 {
|
||||||
|
// BIRD quoted text does not use C-style backslash escapes.
|
||||||
|
// Skipping the following quote could hide a live include.
|
||||||
|
if c == quote {
|
||||||
|
quote = 0
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '#' {
|
||||||
|
lineComment = true
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
|
||||||
|
blockComment = true
|
||||||
|
i += 2
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if c == '"' || c == '\'' {
|
||||||
|
quote = c
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
|
||||||
|
start := i
|
||||||
|
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
if config[start:i] == "include" {
|
||||||
|
offsets = append(offsets, start)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
return offsets
|
||||||
|
}
|
||||||
|
|
||||||
|
func syncTree(root string) error {
|
||||||
|
var dirs []string
|
||||||
|
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
dirs = append(dirs, path)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) })
|
||||||
|
for _, dir := range dirs {
|
||||||
|
if err := syncDir(dir); err != nil {
|
||||||
|
return fmt.Errorf("sync %s: %w", dir, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func restoreLKG(lkg, active string) error {
|
func pruneGenerations(active, current string, previous []byte) error {
|
||||||
if _, err := os.Stat(filepath.Join(lkg, "bird.conf")); err != nil {
|
entries, err := os.ReadDir(active)
|
||||||
return err
|
|
||||||
}
|
|
||||||
_ = filepath.WalkDir(lkg, func(path string, d fs.DirEntry, err error) error {
|
|
||||||
if err != nil || d.IsDir() {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
suffix, _ := filepath.Rel(lkg, path)
|
|
||||||
out := filepath.Join(active, suffix)
|
|
||||||
_ = os.MkdirAll(filepath.Dir(out), 0o755)
|
|
||||||
b, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil
|
return err
|
||||||
|
}
|
||||||
|
var stale []string
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), ".evobgp-generation-") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
path := filepath.Join(active, entry.Name())
|
||||||
|
if path == current || strings.Contains(string(previous), filepath.ToSlash(path)+"/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
marker, err := os.ReadFile(filepath.Join(path, ".evobgp-owned"))
|
||||||
|
if errors.Is(err, fs.ErrNotExist) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if string(marker) != "evobgp-generation-v1\n" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
stale = append(stale, path)
|
||||||
|
}
|
||||||
|
sort.Strings(stale)
|
||||||
|
for _, path := range stale {
|
||||||
|
if err := os.RemoveAll(path); err != nil {
|
||||||
|
return fmt.Errorf("remove %s: %w", path, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return os.WriteFile(out, b, 0o644)
|
|
||||||
})
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func replaceMain(active string, content []byte) (bool, error) {
|
||||||
|
tmp, err := os.CreateTemp(active, ".evobgp-bird-conf-")
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||||
|
if err := tmp.Chmod(0o644); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if _, err := tmp.Write(content); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := tmp.Sync(); err != nil {
|
||||||
|
_ = tmp.Close()
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := tmp.Close(); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmp.Name(), filepath.Join(active, "bird.conf")); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return true, syncDir(active)
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeSynced(path string, b []byte) error {
|
||||||
|
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := f.Write(b); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := f.Sync(); err != nil {
|
||||||
|
_ = f.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return f.Close()
|
||||||
|
}
|
||||||
@@ -0,0 +1,279 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"syscall"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/birdfmt"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
|
||||||
|
t.Helper()
|
||||||
|
bin := t.TempDir()
|
||||||
|
parse := filepath.Join(bin, "bird")
|
||||||
|
configure := filepath.Join(bin, "birdc")
|
||||||
|
parseScript := `#!/bin/sh
|
||||||
|
if grep -q FAIL_PARSE "$2"; then exit 1; fi
|
||||||
|
exit 0
|
||||||
|
`
|
||||||
|
configureScript := fmt.Sprintf(`#!/bin/sh
|
||||||
|
root=%q
|
||||||
|
if [ -e "$root/.configure-wait" ]; then
|
||||||
|
rm "$root/.configure-wait"
|
||||||
|
touch "$root/.configure-started"
|
||||||
|
exec sleep 5
|
||||||
|
fi
|
||||||
|
if [ -e "$root/.configure-fail-once" ]; then
|
||||||
|
rm "$root/.configure-fail-once"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
mkdir "$root/.configuring" || exit 1
|
||||||
|
sleep 0.05
|
||||||
|
rmdir "$root/.configuring"
|
||||||
|
`, active)
|
||||||
|
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
|
||||||
|
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
|
||||||
|
}
|
||||||
|
|
||||||
|
func revision(label string) *store.Revision {
|
||||||
|
return &store.Revision{PreviewFragments: map[string]string{
|
||||||
|
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
|
||||||
|
"bird.d/evobgp_new.conf": "# " + label + "\n",
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
|
||||||
|
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
active := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return active, fakeCtl(t, active), old
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
|
||||||
|
active, ctl, _ := setupActive(t)
|
||||||
|
for _, label := range []string{"one", "two", "three"} {
|
||||||
|
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(main), "# "+label) {
|
||||||
|
t.Fatalf("wrong active generation: %s", main)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(main), `include "bird.d/`) {
|
||||||
|
t.Fatalf("include not pinned: %s", main)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(gens) != 2 {
|
||||||
|
t.Fatalf("want active and previous generations, got %d", len(gens))
|
||||||
|
}
|
||||||
|
for _, gen := range gens {
|
||||||
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
|
||||||
|
t.Fatalf("removed fragment retained in %s: %v", gen, err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
|
||||||
|
t.Fatalf("operator fragment missing: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRevisionFaults(t *testing.T) {
|
||||||
|
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
active, ctl, old := setupActive(t)
|
||||||
|
rev := revision(mode)
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
cfg := Config{ActiveDir: active}
|
||||||
|
switch mode {
|
||||||
|
case "parse":
|
||||||
|
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
|
||||||
|
case "write":
|
||||||
|
rev.PreviewFragments["../escape.conf"] = "bad"
|
||||||
|
case "disk-full":
|
||||||
|
cfg.writeFile = func(path string, b []byte) error {
|
||||||
|
if filepath.Base(path) == "previous-bird.conf" {
|
||||||
|
return syscall.ENOSPC
|
||||||
|
}
|
||||||
|
return writeSynced(path, b)
|
||||||
|
}
|
||||||
|
case "configure":
|
||||||
|
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case "cancel":
|
||||||
|
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result := make(chan error, 1)
|
||||||
|
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
|
||||||
|
if mode == "cancel" {
|
||||||
|
deadline := time.After(3 * time.Second)
|
||||||
|
for {
|
||||||
|
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("configure did not start")
|
||||||
|
case <-time.After(10 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
err := <-result
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected apply failure")
|
||||||
|
}
|
||||||
|
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||||
|
if readErr != nil {
|
||||||
|
t.Fatal(readErr)
|
||||||
|
}
|
||||||
|
if string(main) != string(old) {
|
||||||
|
t.Fatalf("old config not restored: %s; error: %v", main, err)
|
||||||
|
}
|
||||||
|
if mode == "configure" || mode == "cancel" {
|
||||||
|
if !strings.Contains(err.Error(), "recovery configure succeeded") {
|
||||||
|
t.Fatalf("unconfirmed recovery: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRevisionSerializes(t *testing.T) {
|
||||||
|
active, ctl, _ := setupActive(t)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errs := make(chan error, 2)
|
||||||
|
for _, label := range []string{"first", "second"} {
|
||||||
|
wg.Add(1)
|
||||||
|
go func(label string) {
|
||||||
|
defer wg.Done()
|
||||||
|
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
|
||||||
|
}(label)
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errs)
|
||||||
|
for err := range errs {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLockHelperProcess(t *testing.T) {
|
||||||
|
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
|
||||||
|
if active == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = lock.Close() }()
|
||||||
|
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRevisionCrossProcessLock(t *testing.T) {
|
||||||
|
active, ctl, old := setupActive(t)
|
||||||
|
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
|
||||||
|
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
|
||||||
|
if err := cmd.Wait(); err != nil {
|
||||||
|
t.Errorf("lock helper: %v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
deadline := time.After(3 * time.Second)
|
||||||
|
for {
|
||||||
|
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("lock helper did not become ready")
|
||||||
|
case <-time.After(10 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("want lock timeout, got %v", err)
|
||||||
|
}
|
||||||
|
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(main) != string(old) {
|
||||||
|
t.Fatalf("apply bypassed cross-process lock: %s", main)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRealBirdParseGeneration(t *testing.T) {
|
||||||
|
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
|
||||||
|
if bin == "" {
|
||||||
|
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
|
||||||
|
}
|
||||||
|
active := t.TempDir()
|
||||||
|
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctl := fakeCtl(t, active)
|
||||||
|
ctl.Bird = bin
|
||||||
|
rev := &store.Revision{PreviewFragments: map[string]string{
|
||||||
|
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
|
||||||
|
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
|
||||||
|
}}
|
||||||
|
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
|
||||||
|
t.Fatalf("activated config fails real bird -p: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestWriteGenerationPinsCompleteIncludeTree(t *testing.T) {
|
||||||
|
active := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte("# operator"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.d", "evobgp_removed.conf"), []byte("# removed"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fragments := map[string]string{"bird.conf": "include \"bird.d/*.conf\";\n", "bird.d/evobgp_new.conf": "# new"}
|
||||||
|
if err := writeGeneration(active, gen, fragments); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
main, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(main), filepath.ToSlash(filepath.Join(gen, "bird.d", "*.conf"))) {
|
||||||
|
t.Fatalf("include is not pinned to generation: %s", main)
|
||||||
|
}
|
||||||
|
for _, name := range []string{"operator.conf", "evobgp_new.conf"} {
|
||||||
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", name)); err != nil {
|
||||||
|
t.Fatalf("%s missing: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("removed fragment retained: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteGenerationRejectsUnsafePaths(t *testing.T) {
|
||||||
|
for _, bad := range []string{"../escape.conf", "bird.d/../../escape.conf", "other.conf"} {
|
||||||
|
t.Run(bad, func(t *testing.T) {
|
||||||
|
active := t.TempDir()
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", bad: "bad"}
|
||||||
|
if err := writeGeneration(active, gen, fragments); err == nil {
|
||||||
|
t.Fatal("unsafe path accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteGenerationRejectsExternalAndNestedIncludes(t *testing.T) {
|
||||||
|
for _, main := range []string{
|
||||||
|
`include "/outside/bird.d/peer.conf";` + "\n",
|
||||||
|
`include "other/peer.conf";` + "\n",
|
||||||
|
`include "bird.d/peer.conf"; include "other.conf";` + "\n",
|
||||||
|
`router id 192.0.2.1; include "/outside/peer.conf";` + "\n",
|
||||||
|
"/*\ninclude \"bird.d/decoy.conf\";\n*/\nrouter id 192.0.2.1; include \"/outside/live.conf\";\n",
|
||||||
|
} {
|
||||||
|
active := t.TempDir()
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main}); err == nil {
|
||||||
|
t.Fatalf("include accepted: %s", main)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
active := t.TempDir()
|
||||||
|
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte(`define X=1; include "/outside/foo.conf";`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
|
||||||
|
t.Fatal("nested operator include accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCountIncludeTokensIgnoresCommentsAndStrings(t *testing.T) {
|
||||||
|
content := "# include \"ignored.conf\";\n" +
|
||||||
|
"/* include \"ignored.conf\"; */\n" +
|
||||||
|
"define note = \"include in string\";\n" +
|
||||||
|
"include \"bird.d/real.conf\";\n"
|
||||||
|
if got := countIncludeTokens(content); got != 1 {
|
||||||
|
t.Fatalf("want one include, got %d", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIncludeAfterBackslashStringCannotHideNestedInclude(t *testing.T) {
|
||||||
|
content := `define note = "\";` + "\n" + `include "/outside/hidden.conf";` + "\n"
|
||||||
|
if got := countIncludeTokens(content); got != 1 {
|
||||||
|
t.Fatalf("live include hidden after quoted backslash: %d", got)
|
||||||
|
}
|
||||||
|
active := t.TempDir()
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", "bird.d/evobgp_test.conf": content}
|
||||||
|
if err := writeGeneration(active, gen, fragments); err == nil {
|
||||||
|
t.Fatal("nested external include accepted after quoted backslash")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteGenerationRewritesAbsoluteActiveInclude(t *testing.T) {
|
||||||
|
active := t.TempDir()
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
inc := filepath.ToSlash(filepath.Join(active, "bird.d", "peer.conf"))
|
||||||
|
main := "include \"" + inc + "\";\n"
|
||||||
|
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main, "bird.d/peer.conf": "# peer"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(string(b), filepath.ToSlash(filepath.Join(gen, "bird.d", "peer.conf"))) {
|
||||||
|
t.Fatalf("absolute include not rewritten: %s", b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWriteGenerationRejectsSymlinkBirdDRoot(t *testing.T) {
|
||||||
|
active := t.TempDir()
|
||||||
|
outside := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(outside, "operator.conf"), []byte("# operator"), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(outside, filepath.Join(active, "bird.d")); err != nil {
|
||||||
|
t.Skipf("symlinks unavailable: %v", err)
|
||||||
|
}
|
||||||
|
old := []byte("router id 192.0.2.2;\n")
|
||||||
|
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||||
|
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
|
||||||
|
t.Fatal("bird.d symlink accepted")
|
||||||
|
}
|
||||||
|
current, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(current) != string(old) {
|
||||||
|
t.Fatalf("active config changed: %s", current)
|
||||||
|
}
|
||||||
|
if _, err := os.ReadFile(filepath.Join(outside, "operator.conf")); err != nil {
|
||||||
|
t.Fatalf("operator snippet changed: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyRevisionRejectsSymlinkMain(t *testing.T) {
|
||||||
|
active := t.TempDir()
|
||||||
|
target := filepath.Join(t.TempDir(), "operator-bird.conf")
|
||||||
|
old := []byte("router id 192.0.2.2;\n")
|
||||||
|
if err := os.WriteFile(target, old, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Symlink(target, filepath.Join(active, "bird.conf")); err != nil {
|
||||||
|
t.Skipf("symlinks unavailable: %v", err)
|
||||||
|
}
|
||||||
|
err := ApplyRevision(context.Background(), nil, &store.Revision{PreviewFragments: map[string]string{"bird.conf": "router id 192.0.2.1;\n"}}, Config{ActiveDir: active})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||||
|
t.Fatalf("want explicit symlink rejection, got %v", err)
|
||||||
|
}
|
||||||
|
info, err := os.Lstat(filepath.Join(active, "bird.conf"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Mode()&os.ModeSymlink == 0 {
|
||||||
|
t.Fatal("active symlink replaced")
|
||||||
|
}
|
||||||
|
current, err := os.ReadFile(target)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(current) != string(old) {
|
||||||
|
t.Fatalf("operator target changed: %s", current)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
//go:build linux
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
func lockTarget(ctx context.Context, path string) (*os.File, error) {
|
||||||
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
|
||||||
|
if err == nil {
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) {
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(25 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
//go:build !linux && !windows
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
)
|
||||||
|
|
||||||
|
func lockTarget(context.Context, string) (*os.File, error) {
|
||||||
|
return nil, errors.New("birddeploy: cross-process apply lock is unsupported on this platform")
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"golang.org/x/sys/windows"
|
||||||
|
)
|
||||||
|
|
||||||
|
func lockTarget(ctx context.Context, path string) (*os.File, error) {
|
||||||
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var overlapped windows.Overlapped
|
||||||
|
for {
|
||||||
|
err = windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped)
|
||||||
|
if err == nil {
|
||||||
|
return f, nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
_ = f.Close()
|
||||||
|
return nil, ctx.Err()
|
||||||
|
case <-time.After(25 * time.Millisecond):
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
//go:build !windows
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
import "os"
|
||||||
|
|
||||||
|
func syncDir(path string) error {
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return f.Sync()
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
//go:build windows
|
||||||
|
|
||||||
|
package birddeploy
|
||||||
|
|
||||||
|
// Windows does not support syncing a directory handle through os.File.Sync.
|
||||||
|
func syncDir(string) error { return nil }
|
||||||
+78
-12
@@ -3,34 +3,100 @@ package bundle
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
pathpkg "path"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed).
|
// WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed).
|
||||||
func WriteExtractedFiles(root string, v *VerifiedContents) error {
|
func WriteExtractedFiles(root string, v *VerifiedContents) error {
|
||||||
|
return WriteExtractedFilesWithLimits(root, v, Limits{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// WriteExtractedFilesWithLimits confines all filesystem operations to an open
|
||||||
|
// directory handle. Replacing existing files uses unlink + exclusive creation,
|
||||||
|
// so existing hard links and symlinks cannot redirect writes outside the root.
|
||||||
|
func WriteExtractedFilesWithLimits(root string, v *VerifiedContents, limits Limits) error {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if v == nil {
|
||||||
|
return fmt.Errorf("bundle: nil verified contents")
|
||||||
|
}
|
||||||
|
if len(v.Manifest.Files) > l.MaxEntries-2 {
|
||||||
|
return ErrLimit
|
||||||
|
}
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var total int64
|
||||||
|
for _, fe := range v.Manifest.Files {
|
||||||
|
if err := validatePath(fe.Path, l.MaxPathBytes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if seen[strings.ToLower(fe.Path)] {
|
||||||
|
return fmt.Errorf("bundle: duplicate path %q", fe.Path)
|
||||||
|
}
|
||||||
|
seen[strings.ToLower(fe.Path)] = true
|
||||||
|
data, ok := v.Files[fe.Path]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||||
|
}
|
||||||
|
if int64(len(data)) > l.MaxFileBytes || int64(len(data)) > l.MaxExpandedBytes-total {
|
||||||
|
return ErrLimit
|
||||||
|
}
|
||||||
|
total += int64(len(data))
|
||||||
|
}
|
||||||
root = filepath.Clean(root)
|
root = filepath.Clean(root)
|
||||||
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
confined, err := os.OpenRoot(root)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer func() { _ = confined.Close() }()
|
||||||
for _, fe := range v.Manifest.Files {
|
for _, fe := range v.Manifest.Files {
|
||||||
data, ok := v.Files[fe.Path]
|
data, ok := v.Files[fe.Path]
|
||||||
if !ok {
|
if !ok {
|
||||||
return fmt.Errorf("bundle: missing file %q", fe.Path)
|
return fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||||
}
|
}
|
||||||
rel := pathpkg.Clean(strings.TrimPrefix(fe.Path, "/"))
|
parts := strings.Split(fe.Path, "/")
|
||||||
if rel == "." || strings.HasPrefix(rel, "..") {
|
for i := 1; i < len(parts); i++ {
|
||||||
return fmt.Errorf("bundle: unsafe path %q", fe.Path)
|
dir := filepath.FromSlash(strings.Join(parts[:i], "/"))
|
||||||
}
|
if err := confined.Mkdir(dir, 0o755); err != nil && !os.IsExist(err) {
|
||||||
dest := filepath.Join(root, filepath.FromSlash(rel))
|
|
||||||
relToRoot, err := filepath.Rel(root, dest)
|
|
||||||
if err != nil || strings.HasPrefix(relToRoot, "..") {
|
|
||||||
return fmt.Errorf("bundle: path escapes root: %q", fe.Path)
|
|
||||||
}
|
|
||||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(dest, data, 0o644); err != nil {
|
info, err := confined.Lstat(dir)
|
||||||
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||||
|
return fmt.Errorf("bundle: unsafe directory %q", dir)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dest := filepath.FromSlash(fe.Path)
|
||||||
|
info, err := confined.Lstat(dest)
|
||||||
|
if err == nil {
|
||||||
|
if !info.Mode().IsRegular() {
|
||||||
|
return fmt.Errorf("bundle: unsafe destination %q", dest)
|
||||||
|
}
|
||||||
|
if err = confined.Remove(dest); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f, err := confined.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, writeErr := f.Write(data)
|
||||||
|
closeErr := f.Close()
|
||||||
|
if writeErr != nil {
|
||||||
|
return writeErr
|
||||||
|
}
|
||||||
|
if closeErr != nil {
|
||||||
|
return closeErr
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"sort"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// legacyManifestV1 mirrors the decoder fields present in the locally available
|
||||||
|
// v1.0.0 source. It deliberately has no future capability or deployment fields.
|
||||||
|
type legacyManifestV1 struct {
|
||||||
|
RevisionID string `json:"revision_id"`
|
||||||
|
SpeakerID string `json:"speaker_id"`
|
||||||
|
CreatedAt string `json:"created_at"`
|
||||||
|
Files []struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
SHA256 string `json:"sha256"`
|
||||||
|
} `json:"files"`
|
||||||
|
Algorithm string `json:"signature_algorithm"`
|
||||||
|
PublicKeyBase64 string `json:"public_key_base64"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyCompatibilityFrozenSignedCurrentBaseline(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("testdata/current_baseline_signed_bundle.tar.gz")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pub := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{0x42}, ed25519.SeedSize)).Public().(ed25519.PublicKey)
|
||||||
|
verified, err := VerifyGzippedTar(raw, pub)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("frozen signed baseline rejected: %v", err)
|
||||||
|
}
|
||||||
|
if verified.Manifest.RevisionID != "revision-legacy" || verified.Manifest.SpeakerID != "speaker-legacy" || len(verified.Files) != 2 {
|
||||||
|
t.Fatalf("frozen baseline contents: %+v", verified.Manifest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyCompatibilityBundleWire(t *testing.T) {
|
||||||
|
seed := bytes.Repeat([]byte{0x42}, ed25519.SeedSize)
|
||||||
|
priv := ed25519.NewKeyFromSeed(seed)
|
||||||
|
pub := priv.Public().(ed25519.PublicKey)
|
||||||
|
fragments := map[string]string{
|
||||||
|
"bird.conf": "router id 192.0.2.1;\n",
|
||||||
|
"bird.d/evobgp_prefixes_v4.conf": "# revision\n",
|
||||||
|
"_preview": "must not ship",
|
||||||
|
}
|
||||||
|
raw, err := BuildGzippedTar("revision-legacy", "speaker-legacy", fragments, priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = gr.Close() }()
|
||||||
|
tr := tar.NewReader(gr)
|
||||||
|
entries := map[string][]byte{}
|
||||||
|
var order []string
|
||||||
|
for {
|
||||||
|
h, err := tr.Next()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(tr)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
order = append(order, h.Name)
|
||||||
|
entries[h.Name] = body
|
||||||
|
}
|
||||||
|
wantOrder := []string{"manifest.json", "bundle.sig", "bird.conf", "bird.d/evobgp_prefixes_v4.conf"}
|
||||||
|
if !reflect.DeepEqual(order, wantOrder) {
|
||||||
|
t.Fatalf("legacy tar layout: got %v, want %v", order, wantOrder)
|
||||||
|
}
|
||||||
|
var top map[string]json.RawMessage
|
||||||
|
if err := json.Unmarshal(entries["manifest.json"], &top); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantFields := []string{"created_at", "files", "public_key_base64", "revision_id", "signature_algorithm", "speaker_id"}
|
||||||
|
fields := make([]string, 0, len(top))
|
||||||
|
for k := range top {
|
||||||
|
fields = append(fields, k)
|
||||||
|
}
|
||||||
|
sort.Strings(fields)
|
||||||
|
if !reflect.DeepEqual(fields, wantFields) {
|
||||||
|
t.Fatalf("manifest fields: got %v, want %v", fields, wantFields)
|
||||||
|
}
|
||||||
|
var m legacyManifestV1
|
||||||
|
if err := json.Unmarshal(entries["manifest.json"], &m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m.RevisionID != "revision-legacy" || m.SpeakerID != "speaker-legacy" || m.CreatedAt == "" || m.Algorithm != "ed25519" || m.PublicKeyBase64 != base64.StdEncoding.EncodeToString(pub) {
|
||||||
|
t.Fatalf("legacy manifest fields: %+v", m)
|
||||||
|
}
|
||||||
|
if !ed25519.Verify(pub, entries["manifest.json"], entries["bundle.sig"]) {
|
||||||
|
t.Fatal("legacy Ed25519 verifier rejected current signed manifest")
|
||||||
|
}
|
||||||
|
if len(m.Files) != 2 {
|
||||||
|
t.Fatalf("legacy manifest file count: %d", len(m.Files))
|
||||||
|
}
|
||||||
|
for _, f := range m.Files {
|
||||||
|
sum := sha256.Sum256(entries[f.Path])
|
||||||
|
if hex.EncodeToString(sum[:]) != f.SHA256 {
|
||||||
|
t.Fatalf("legacy SHA-256 mismatch for %q", f.Path)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrLimit reports that a transport, archive or extraction budget was exceeded.
|
||||||
|
var ErrLimit = errors.New("bundle: resource limit exceeded")
|
||||||
|
|
||||||
|
// Limits bounds both archive transport and decoded tar data, including metadata.
|
||||||
|
// Zero fields select defaults. These ceilings permit large (1M-prefix) configs;
|
||||||
|
// they are safety ceilings, not a promise of suitability for a small-RAM node.
|
||||||
|
type Limits struct {
|
||||||
|
MaxCompressedBytes int64
|
||||||
|
MaxExpandedBytes int64
|
||||||
|
MaxFileBytes int64
|
||||||
|
MaxManifestBytes int64
|
||||||
|
MaxEntries int
|
||||||
|
MaxPathBytes int
|
||||||
|
}
|
||||||
|
|
||||||
|
// DefaultLimits permits large production configs while bounding untrusted input.
|
||||||
|
func DefaultLimits() Limits {
|
||||||
|
return Limits{MaxCompressedBytes: 128 << 20, MaxExpandedBytes: 512 << 20, MaxFileBytes: 256 << 20, MaxManifestBytes: 2 << 20, MaxEntries: 4096, MaxPathBytes: 1024}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Normalized fills zero fields with defaults and rejects invalid budgets.
|
||||||
|
func (l Limits) Normalized() (Limits, error) {
|
||||||
|
d := DefaultLimits()
|
||||||
|
values := []struct {
|
||||||
|
v *int64
|
||||||
|
fallback int64
|
||||||
|
}{{&l.MaxCompressedBytes, d.MaxCompressedBytes}, {&l.MaxExpandedBytes, d.MaxExpandedBytes}, {&l.MaxFileBytes, d.MaxFileBytes}, {&l.MaxManifestBytes, d.MaxManifestBytes}}
|
||||||
|
for _, x := range values {
|
||||||
|
if *x.v < 0 || *x.v == int64(^uint64(0)>>1) {
|
||||||
|
return l, fmt.Errorf("bundle: invalid byte limit")
|
||||||
|
}
|
||||||
|
if *x.v == 0 {
|
||||||
|
*x.v = x.fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if l.MaxEntries < 0 || l.MaxPathBytes < 0 {
|
||||||
|
return l, fmt.Errorf("bundle: invalid entry/path limit")
|
||||||
|
}
|
||||||
|
if l.MaxEntries == 0 {
|
||||||
|
l.MaxEntries = d.MaxEntries
|
||||||
|
}
|
||||||
|
if l.MaxPathBytes == 0 {
|
||||||
|
l.MaxPathBytes = d.MaxPathBytes
|
||||||
|
}
|
||||||
|
return l, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validatePath(name string, limit int) error {
|
||||||
|
if name == "" || len(name) > limit || path.IsAbs(name) || path.Clean(name) != name || name == "." || strings.ContainsAny(name, "\\:\x00") {
|
||||||
|
return fmt.Errorf("bundle: unsafe path %q", name)
|
||||||
|
}
|
||||||
|
for _, part := range strings.Split(name, "/") {
|
||||||
|
if part == ".." || strings.TrimRight(part, " .") != part {
|
||||||
|
return fmt.Errorf("bundle: unsafe path %q", name)
|
||||||
|
}
|
||||||
|
base := strings.ToUpper(strings.SplitN(part, ".", 2)[0])
|
||||||
|
if base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || (len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9') {
|
||||||
|
return fmt.Errorf("bundle: reserved path %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type limitedWriter struct {
|
||||||
|
w io.Writer
|
||||||
|
remaining int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *limitedWriter) Write(p []byte) (int, error) {
|
||||||
|
if int64(len(p)) > w.remaining {
|
||||||
|
return 0, ErrLimit
|
||||||
|
}
|
||||||
|
n, err := w.w.Write(p)
|
||||||
|
w.remaining -= int64(n)
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReadCompressed rejects oversized input while reading, including chunked HTTP
|
||||||
|
// bodies and local files whose size changes after stat.
|
||||||
|
func ReadCompressed(r io.Reader, limits Limits) ([]byte, error) {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
b, err := io.ReadAll(io.LimitReader(r, l.MaxCompressedBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(b)) > l.MaxCompressedBytes {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,286 @@
|
|||||||
|
package bundle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type testEntry struct {
|
||||||
|
header tar.Header
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveEntries(t *testing.T, entries []testEntry) []byte {
|
||||||
|
t.Helper()
|
||||||
|
var b bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&b)
|
||||||
|
tw := tar.NewWriter(gz)
|
||||||
|
for _, entry := range entries {
|
||||||
|
h := entry.header
|
||||||
|
if h.Typeflag == 0 {
|
||||||
|
h.Typeflag = tar.TypeReg
|
||||||
|
}
|
||||||
|
h.Size = int64(len(entry.body))
|
||||||
|
if err := tw.WriteHeader(&h); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tw.Write(entry.body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := gz.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b.Bytes()
|
||||||
|
}
|
||||||
|
func testKey() ed25519.PrivateKey {
|
||||||
|
return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleResourceLimits(t *testing.T) {
|
||||||
|
pub := testKey().Public().(ed25519.PublicKey)
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
entries []testEntry
|
||||||
|
limits Limits
|
||||||
|
}{
|
||||||
|
{"file", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxFileBytes: 1024}},
|
||||||
|
{"expanded", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 8192)}}, Limits{MaxExpandedBytes: 4096}},
|
||||||
|
{"manifest", []testEntry{{header: tar.Header{Name: manifestName}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxManifestBytes: 1024}},
|
||||||
|
{"entries", []testEntry{{header: tar.Header{Name: "a"}}, {header: tar.Header{Name: "b"}}, {header: tar.Header{Name: "c"}}}, Limits{MaxEntries: 2}},
|
||||||
|
{"pax-metadata", []testEntry{{header: tar.Header{Name: "bird.conf", Format: tar.FormatPAX, PAXRecords: map[string]string{"comment": strings.Repeat("a", 8192)}}}}, Limits{MaxExpandedBytes: 4096}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
raw := archiveEntries(t, tc.entries)
|
||||||
|
if _, err := VerifyGzippedTarWithLimits(raw, pub, tc.limits); !errors.Is(err, ErrLimit) {
|
||||||
|
t.Fatalf("want limit, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
if _, err := VerifyGzippedTarWithLimits(make([]byte, 65), pub, Limits{MaxCompressedBytes: 64}); !errors.Is(err, ErrLimit) {
|
||||||
|
t.Fatalf("compressed cap: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleRejectsUnsafeEntries(t *testing.T) {
|
||||||
|
pub := testKey().Public().(ed25519.PublicKey)
|
||||||
|
for _, name := range []string{"../escape", "/etc/bird.conf", `bird.d\escape`, "C:/escape", "bird.d/../escape", "a//b", "a/./b", "NUL", "bird.conf:stream", "a. /b"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: name}, body: []byte("x")}})
|
||||||
|
if _, err := VerifyGzippedTar(raw, pub); err == nil {
|
||||||
|
t.Fatal("unsafe path accepted")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
for _, kind := range []byte{tar.TypeSymlink, tar.TypeLink, tar.TypeDir, tar.TypeFifo} {
|
||||||
|
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf", Typeflag: kind, Linkname: "outside"}}})
|
||||||
|
if _, err := VerifyGzippedTar(raw, pub); err == nil {
|
||||||
|
t.Fatalf("type %d accepted", kind)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf"}}, {header: tar.Header{Name: "bird.conf"}}})
|
||||||
|
if _, err := VerifyGzippedTar(raw, pub); err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||||
|
t.Fatalf("duplicate accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleChecksTrailerAndTrailingExpansion(t *testing.T) {
|
||||||
|
priv := testKey()
|
||||||
|
pub := priv.Public().(ed25519.PublicKey)
|
||||||
|
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "router id 192.0.2.1;"}, priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
corrupt := append([]byte{}, raw...)
|
||||||
|
corrupt[len(corrupt)-8] ^= 0xff
|
||||||
|
if _, err = VerifyGzippedTar(corrupt, pub); err == nil {
|
||||||
|
t.Fatal("gzip checksum ignored after tar EOF")
|
||||||
|
}
|
||||||
|
var extra bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&extra)
|
||||||
|
if _, err = gz.Write(bytes.Repeat([]byte{'x'}, 32<<10)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = gz.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
joined := append(append([]byte{}, raw...), extra.Bytes()...)
|
||||||
|
if _, err = VerifyGzippedTarWithLimits(joined, pub, Limits{MaxExpandedBytes: 8 << 10}); !errors.Is(err, ErrLimit) {
|
||||||
|
t.Fatalf("trailing gzip member bypassed budget: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleRejectsPAXSparseRegularFile(t *testing.T) {
|
||||||
|
// archive/tar expands PAX sparse holes even though the returned type is a
|
||||||
|
// regular file. Build a physical one-byte entry with a 4096-byte logical size.
|
||||||
|
record := func(key, value string) string {
|
||||||
|
body := key + "=" + value + "\n"
|
||||||
|
n := len(body) + 2
|
||||||
|
for {
|
||||||
|
next := len(fmt.Sprint(n)) + 1 + len(body)
|
||||||
|
if next == n {
|
||||||
|
return fmt.Sprintf("%d %s", n, body)
|
||||||
|
}
|
||||||
|
n = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pax := record("GNU.sparse.size", "4096") + record("GNU.sparse.numblocks", "1") + record("GNU.sparse.map", "0,1")
|
||||||
|
var physical bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&physical)
|
||||||
|
for _, entry := range []testEntry{{header: tar.Header{Name: "PaxHeader", Size: int64(len(pax))}, body: []byte(pax)}, {header: tar.Header{Name: "bird.conf", Size: 1}, body: []byte("x")}} {
|
||||||
|
if err := tw.WriteHeader(&entry.header); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tw.Write(entry.body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data := physical.Bytes()
|
||||||
|
data[156] = tar.TypeXHeader
|
||||||
|
for i := 148; i < 156; i++ {
|
||||||
|
data[i] = ' '
|
||||||
|
}
|
||||||
|
sum := 0
|
||||||
|
for _, b := range data[:512] {
|
||||||
|
sum += int(b)
|
||||||
|
}
|
||||||
|
copy(data[148:156], fmt.Sprintf("%06o\x00 ", sum))
|
||||||
|
h, err := tar.NewReader(bytes.NewReader(data)).Next()
|
||||||
|
if err != nil || h.Typeflag != tar.TypeReg || h.Size != 4096 {
|
||||||
|
t.Fatalf("invalid sparse regression fixture %+v %v", h, err)
|
||||||
|
}
|
||||||
|
var compressed bytes.Buffer
|
||||||
|
gz := gzip.NewWriter(&compressed)
|
||||||
|
if _, err = gz.Write(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = gz.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = VerifyGzippedTar(compressed.Bytes(), testKey().Public().(ed25519.PublicKey)); err == nil || !strings.Contains(err.Error(), "sparse") {
|
||||||
|
t.Fatalf("PAX sparse accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleRejectsUnmanifestedPayload(t *testing.T) {
|
||||||
|
priv := testKey()
|
||||||
|
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "x"}, priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = gr.Close() }()
|
||||||
|
tr := tar.NewReader(gr)
|
||||||
|
var entries []testEntry
|
||||||
|
for {
|
||||||
|
h, e := tr.Next()
|
||||||
|
if e == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
b, e := io.ReadAll(tr)
|
||||||
|
if e != nil {
|
||||||
|
t.Fatal(e)
|
||||||
|
}
|
||||||
|
entries = append(entries, testEntry{header: *h, body: b})
|
||||||
|
}
|
||||||
|
entries = append(entries, testEntry{header: tar.Header{Name: "unsigned.conf"}, body: []byte("unsigned")})
|
||||||
|
if _, err = VerifyGzippedTar(archiveEntries(t, entries), priv.Public().(ed25519.PublicKey)); err == nil {
|
||||||
|
t.Fatal("unmanifested payload accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractionDoesNotFollowLinks(t *testing.T) {
|
||||||
|
root := t.TempDir()
|
||||||
|
outside := filepath.Join(t.TempDir(), "outside")
|
||||||
|
if err := os.WriteFile(outside, []byte("preserve"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dest := filepath.Join(root, "bird.conf")
|
||||||
|
if err := os.Link(outside, dest); err != nil {
|
||||||
|
if runtime.GOOS == "linux" {
|
||||||
|
t.Fatalf("Linux link regression requires hard links: %v", err)
|
||||||
|
}
|
||||||
|
t.Skipf("hard links unavailable: %v", err)
|
||||||
|
}
|
||||||
|
v := &VerifiedContents{Manifest: Manifest{Files: []FileEntry{{Path: "bird.conf"}}}, Files: map[string][]byte{"bird.conf": []byte("replacement")}}
|
||||||
|
if err := WriteExtractedFiles(root, v); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := os.ReadFile(outside)
|
||||||
|
if err != nil || string(got) != "preserve" {
|
||||||
|
t.Fatalf("hard link modified outside: %s %v", got, err)
|
||||||
|
}
|
||||||
|
if err = os.Remove(dest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = os.Symlink(outside, dest); err != nil {
|
||||||
|
if runtime.GOOS == "linux" {
|
||||||
|
t.Fatalf("Linux link regression requires symlinks: %v", err)
|
||||||
|
}
|
||||||
|
t.Skipf("symlinks unavailable: %v", err)
|
||||||
|
}
|
||||||
|
if err = WriteExtractedFiles(root, v); err == nil {
|
||||||
|
t.Fatal("symlink destination accepted")
|
||||||
|
}
|
||||||
|
got, err = os.ReadFile(outside)
|
||||||
|
if err != nil || string(got) != "preserve" {
|
||||||
|
t.Fatal("symlink modified outside")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleBuilderBoundsAndPortablePaths(t *testing.T) {
|
||||||
|
for _, name := range []string{"/bird.conf", "a/../bird.conf", manifestName, sigFileName, `bird.d\bad`} {
|
||||||
|
if _, err := BuildGzippedTar("r", "s", map[string]string{name: "x"}, testKey()); err == nil {
|
||||||
|
t.Fatalf("invalid builder path accepted: %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": strings.Repeat("x", 1025)}, testKey(), Limits{MaxFileBytes: 1024}); !errors.Is(err, ErrLimit) {
|
||||||
|
t.Fatalf("builder file cap %v", err)
|
||||||
|
}
|
||||||
|
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": "x"}, testKey(), Limits{MaxExpandedBytes: 1024}); !errors.Is(err, ErrLimit) {
|
||||||
|
t.Fatalf("builder tar overhead cap %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// This is a decoded-size regression using repeated representative route lines,
|
||||||
|
// not a unique-prefix cardinality test or a BIRD validity/performance benchmark.
|
||||||
|
func TestDefaultLimitsAllowMillionRepresentativeIPv6RouteLines(t *testing.T) {
|
||||||
|
if testing.Short() {
|
||||||
|
t.Skip("large representative config")
|
||||||
|
}
|
||||||
|
config := strings.Repeat("route 2001:db8:1234:5678::/64 blackhole;\n", 1_000_000)
|
||||||
|
raw, err := BuildGzippedTar("large", "speaker", map[string]string{"bird.conf": config}, testKey())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v, err := VerifyGzippedTar(raw, testKey().Public().(ed25519.PublicKey))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(v.Files["bird.conf"]) != len(config) {
|
||||||
|
t.Fatal("large config truncated")
|
||||||
|
}
|
||||||
|
}
|
||||||
+35
-8
@@ -9,7 +9,6 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -36,22 +35,47 @@ const manifestName = "manifest.json"
|
|||||||
|
|
||||||
// BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files.
|
// BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files.
|
||||||
func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||||
|
return BuildGzippedTarWithLimits(revisionID, speakerID, fragments, priv, Limits{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// BuildGzippedTarWithLimits preserves the manifest/signature wire format while
|
||||||
|
// refusing payloads that exceed the receiving node's configured budgets.
|
||||||
|
func BuildGzippedTarWithLimits(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey, limits Limits) ([]byte, error) {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
if len(priv) != ed25519.PrivateKeySize {
|
if len(priv) != ed25519.PrivateKeySize {
|
||||||
return nil, fmt.Errorf("bundle: invalid ed25519 private key size")
|
return nil, fmt.Errorf("bundle: invalid ed25519 private key size")
|
||||||
}
|
}
|
||||||
pub := priv.Public().(ed25519.PublicKey)
|
pub := priv.Public().(ed25519.PublicKey)
|
||||||
|
|
||||||
norm := make(map[string]string, len(fragments))
|
norm := make(map[string]string)
|
||||||
|
seen := map[string]bool{}
|
||||||
|
var total int64
|
||||||
for p, content := range fragments {
|
for p, content := range fragments {
|
||||||
p = path.Clean(strings.TrimPrefix(p, "/"))
|
|
||||||
if p == "." || strings.HasPrefix(p, "..") {
|
|
||||||
return nil, fmt.Errorf("bundle: invalid path %q", p)
|
|
||||||
}
|
|
||||||
if strings.HasPrefix(p, "_") {
|
if strings.HasPrefix(p, "_") {
|
||||||
continue // UI-only preview aggregate, not shipped to nodes
|
continue // UI-only preview aggregate, not shipped to nodes
|
||||||
}
|
}
|
||||||
|
if len(norm) >= l.MaxEntries-2 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
if err := validatePath(p, l.MaxPathBytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if p == manifestName || p == sigFileName || seen[strings.ToLower(p)] {
|
||||||
|
return nil, fmt.Errorf("bundle: duplicate or reserved path %q", p)
|
||||||
|
}
|
||||||
|
seen[strings.ToLower(p)] = true
|
||||||
|
if int64(len(content)) > l.MaxFileBytes || int64(len(content)) > l.MaxExpandedBytes-total {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
total += int64(len(content))
|
||||||
norm[p] = content
|
norm[p] = content
|
||||||
}
|
}
|
||||||
|
if len(norm) > l.MaxEntries-2 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
|
||||||
var files []FileEntry
|
var files []FileEntry
|
||||||
var payload [][]byte
|
var payload [][]byte
|
||||||
@@ -75,11 +99,14 @@ func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if int64(len(manifestJSON)) > l.MaxManifestBytes {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
sig := ed25519.Sign(priv, manifestJSON)
|
sig := ed25519.Sign(priv, manifestJSON)
|
||||||
|
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
gw := gzip.NewWriter(&buf)
|
gw := gzip.NewWriter(&limitedWriter{w: &buf, remaining: l.MaxCompressedBytes})
|
||||||
tw := tar.NewWriter(gw)
|
tw := tar.NewWriter(&limitedWriter{w: gw, remaining: l.MaxExpandedBytes})
|
||||||
|
|
||||||
add := func(name string, body []byte) error {
|
add := func(name string, body []byte) error {
|
||||||
hdr := &tar.Header{
|
hdr := &tar.Header{
|
||||||
|
|||||||
Binary file not shown.
+92
-18
@@ -11,44 +11,95 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"path"
|
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
// VerifiedContents is the result of verifying a downloaded bundle on evobgp-node.
|
// VerifiedContents contains only payloads covered by a verified signed manifest.
|
||||||
type VerifiedContents struct {
|
type VerifiedContents struct {
|
||||||
Manifest Manifest
|
Manifest Manifest
|
||||||
Files map[string][]byte
|
Files map[string][]byte
|
||||||
}
|
}
|
||||||
|
|
||||||
// VerifyGzippedTar checks Ed25519 signature over manifest.json and SHA-256 of each file.
|
// VerifyGzippedTar verifies the unchanged signed manifest format with default limits.
|
||||||
func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, error) {
|
func VerifyGzippedTar(raw []byte, pub ed25519.PublicKey) (*VerifiedContents, error) {
|
||||||
gr, err := gzip.NewReader(bytes.NewReader(bundle))
|
return VerifyGzippedTarWithLimits(raw, pub, Limits{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// VerifyGzippedTarWithLimits verifies the signature and hashes under explicit
|
||||||
|
// compressed, expanded, per-entry and metadata budgets.
|
||||||
|
func VerifyGzippedTarWithLimits(raw []byte, pub ed25519.PublicKey, limits Limits) (*VerifiedContents, error) {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(raw)) > l.MaxCompressedBytes {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
if len(pub) != ed25519.PublicKeySize {
|
||||||
|
return nil, fmt.Errorf("bundle: invalid public key size")
|
||||||
|
}
|
||||||
|
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
defer func() { _ = gr.Close() }()
|
defer func() { _ = gr.Close() }()
|
||||||
|
decoded := &io.LimitedReader{R: gr, N: l.MaxExpandedBytes + 1}
|
||||||
var manifestRaw []byte
|
tr := tar.NewReader(decoded)
|
||||||
var sig []byte
|
var manifestRaw, sig []byte
|
||||||
files := make(map[string][]byte)
|
files := map[string][]byte{}
|
||||||
tr := tar.NewReader(gr)
|
seen := map[string]bool{}
|
||||||
|
count := 0
|
||||||
|
var logicalBytes int64
|
||||||
for {
|
for {
|
||||||
hdr, err := tr.Next()
|
hdr, err := tr.Next()
|
||||||
|
if decoded.N == 0 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
if err == io.EOF {
|
if err == io.EOF {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
name := path.Clean(hdr.Name)
|
count++
|
||||||
if strings.HasPrefix(name, "..") {
|
if count > l.MaxEntries {
|
||||||
return nil, fmt.Errorf("bundle: illegal tar entry %q", hdr.Name)
|
return nil, ErrLimit
|
||||||
}
|
}
|
||||||
data, err := io.ReadAll(tr)
|
if err = validatePath(hdr.Name, l.MaxPathBytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
name := hdr.Name
|
||||||
|
folded := strings.ToLower(name)
|
||||||
|
if seen[folded] {
|
||||||
|
return nil, fmt.Errorf("bundle: duplicate tar entry %q", name)
|
||||||
|
}
|
||||||
|
seen[folded] = true
|
||||||
|
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
||||||
|
return nil, fmt.Errorf("bundle: nonregular tar entry %q", name)
|
||||||
|
}
|
||||||
|
for key := range hdr.PAXRecords {
|
||||||
|
if strings.HasPrefix(key, "GNU.sparse.") {
|
||||||
|
return nil, fmt.Errorf("bundle: sparse entry %q is unsupported", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
max := l.MaxFileBytes
|
||||||
|
if name == manifestName {
|
||||||
|
max = l.MaxManifestBytes
|
||||||
|
}
|
||||||
|
if name == sigFileName {
|
||||||
|
max = ed25519.SignatureSize
|
||||||
|
}
|
||||||
|
if hdr.Size < 0 || hdr.Size > max || hdr.Size >= decoded.N || hdr.Size > l.MaxExpandedBytes-logicalBytes {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
logicalBytes += hdr.Size
|
||||||
|
data, err := io.ReadAll(io.LimitReader(tr, max+1))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if int64(len(data)) > max || decoded.N == 0 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
switch name {
|
switch name {
|
||||||
case manifestName:
|
case manifestName:
|
||||||
manifestRaw = data
|
manifestRaw = data
|
||||||
@@ -58,17 +109,36 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
|
|||||||
files[name] = data
|
files[name] = data
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(manifestRaw) == 0 || len(sig) == 0 {
|
// tar EOF precedes the gzip trailer. Drain through the same byte budget so
|
||||||
|
// checksums, extra gzip members and trailing decompression are also bounded.
|
||||||
|
if _, err = io.Copy(io.Discard, decoded); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if decoded.N == 0 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
if len(manifestRaw) == 0 || len(sig) != ed25519.SignatureSize {
|
||||||
return nil, fmt.Errorf("bundle: missing manifest or signature")
|
return nil, fmt.Errorf("bundle: missing manifest or signature")
|
||||||
}
|
}
|
||||||
if !ed25519.Verify(pub, manifestRaw, sig) {
|
if !ed25519.Verify(pub, manifestRaw, sig) {
|
||||||
return nil, fmt.Errorf("bundle: ed25519 signature mismatch")
|
return nil, fmt.Errorf("bundle: ed25519 signature mismatch")
|
||||||
}
|
}
|
||||||
var m Manifest
|
var m Manifest
|
||||||
if err := json.Unmarshal(manifestRaw, &m); err != nil {
|
if err = json.Unmarshal(manifestRaw, &m); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
if len(m.Files) > l.MaxEntries-2 {
|
||||||
|
return nil, ErrLimit
|
||||||
|
}
|
||||||
|
listed := map[string]bool{}
|
||||||
for _, fe := range m.Files {
|
for _, fe := range m.Files {
|
||||||
|
if err = validatePath(fe.Path, l.MaxPathBytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if fe.Path == manifestName || fe.Path == sigFileName || listed[strings.ToLower(fe.Path)] {
|
||||||
|
return nil, fmt.Errorf("bundle: duplicate or reserved manifest path %q", fe.Path)
|
||||||
|
}
|
||||||
|
listed[strings.ToLower(fe.Path)] = true
|
||||||
body, ok := files[fe.Path]
|
body, ok := files[fe.Path]
|
||||||
if !ok {
|
if !ok {
|
||||||
return nil, fmt.Errorf("bundle: missing file %q", fe.Path)
|
return nil, fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||||
@@ -78,10 +148,13 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
|
|||||||
return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path)
|
return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if len(listed) != len(files) {
|
||||||
|
return nil, fmt.Errorf("bundle: payload is not covered by manifest")
|
||||||
|
}
|
||||||
return &VerifiedContents{Manifest: m, Files: files}, nil
|
return &VerifiedContents{Manifest: m, Files: files}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ParsePublicKeyHex decodes a 64-char hex Ed25519 public key (32 bytes).
|
// ParsePublicKeyHex decodes a 32-byte Ed25519 public key from hexadecimal.
|
||||||
func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
|
func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
|
||||||
s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X"))
|
s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X"))
|
||||||
raw, err := hex.DecodeString(s)
|
raw, err := hex.DecodeString(s)
|
||||||
@@ -94,7 +167,8 @@ func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
|
|||||||
return ed25519.PublicKey(raw), nil
|
return ed25519.PublicKey(raw), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// PublicKeyFromManifestBase64 uses the key embedded in the signed manifest (cross-check with expected pubkey optional).
|
// PublicKeyFromManifestBase64 decodes the manifest key. Callers must still trust
|
||||||
|
// an independently provisioned key when authenticating the bundle.
|
||||||
func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) {
|
func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) {
|
||||||
if m.PublicKeyB64 == "" {
|
if m.PublicKeyB64 == "" {
|
||||||
return nil, fmt.Errorf("bundle: manifest missing public_key_base64")
|
return nil, fmt.Errorf("bundle: manifest missing public_key_base64")
|
||||||
|
|||||||
+265
-21
@@ -8,48 +8,292 @@ import (
|
|||||||
"path"
|
"path"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
)
|
)
|
||||||
|
|
||||||
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree (idempotent via schema_migrations).
|
const (
|
||||||
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) error {
|
postgresMigrationLockKey int64 = 0x45766f4247504d47 // "EvoBGPMG", scoped to the database.
|
||||||
if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil {
|
migrationLockTimeout = 30 * time.Second
|
||||||
return fmt.Errorf("db: schema_migrations: %w", err)
|
migrationCleanupTimeout = 5 * time.Second
|
||||||
}
|
)
|
||||||
|
|
||||||
|
// PostgreSQL migrations are transaction-only. Non-transactional statements
|
||||||
|
// need an explicit, separately reviewed execution path.
|
||||||
|
|
||||||
|
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree.
|
||||||
|
// Each migration and its version row commit atomically under a session advisory lock.
|
||||||
|
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) (retErr error) {
|
||||||
entries, err := fs.ReadDir(fsys, subdir)
|
entries, err := fs.ReadDir(fsys, subdir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("db: read migrations: %w", err)
|
||||||
}
|
}
|
||||||
var ups []string
|
var ups []string
|
||||||
for _, e := range entries {
|
for _, e := range entries {
|
||||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") {
|
if !e.IsDir() && strings.HasSuffix(e.Name(), ".up.sql") {
|
||||||
continue
|
|
||||||
}
|
|
||||||
ups = append(ups, e.Name())
|
ups = append(ups, e.Name())
|
||||||
}
|
}
|
||||||
|
}
|
||||||
sort.Strings(ups)
|
sort.Strings(ups)
|
||||||
for _, name := range ups {
|
// Include pool acquisition in the lock timeout. The lock precedes even
|
||||||
ver := strings.TrimSuffix(name, ".up.sql")
|
// schema_migrations creation, because another process may start from an empty DB.
|
||||||
var dummy int
|
lockCtx, cancelLock := context.WithTimeout(ctx, migrationLockTimeout)
|
||||||
err := pool.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, ver).Scan(&dummy)
|
conn, err := pool.Acquire(lockCtx)
|
||||||
|
if err != nil {
|
||||||
|
cancelLock()
|
||||||
|
return fmt.Errorf("db: acquire migration connection: %w", err)
|
||||||
|
}
|
||||||
|
defer conn.Release()
|
||||||
|
if _, err := conn.Exec(lockCtx, `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
|
||||||
|
cancelLock()
|
||||||
|
// Cancellation may race with a successful server-side lock acquisition.
|
||||||
|
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||||
|
defer closeCancel()
|
||||||
|
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
|
||||||
|
return errors.Join(fmt.Errorf("db: acquire migration lock: %w", err), fmt.Errorf("db: close migration connection: %w", closeErr))
|
||||||
|
}
|
||||||
|
return fmt.Errorf("db: acquire migration lock: %w", err)
|
||||||
|
}
|
||||||
|
cancelLock()
|
||||||
|
defer func() {
|
||||||
|
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
var unlocked bool
|
||||||
|
if err := conn.QueryRow(cleanupCtx, `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
|
||||||
if err == nil {
|
if err == nil {
|
||||||
continue
|
err = errors.New("lock was not held")
|
||||||
}
|
}
|
||||||
if !errors.Is(err, pgx.ErrNoRows) {
|
retErr = errors.Join(retErr, fmt.Errorf("db: release migration lock: %w", err))
|
||||||
return err
|
// Never return a possibly locked session to the pool.
|
||||||
|
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||||
|
defer closeCancel()
|
||||||
|
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
|
||||||
|
retErr = errors.Join(retErr, fmt.Errorf("db: close migration connection: %w", closeErr))
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil {
|
||||||
|
return fmt.Errorf("db: schema_migrations: %w", err)
|
||||||
|
}
|
||||||
|
for _, name := range ups {
|
||||||
body, err := fs.ReadFile(fsys, path.Join(subdir, name))
|
body, err := fs.ReadFile(fsys, path.Join(subdir, name))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return fmt.Errorf("db: read migration %s: %w", name, err)
|
||||||
}
|
}
|
||||||
if _, err := pool.Exec(ctx, string(body)); err != nil {
|
if bad := unsupportedMigrationStatement(string(body)); bad != "" {
|
||||||
|
return fmt.Errorf("db: migration %s uses unsupported non-transactional SQL %q", name, bad)
|
||||||
|
}
|
||||||
|
if err := applyPostgresMigration(ctx, conn, strings.TrimSuffix(name, ".up.sql"), body); err != nil {
|
||||||
return fmt.Errorf("db: migrate %s: %w", name, err)
|
return fmt.Errorf("db: migrate %s: %w", name, err)
|
||||||
}
|
}
|
||||||
if _, err := pool.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, ver); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// unsupportedMigrationStatement inspects statement starts after masking SQL
|
||||||
|
// comments and quoted strings. In particular, a comment after a semicolon
|
||||||
|
// must not conceal COMMIT or another command that would escape our transaction.
|
||||||
|
func unsupportedMigrationStatement(sql string) string {
|
||||||
|
const (
|
||||||
|
normal = iota
|
||||||
|
singleQuote
|
||||||
|
doubleQuote
|
||||||
|
lineComment
|
||||||
|
blockComment
|
||||||
|
dollarQuote
|
||||||
|
)
|
||||||
|
state := normal
|
||||||
|
blockDepth := 0
|
||||||
|
dollarTag := ""
|
||||||
|
escapeString := false
|
||||||
|
var clean strings.Builder
|
||||||
|
for i := 0; i < len(sql); {
|
||||||
|
switch state {
|
||||||
|
case normal:
|
||||||
|
switch {
|
||||||
|
case sql[i] == '\'':
|
||||||
|
escapeString = i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !isSQLIdentifierByte(sql[i-2]))
|
||||||
|
state = singleQuote
|
||||||
|
clean.WriteByte(' ')
|
||||||
|
i++
|
||||||
|
case sql[i] == '"':
|
||||||
|
state = doubleQuote
|
||||||
|
clean.WriteByte(' ')
|
||||||
|
i++
|
||||||
|
case strings.HasPrefix(sql[i:], "--"):
|
||||||
|
state = lineComment
|
||||||
|
clean.WriteByte(' ')
|
||||||
|
i += 2
|
||||||
|
case strings.HasPrefix(sql[i:], "/*"):
|
||||||
|
state = blockComment
|
||||||
|
blockDepth = 1
|
||||||
|
clean.WriteByte(' ')
|
||||||
|
i += 2
|
||||||
|
case sql[i] == '$' && (i == 0 || !isSQLIdentifierByte(sql[i-1])):
|
||||||
|
end := i + 1
|
||||||
|
if end < len(sql) && isDollarTagStartByte(sql[end]) {
|
||||||
|
end++
|
||||||
|
for end < len(sql) && isDollarTagByte(sql[end]) {
|
||||||
|
end++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if end < len(sql) && sql[end] == '$' {
|
||||||
|
dollarTag = sql[i : end+1]
|
||||||
|
state = dollarQuote
|
||||||
|
clean.WriteByte(' ')
|
||||||
|
i = end + 1
|
||||||
|
} else {
|
||||||
|
clean.WriteByte(sql[i])
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
clean.WriteByte(sql[i])
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case singleQuote:
|
||||||
|
switch {
|
||||||
|
case escapeString && sql[i] == '\\' && i+1 < len(sql):
|
||||||
|
i += 2
|
||||||
|
case sql[i] == '\'' && i+1 < len(sql) && sql[i+1] == '\'':
|
||||||
|
i += 2
|
||||||
|
case sql[i] == '\'':
|
||||||
|
state = normal
|
||||||
|
i++
|
||||||
|
default:
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case doubleQuote:
|
||||||
|
if sql[i] == '"' {
|
||||||
|
if i+1 < len(sql) && sql[i+1] == '"' {
|
||||||
|
i += 2
|
||||||
|
} else {
|
||||||
|
state = normal
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case lineComment:
|
||||||
|
if sql[i] == '\n' || sql[i] == '\r' {
|
||||||
|
clean.WriteByte('\n')
|
||||||
|
state = normal
|
||||||
|
}
|
||||||
|
i++
|
||||||
|
case blockComment:
|
||||||
|
switch {
|
||||||
|
case strings.HasPrefix(sql[i:], "/*"):
|
||||||
|
blockDepth++
|
||||||
|
i += 2
|
||||||
|
case strings.HasPrefix(sql[i:], "*/"):
|
||||||
|
blockDepth--
|
||||||
|
i += 2
|
||||||
|
if blockDepth == 0 {
|
||||||
|
state = normal
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
case dollarQuote:
|
||||||
|
if strings.HasPrefix(sql[i:], dollarTag) {
|
||||||
|
i += len(dollarTag)
|
||||||
|
state = normal
|
||||||
|
} else {
|
||||||
|
i++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, statement := range strings.Split(clean.String(), ";") {
|
||||||
|
words := strings.Fields(strings.ToUpper(statement))
|
||||||
|
if isUnsupportedMigrationCommand(words) {
|
||||||
|
return strings.Join(words[:min(len(words), 4)], " ")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func isSQLIdentifierByte(b byte) bool {
|
||||||
|
return isDollarTagByte(b) || b == '$'
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDollarTagStartByte(b byte) bool {
|
||||||
|
return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b == '_' || b >= 0x80
|
||||||
|
}
|
||||||
|
|
||||||
|
func isDollarTagByte(b byte) bool {
|
||||||
|
return isDollarTagStartByte(b) || b >= '0' && b <= '9'
|
||||||
|
}
|
||||||
|
|
||||||
|
func isUnsupportedMigrationCommand(words []string) bool {
|
||||||
|
if len(words) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
second := ""
|
||||||
|
if len(words) > 1 {
|
||||||
|
second = words[1]
|
||||||
|
}
|
||||||
|
switch words[0] {
|
||||||
|
case "BEGIN", "COMMIT", "END", "ROLLBACK", "ABORT", "SAVEPOINT", "REINDEX", "VACUUM", "CLUSTER":
|
||||||
|
return true
|
||||||
|
case "START", "PREPARE":
|
||||||
|
return second == "TRANSACTION"
|
||||||
|
case "RELEASE":
|
||||||
|
return second == "SAVEPOINT" || second != ""
|
||||||
|
case "ALTER":
|
||||||
|
return second == "SYSTEM"
|
||||||
|
case "CREATE":
|
||||||
|
if second == "DATABASE" || second == "TABLESPACE" {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
idx := 2
|
||||||
|
if second == "UNIQUE" {
|
||||||
|
idx = 3
|
||||||
|
}
|
||||||
|
return len(words) > idx && words[idx-1] == "INDEX" && words[idx] == "CONCURRENTLY"
|
||||||
|
case "DROP":
|
||||||
|
return second == "DATABASE" || second == "TABLESPACE" || second == "INDEX" && len(words) > 2 && words[2] == "CONCURRENTLY"
|
||||||
|
case "REFRESH":
|
||||||
|
return len(words) > 3 && second == "MATERIALIZED" && words[2] == "VIEW" && words[3] == "CONCURRENTLY"
|
||||||
|
case "DISCARD":
|
||||||
|
return second == "ALL"
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func applyPostgresMigration(ctx context.Context, conn *pgxpool.Conn, version string, body []byte) (retErr error) {
|
||||||
|
tx, err := conn.Begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
committed := false
|
||||||
|
defer func() {
|
||||||
|
if committed {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||||
|
defer cancel()
|
||||||
|
if err := tx.Rollback(cleanupCtx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
|
||||||
|
retErr = errors.Join(retErr, fmt.Errorf("db: rollback migration %s: %w", version, err))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
var present int
|
||||||
|
err = tx.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, version).Scan(&present)
|
||||||
|
if err == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if !errors.Is(err, pgx.ErrNoRows) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, string(body)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, version); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := tx.Commit(ctx); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
committed = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io/fs"
|
||||||
|
"path"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/migrations"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPostgresEmbeddedMigrationsTransactional(t *testing.T) {
|
||||||
|
entries, err := fs.ReadDir(migrations.Postgres, "postgres")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, entry := range entries {
|
||||||
|
if !strings.HasSuffix(entry.Name(), ".up.sql") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
body, err := fs.ReadFile(migrations.Postgres, path.Join("postgres", entry.Name()))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if bad := unsupportedMigrationStatement(string(body)); bad != "" {
|
||||||
|
t.Errorf("%s contains unsupported transaction control or DDL: %q", entry.Name(), bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationRejectsTransactionControl(t *testing.T) {
|
||||||
|
for _, statement := range []string{
|
||||||
|
"COMMIT;", "END;", "ROLLBACK;", "ABORT;", "BEGIN;",
|
||||||
|
"CREATE INDEX CONCURRENTLY idx ON t (id);", "VACUUM t;",
|
||||||
|
"CREATE TABLE t(id int); /* boundary */ COMMIT; SELECT 1/0;",
|
||||||
|
"/* comment */COMMIT;",
|
||||||
|
"-- comment\nCOMMIT;",
|
||||||
|
"/* nested /* comment */ still comment */ END;",
|
||||||
|
"CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$",
|
||||||
|
"-- comment\rCOMMIT;",
|
||||||
|
`foo$E'abc\'; COMMIT;`,
|
||||||
|
`éE'abc\'; COMMIT;`,
|
||||||
|
} {
|
||||||
|
if unsupportedMigrationStatement(statement) == "" {
|
||||||
|
t.Errorf("did not reject %q", statement)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationIgnoresQuotedControlWords(t *testing.T) {
|
||||||
|
for _, statement := range []string{
|
||||||
|
"SELECT 'COMMIT;'::text;",
|
||||||
|
"SELECT $$COMMIT;$$;",
|
||||||
|
"SELECT $tag$END;$tag$;",
|
||||||
|
"SELECT E'ABORT;\\'';",
|
||||||
|
"SELECT \"ROLLBACK\" FROM t;",
|
||||||
|
"-- COMMIT;\nSELECT 1;",
|
||||||
|
} {
|
||||||
|
if bad := unsupportedMigrationStatement(statement); bad != "" {
|
||||||
|
t.Errorf("rejected quoted or commented SQL %q: %s", statement, bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+24
-11
@@ -25,17 +25,7 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if max := os.Getenv("EVOBGP_DB_MAX_CONNS"); max != "" {
|
applyPostgresPoolConfig(cfg)
|
||||||
if n, err := strconv.Atoi(strings.TrimSpace(max)); err == nil && n > 0 {
|
|
||||||
cfg.MaxConns = int32(n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if min := os.Getenv("EVOBGP_DB_MIN_CONNS"); min != "" {
|
|
||||||
if n, err := strconv.Atoi(strings.TrimSpace(min)); err == nil && n >= 0 {
|
|
||||||
cfg.MinConns = int32(n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cfg.MaxConnLifetime = 30 * time.Minute
|
|
||||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -47,6 +37,29 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
|||||||
return pool, nil
|
return pool, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// applyPostgresPoolConfig sets pool limits. An explicit EVOBGP_DB_MAX_CONNS is kept.
|
||||||
|
// When it is unset, the pool defaults to 8 connections so it can cover the default
|
||||||
|
// worker count instead of pgx's default of 4.
|
||||||
|
func applyPostgresPoolConfig(cfg *pgxpool.Config) {
|
||||||
|
if cfg == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if max := strings.TrimSpace(os.Getenv("EVOBGP_DB_MAX_CONNS")); max != "" {
|
||||||
|
if n, err := strconv.Atoi(max); err == nil && n > 0 {
|
||||||
|
cfg.MaxConns = int32(n)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cfg.MaxConns = 8
|
||||||
|
}
|
||||||
|
if min := strings.TrimSpace(os.Getenv("EVOBGP_DB_MIN_CONNS")); min != "" {
|
||||||
|
if n, err := strconv.Atoi(min); err == nil && n >= 0 {
|
||||||
|
cfg.MinConns = int32(n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cfg.MaxConnLifetime = 30 * time.Minute
|
||||||
|
cfg.MaxConnIdleTime = 5 * time.Minute
|
||||||
|
}
|
||||||
|
|
||||||
// OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on).
|
// OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on).
|
||||||
func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) {
|
func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) {
|
||||||
dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:"))
|
dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:"))
|
||||||
|
|||||||
@@ -0,0 +1,34 @@
|
|||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestApplyPostgresPoolConfigDefaultAndExplicit(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_DB_MAX_CONNS", "")
|
||||||
|
t.Setenv("EVOBGP_DB_MIN_CONNS", "")
|
||||||
|
cfg, err := pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
applyPostgresPoolConfig(cfg)
|
||||||
|
if cfg.MaxConns != 8 {
|
||||||
|
t.Fatalf("default max conns: %d", cfg.MaxConns)
|
||||||
|
}
|
||||||
|
if cfg.MaxConnIdleTime != 5*time.Minute {
|
||||||
|
t.Fatalf("idle time: %s", cfg.MaxConnIdleTime)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Setenv("EVOBGP_DB_MAX_CONNS", "25")
|
||||||
|
cfg, err = pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
applyPostgresPoolConfig(cfg)
|
||||||
|
if cfg.MaxConns != 25 {
|
||||||
|
t.Fatalf("explicit max conns overridden: %d", cfg.MaxConns)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
package db
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/migrations"
|
||||||
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPostgresMigrationsIntegration(t *testing.T) {
|
||||||
|
pool := isolatedMigrationPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||||
|
t.Fatalf("first migration pass: %v", err)
|
||||||
|
}
|
||||||
|
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||||
|
t.Fatalf("second migration pass: %v", err)
|
||||||
|
}
|
||||||
|
files, err := fs.ReadDir(migrations.Postgres, "postgres")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := 0
|
||||||
|
for _, file := range files {
|
||||||
|
if strings.HasSuffix(file.Name(), ".up.sql") {
|
||||||
|
want++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var got int
|
||||||
|
if err := pool.QueryRow(ctx, `SELECT count(*)::int FROM schema_migrations`).Scan(&got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("applied migrations = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
var extension string
|
||||||
|
if err := pool.QueryRow(ctx, `SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements'`).Scan(&extension); err != nil {
|
||||||
|
t.Fatalf("pg_stat_statements extension missing: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationConcurrentApplyIntegration(t *testing.T) {
|
||||||
|
pool := isolatedMigrationPool(t)
|
||||||
|
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE migration_probe (id integer PRIMARY KEY); SELECT pg_sleep(0.1);`)}}
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errors := make(chan error, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
errors <- ApplyPostgresMigrations(context.Background(), pool, files, "sql")
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errors)
|
||||||
|
for err := range errors {
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertMigrationVersionCount(t, pool, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationRollbackIntegration(t *testing.T) {
|
||||||
|
pool := isolatedMigrationPool(t)
|
||||||
|
files := fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); SELECT 1 / 0;`)}}
|
||||||
|
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil {
|
||||||
|
t.Fatal("expected migration error")
|
||||||
|
}
|
||||||
|
assertMigrationVersionCount(t, pool, 0)
|
||||||
|
var exists bool
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatal("failed migration left a table behind")
|
||||||
|
}
|
||||||
|
// A transaction terminator hidden behind a comment must be rejected before
|
||||||
|
// the first DDL statement can run.
|
||||||
|
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); /* boundary */ COMMIT; SELECT 1 / 0;`)}}
|
||||||
|
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
|
||||||
|
t.Fatalf("comment-hidden COMMIT error = %v", err)
|
||||||
|
}
|
||||||
|
assertMigrationVersionCount(t, pool, 0)
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatal("rejected migration left a table behind")
|
||||||
|
}
|
||||||
|
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$`)}}
|
||||||
|
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
|
||||||
|
t.Fatalf("identifier-embedded dollar tag error = %v", err)
|
||||||
|
}
|
||||||
|
assertMigrationVersionCount(t, pool, 0)
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('foo$tag$') IS NOT NULL`).Scan(&exists); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatal("identifier-embedded dollar tag left a table behind")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationIdempotentIntegration(t *testing.T) {
|
||||||
|
pool := isolatedMigrationPool(t)
|
||||||
|
files := fstest.MapFS{"sql/000001_once.up.sql": {Data: []byte(`CREATE TABLE migration_once (id integer); INSERT INTO migration_once VALUES (1);`)}}
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assertMigrationVersionCount(t, pool, 1)
|
||||||
|
var rows int
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM migration_once`).Scan(&rows); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if rows != 1 {
|
||||||
|
t.Fatalf("migration rows = %d, want 1", rows)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPostgresMigrationLockCancellationIntegration(t *testing.T) {
|
||||||
|
pool := isolatedMigrationPool(t)
|
||||||
|
locker, err := pool.Acquire(context.Background())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer locker.Release()
|
||||||
|
if _, err := locker.Exec(context.Background(), `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_, _ = locker.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey)
|
||||||
|
}()
|
||||||
|
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE lock_probe (id integer);`)}}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
err = ApplyPostgresMigrations(ctx, pool, files, "sql")
|
||||||
|
if !errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
t.Fatalf("lock wait error = %v, want context deadline exceeded", err)
|
||||||
|
}
|
||||||
|
var exists bool
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('schema_migrations') IS NOT NULL`).Scan(&exists); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if exists {
|
||||||
|
t.Fatal("schema_migrations was created before migration lock acquisition")
|
||||||
|
}
|
||||||
|
var unlocked bool
|
||||||
|
if err := locker.QueryRow(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
|
||||||
|
t.Fatalf("release held lock: unlocked=%v err=%v", unlocked, err)
|
||||||
|
}
|
||||||
|
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
|
||||||
|
t.Fatalf("apply after lock release: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func isolatedMigrationPool(t *testing.T) *pgxpool.Pool {
|
||||||
|
t.Helper()
|
||||||
|
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||||
|
if dsn == "" {
|
||||||
|
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
admin, err := pgxpool.New(ctx, dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var nonce [8]byte
|
||||||
|
if _, err := rand.Read(nonce[:]); err != nil {
|
||||||
|
admin.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
schema := "evobgp_mig_test_" + hex.EncodeToString(nonce[:])
|
||||||
|
if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil {
|
||||||
|
admin.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var pool *pgxpool.Pool
|
||||||
|
t.Cleanup(func() {
|
||||||
|
if pool != nil {
|
||||||
|
pool.Close()
|
||||||
|
}
|
||||||
|
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cleanupCancel()
|
||||||
|
if _, err := admin.Exec(cleanupCtx, `DROP SCHEMA `+schema+` CASCADE`); err != nil {
|
||||||
|
t.Errorf("drop isolated schema %s: %v", schema, err)
|
||||||
|
}
|
||||||
|
admin.Close()
|
||||||
|
})
|
||||||
|
cfg, err := pgxpool.ParseConfig(dsn)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cfg.ConnConfig.RuntimeParams == nil {
|
||||||
|
cfg.ConnConfig.RuntimeParams = make(map[string]string)
|
||||||
|
}
|
||||||
|
cfg.ConnConfig.RuntimeParams["search_path"] = schema
|
||||||
|
cfg.MaxConns = 3
|
||||||
|
pool, err = pgxpool.NewWithConfig(ctx, cfg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return pool
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertMigrationVersionCount(t *testing.T, pool *pgxpool.Pool, want int) {
|
||||||
|
t.Helper()
|
||||||
|
var got int
|
||||||
|
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM schema_migrations`).Scan(&got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got != want {
|
||||||
|
t.Fatalf("migration versions = %d, want %d", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// These regressions ensure unobserved live state cannot inherit persisted health.
|
||||||
|
func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) {
|
||||||
|
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
views []speakerBGPLive
|
||||||
|
}{
|
||||||
|
{name: "no_collector"},
|
||||||
|
{name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}},
|
||||||
|
{name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
row := map[string]any{"session_state": "Established"}
|
||||||
|
applyPeerLiveFields(row, peer, tc.views)
|
||||||
|
if row["session_state"] == "Established" {
|
||||||
|
t.Fatal("S01: persisted Established is exposed without a current session observation")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) {
|
||||||
|
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||||
|
// collectSpeakerLiveStatus constructs this view when no poll result exists.
|
||||||
|
view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"}
|
||||||
|
row := speakerLiveStatusJSON(sp, view, nil)
|
||||||
|
if row["bgp_poll_ok"] == true || row["agent_ok"] == true {
|
||||||
|
t.Fatal("S01: absent poll is exposed as successful BGP/agent health")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -7,9 +7,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/db"
|
"evobgp/internal/db"
|
||||||
"evobgp/internal/httpclient"
|
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
|
"evobgp/internal/pipeline"
|
||||||
"evobgp/internal/repository"
|
"evobgp/internal/repository"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
|
|
||||||
@@ -18,7 +18,7 @@ import (
|
|||||||
|
|
||||||
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
||||||
func NewCDNHTTPClient() *http.Client {
|
func NewCDNHTTPClient() *http.Client {
|
||||||
return httpclient.New(httpclient.DefaultTimeout)
|
return pipeline.NewCDNHTTPClient()
|
||||||
}
|
}
|
||||||
|
|
||||||
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
||||||
@@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
|
|||||||
backend = mem
|
backend = mem
|
||||||
}
|
}
|
||||||
|
|
||||||
cdnHTTP := NewCDNHTTPClient()
|
// Configured DoH profiles and RIPE service overrides retain the previous
|
||||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP}
|
// general HTTP/HTTPS transport, including explicitly configured private
|
||||||
|
// endpoints. CDN calls independently secure this client in UpstreamHTTPDo.
|
||||||
|
workerHTTP := pipeline.NewPipelineHTTPClient()
|
||||||
|
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP}
|
||||||
reg := jobs.NewRegistry(wk.Process)
|
reg := jobs.NewRegistry(wk.Process)
|
||||||
wk.Registry = reg
|
wk.Registry = reg
|
||||||
if pool != nil {
|
if pool != nil {
|
||||||
audit := repository.NewJobAuditWriter(pool)
|
reg.UseDurableQueue(backend)
|
||||||
jobMeta := func(j *jobs.Job) map[string]any {
|
|
||||||
if j == nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
st := j.Snapshot()
|
|
||||||
meta, _ := st["meta"].(map[string]any)
|
|
||||||
return meta
|
|
||||||
}
|
|
||||||
reg.SetPersistHooks(
|
|
||||||
func(j *jobs.Job) {
|
|
||||||
if j == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
|
|
||||||
},
|
|
||||||
func(j *jobs.Job) {
|
|
||||||
if j == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
|
|
||||||
},
|
|
||||||
func(j *jobs.Job) {
|
|
||||||
if j == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
st := j.Snapshot()
|
|
||||||
status, _ := st["status"].(string)
|
|
||||||
var errMsg *string
|
|
||||||
if e, ok := st["error"].(string); ok && e != "" {
|
|
||||||
errMsg = &e
|
|
||||||
}
|
|
||||||
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
|
|
||||||
},
|
|
||||||
)
|
|
||||||
}
|
}
|
||||||
observability.RegisterStoreBackend(backend)
|
observability.RegisterStoreBackend(backend)
|
||||||
return backend, reg, pool, nil
|
return backend, reg, pool, nil
|
||||||
|
|||||||
@@ -0,0 +1,110 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/base64"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/pipeline"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
"github.com/miekg/dns"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||||
|
t.Setenv("EVOBGP_BIRDC_SOCKET", "")
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
|
||||||
|
var calls atomic.Int32
|
||||||
|
resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns"))
|
||||||
|
if err != nil {
|
||||||
|
w.WriteHeader(400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
query := new(dns.Msg)
|
||||||
|
if err = query.Unpack(raw); err != nil || len(query.Question) == 0 {
|
||||||
|
w.WriteHeader(400)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
answer := new(dns.Msg)
|
||||||
|
answer.SetReply(query)
|
||||||
|
if query.Question[0].Qtype == dns.TypeA {
|
||||||
|
answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}}
|
||||||
|
}
|
||||||
|
body, err := answer.Pack()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/dns-message")
|
||||||
|
_, _ = w.Write(body)
|
||||||
|
}))
|
||||||
|
defer resolver.Close()
|
||||||
|
st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
reg.RequestCancelAll()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if e := reg.Drain(ctx); e != nil {
|
||||||
|
t.Error(e)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
deadline := time.After(5 * time.Second)
|
||||||
|
ticker := time.NewTicker(5 * time.Millisecond)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
snap := j.Snapshot()
|
||||||
|
status := snap["status"]
|
||||||
|
if status == jobs.StatusSucceeded {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if status == jobs.StatusFailed || status == jobs.StatusCancelled {
|
||||||
|
t.Fatalf("worker refresh failed: %+v", snap)
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-deadline:
|
||||||
|
t.Fatal("worker did not finish")
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls.Load() < 2 {
|
||||||
|
t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load())
|
||||||
|
}
|
||||||
|
// The same general-client choice must not remove CDN's independent ACL.
|
||||||
|
cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") }))
|
||||||
|
defer cdn.Close()
|
||||||
|
req, err := http.NewRequest(http.MethodGet, cdn.URL, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil {
|
||||||
|
t.Fatal("private CDN accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||||
|
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1)))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
|
||||||
|
st := store.NewMemory()
|
||||||
|
st.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := st.DemoIDs()
|
||||||
|
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
s := &Server{store: st, cdnHTTP: srv.Client()}
|
||||||
|
body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body)))
|
||||||
|
req.SetPathValue("module_id", mod.ID)
|
||||||
|
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||||
|
recorder := httptest.NewRecorder()
|
||||||
|
s.handlePreviewCDNSource(recorder, req)
|
||||||
|
if recorder.Code != http.StatusBadGateway {
|
||||||
|
t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||||
|
t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, module, _, _, _ := m.DemoIDs()
|
||||||
|
var calls atomic.Int32
|
||||||
|
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
w.WriteHeader(http.StatusNotFound)
|
||||||
|
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
// No EOF: an unbounded drain would wait until the caller deadline.
|
||||||
|
<-r.Context().Done()
|
||||||
|
}))
|
||||||
|
defer upstream.Close()
|
||||||
|
s := &Server{store: m, cdnHTTP: upstream.Client()}
|
||||||
|
request := func() *http.Request {
|
||||||
|
r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`))
|
||||||
|
r.SetPathValue("module_id", module)
|
||||||
|
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||||
|
}
|
||||||
|
first := httptest.NewRecorder()
|
||||||
|
firstRequest := request()
|
||||||
|
ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx))
|
||||||
|
if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil {
|
||||||
|
t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err())
|
||||||
|
}
|
||||||
|
second := httptest.NewRecorder()
|
||||||
|
s.handlePreviewCDNSource(second, request())
|
||||||
|
if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" {
|
||||||
|
t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler {
|
|||||||
}
|
}
|
||||||
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
|
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
|
||||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
|
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
|
||||||
|
w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After")
|
||||||
if r.Method == http.MethodOptions {
|
if r.Method == http.MethodOptions {
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool {
|
||||||
|
j, found, err := s.jobs.ReplayContext(r.Context(), in)
|
||||||
|
if err != nil {
|
||||||
|
writeJobEnqueueError(w, err)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
|
snap := j.Snapshot()
|
||||||
|
body := map[string]any{"job_id": j.ID, "status": snap["status"]}
|
||||||
|
if j.Kind == jobs.KindTenantRefresh {
|
||||||
|
ids := snap["meta"].(map[string]any)["module_ids"]
|
||||||
|
n := 0
|
||||||
|
switch v := ids.(type) {
|
||||||
|
case []any:
|
||||||
|
n = len(v)
|
||||||
|
case []string:
|
||||||
|
n = len(v)
|
||||||
|
}
|
||||||
|
body["module_ids"] = ids
|
||||||
|
body["modules_due"] = n
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, body)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJobEnqueueError(w http.ResponseWriter, err error) {
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, store.ErrJobConflict):
|
||||||
|
writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request")
|
||||||
|
case errors.Is(err, store.ErrJobQueueFull):
|
||||||
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full")
|
||||||
|
default:
|
||||||
|
writeInternalError(w, "job_enqueue", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJobReadError(w http.ResponseWriter, err error) {
|
||||||
|
if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) {
|
||||||
|
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeInternalError(w, "job_read", err)
|
||||||
|
}
|
||||||
@@ -0,0 +1,99 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type unavailableJobQueue struct{ store.JobQueue }
|
||||||
|
|
||||||
|
func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) {
|
||||||
|
return store.JobRecord{}, false, errors.New("database unavailable")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestJobEnqueueFailureNeverAccepts(t *testing.T) {
|
||||||
|
r := jobs.NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(unavailableJobQueue{})
|
||||||
|
s := &Server{jobs: r}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil)
|
||||||
|
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"}))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.handleBirdReload(w, req)
|
||||||
|
if w.Code == http.StatusAccepted || w.Code < 500 {
|
||||||
|
t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAutomaticEnqueueFailureIsVisible(t *testing.T) {
|
||||||
|
r := jobs.NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(unavailableJobQueue{})
|
||||||
|
s := &Server{jobs: r}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil)
|
||||||
|
s.enqueuePeerReconcile(w, req, "tenant", "peer_create")
|
||||||
|
if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" {
|
||||||
|
t.Fatalf("scheduling failure hidden: %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type selectorBackend struct {
|
||||||
|
store.Backend
|
||||||
|
modules []*store.Module
|
||||||
|
reads int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules }
|
||||||
|
|
||||||
|
func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) {
|
||||||
|
m := store.NewMemory()
|
||||||
|
b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}}
|
||||||
|
r := jobs.NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(m)
|
||||||
|
s := &Server{store: b, jobs: r}
|
||||||
|
request := func() *httptest.ResponseRecorder {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`))
|
||||||
|
req.Header.Set("Idempotency-Key", "same-request")
|
||||||
|
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"}))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.handleTenantRefresh(w, req)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
first := request()
|
||||||
|
if first.Code != http.StatusAccepted {
|
||||||
|
t.Fatalf("first=%d %s", first.Code, first.Body.String())
|
||||||
|
}
|
||||||
|
var accepted map[string]any
|
||||||
|
if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute)
|
||||||
|
if err != nil || len(claimed) != 1 {
|
||||||
|
t.Fatalf("claim %+v %v", claimed, err)
|
||||||
|
}
|
||||||
|
claimed[0].Status = "succeeded"
|
||||||
|
if err = m.FinishJob(context.Background(), claimed[0]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.modules = nil
|
||||||
|
reads := b.reads
|
||||||
|
replay := request()
|
||||||
|
if replay.Code != http.StatusAccepted {
|
||||||
|
t.Fatalf("replay=%d %s", replay.Code, replay.Body.String())
|
||||||
|
}
|
||||||
|
var got map[string]any
|
||||||
|
if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads {
|
||||||
|
t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestJobReportDurableProgressAndTenantScope(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||||
|
j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||||
|
if err != nil || len(claimed) != 1 {
|
||||||
|
t.Fatalf("claim: %v %v", claimed, err)
|
||||||
|
}
|
||||||
|
f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken}
|
||||||
|
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A fresh registry has no in-process job or progress metadata.
|
||||||
|
r := jobs.NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(m)
|
||||||
|
s := &Server{store: m, jobs: r}
|
||||||
|
request := func(tenantID, jobID string) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil)
|
||||||
|
req.SetPathValue("job_id", jobID)
|
||||||
|
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"}))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.handleGetJobReport(w, req)
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
check := func(id string) {
|
||||||
|
t.Helper()
|
||||||
|
w := request(tenant, id)
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("report: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
var body struct {
|
||||||
|
Progress []struct {
|
||||||
|
State, Confirmation string
|
||||||
|
JobID string `json:"job_id"`
|
||||||
|
} `json:"deployment_progress"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" {
|
||||||
|
t.Fatalf("lost persisted progress: %s", w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
check(j.ID)
|
||||||
|
recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
check(recovery.ID)
|
||||||
|
// A later explicit completion releases the delivery reservation. Once recovery
|
||||||
|
// has its own persisted row, its report must not show the parent's old state.
|
||||||
|
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute)
|
||||||
|
if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID {
|
||||||
|
t.Fatalf("recovery claim: %v %v", recoveryRows, err)
|
||||||
|
}
|
||||||
|
rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken}
|
||||||
|
rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
w := request(tenant, recovery.ID)
|
||||||
|
var report struct {
|
||||||
|
Progress []struct {
|
||||||
|
State string
|
||||||
|
JobID string `json:"job_id"`
|
||||||
|
} `json:"deployment_progress"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" {
|
||||||
|
t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound {
|
||||||
|
t.Fatalf("cross-tenant report: %d", w.Code)
|
||||||
|
}
|
||||||
|
foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound {
|
||||||
|
t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"net/http/httptest"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old
|
||||||
|
// CLI against today's HTTP handlers. It never calls apply or touches host BIRD.
|
||||||
|
func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) {
|
||||||
|
binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN")
|
||||||
|
if binary == "" {
|
||||||
|
t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set")
|
||||||
|
}
|
||||||
|
binary, err := filepath.Abs(binary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(binary); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer srv.Close()
|
||||||
|
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||||
|
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node")
|
||||||
|
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
server := httptest.NewServer(srv.Handler())
|
||||||
|
defer server.Close()
|
||||||
|
run := func(wantSuccess bool, args ...string) {
|
||||||
|
t.Helper()
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
t.Fatalf("legacy CLI timed out: %v", ctx.Err())
|
||||||
|
}
|
||||||
|
if wantSuccess && err != nil {
|
||||||
|
t.Fatalf("legacy CLI failed: %v: %s", err, output)
|
||||||
|
}
|
||||||
|
if !wantSuccess && err == nil {
|
||||||
|
t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
path := filepath.Join(t.TempDir(), "current-bundle.tar.gz")
|
||||||
|
run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path)
|
||||||
|
run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||||
|
raw, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||||
|
if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker {
|
||||||
|
t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err)
|
||||||
|
}
|
||||||
|
// The archive is valid, but verification with an unrelated key must fail.
|
||||||
|
run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize))
|
||||||
|
for _, token := range []string{"invalid-node", "other-node"} {
|
||||||
|
denied := filepath.Join(t.TempDir(), "denied.tar.gz")
|
||||||
|
run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied)
|
||||||
|
if _, err := os.Stat(denied); !os.IsNotExist(err) {
|
||||||
|
t.Fatalf("unauthorized CLI wrote a bundle: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A damaged payload must not pass the old verifier with the correct key.
|
||||||
|
if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||||
|
}
|
||||||
@@ -0,0 +1,310 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/binary"
|
||||||
|
"encoding/hex"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
"evobgp/internal/pipeline"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained.
|
||||||
|
nodeBundleCacheMaxEntries = 32
|
||||||
|
// nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB).
|
||||||
|
nodeBundleCacheMaxBytes = 64 << 20
|
||||||
|
)
|
||||||
|
|
||||||
|
// errNodeBundleRevisionMissing is returned when the revision cannot be loaded.
|
||||||
|
// It is not cached: the next pull tries again.
|
||||||
|
var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found")
|
||||||
|
|
||||||
|
// nodeBundleOpError carries the writeInternalError operation name for a failed build.
|
||||||
|
type nodeBundleOpError struct {
|
||||||
|
op string
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *nodeBundleOpError) Error() string {
|
||||||
|
if e == nil || e.err == nil {
|
||||||
|
return "httpapi: node bundle"
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("httpapi: %s: %v", e.op, e.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *nodeBundleOpError) Unwrap() error { return e.err }
|
||||||
|
|
||||||
|
// nodeBundleBuilder produces the gzip body served to evobgp-node.
|
||||||
|
// The default implementation is bundle.BuildGzippedTar, whose archive contains
|
||||||
|
// manifest.json and bundle.sig. Tests replace it to count rebuilds.
|
||||||
|
type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error)
|
||||||
|
|
||||||
|
// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the
|
||||||
|
// build that filled the entry, signature included. Errors are not stored.
|
||||||
|
// Concurrent loads of one key share a single build.
|
||||||
|
type nodeBundleCache struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
maxN int
|
||||||
|
maxBytes int
|
||||||
|
total int
|
||||||
|
items map[string]*nodeBundleEntry
|
||||||
|
lru []string // index 0 is the oldest idle entry
|
||||||
|
flights map[string]*nodeBundleFlight
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeBundleEntry struct {
|
||||||
|
body []byte
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeBundleFlight struct {
|
||||||
|
done chan struct{}
|
||||||
|
body []byte
|
||||||
|
err error
|
||||||
|
joined int
|
||||||
|
}
|
||||||
|
|
||||||
|
func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache {
|
||||||
|
return &nodeBundleCache{
|
||||||
|
maxN: maxEntries,
|
||||||
|
maxBytes: maxBytes,
|
||||||
|
items: make(map[string]*nodeBundleEntry),
|
||||||
|
flights: make(map[string]*nodeBundleFlight),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeBundleCacheKey identifies one servable bundle.
|
||||||
|
// overlayFP is a digest of the speaker overlay output (router id / source addresses,
|
||||||
|
// node IPv4 fallback, peer and discovery render). Revision content itself is immutable,
|
||||||
|
// so the revision id stands in for the shared prefix files.
|
||||||
|
func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3)
|
||||||
|
b.WriteString(tenantID)
|
||||||
|
b.WriteByte(0)
|
||||||
|
b.WriteString(revisionID)
|
||||||
|
b.WriteByte(0)
|
||||||
|
b.WriteString(speakerID)
|
||||||
|
b.WriteByte(0)
|
||||||
|
b.WriteString(overlayFP)
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle
|
||||||
|
// would apply on top of an immutable revision. Empty base fragments are enough:
|
||||||
|
// bird.conf and the peers fragment are fully re-rendered from tenant settings,
|
||||||
|
// speaker source addresses, and the live peer set. Revision file bytes are not
|
||||||
|
// part of this digest.
|
||||||
|
func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) {
|
||||||
|
frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||||
|
}
|
||||||
|
keys := make([]string, 0, len(frags))
|
||||||
|
for k := range frags {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
h := sha256.New()
|
||||||
|
for _, k := range keys {
|
||||||
|
if err := hashWriteString(h, k); err != nil {
|
||||||
|
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||||
|
}
|
||||||
|
if err := hashWriteString(h, frags[k]); err != nil {
|
||||||
|
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return hex.EncodeToString(h.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error {
|
||||||
|
var buf [8]byte
|
||||||
|
binary.BigEndian.PutUint64(buf[:], uint64(len(s)))
|
||||||
|
if _, err := h.Write(buf[:]); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
_, err := h.Write([]byte(s))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// do returns the cached body or runs build once for key.
|
||||||
|
// build's bool is false when the result must not be stored (failure is also not stored).
|
||||||
|
// Callers must not mutate the returned slice.
|
||||||
|
func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) {
|
||||||
|
if c == nil {
|
||||||
|
return nil, errors.New("httpapi: nil node bundle cache")
|
||||||
|
}
|
||||||
|
if ctx == nil {
|
||||||
|
ctx = context.Background()
|
||||||
|
}
|
||||||
|
c.mu.Lock()
|
||||||
|
if e, ok := c.items[key]; ok {
|
||||||
|
c.touchLocked(key)
|
||||||
|
body = e.body
|
||||||
|
c.mu.Unlock()
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
if f, ok := c.flights[key]; ok {
|
||||||
|
f.joined++
|
||||||
|
c.mu.Unlock()
|
||||||
|
return waitNodeBundleFlight(ctx, f)
|
||||||
|
}
|
||||||
|
if err = ctx.Err(); err != nil {
|
||||||
|
c.mu.Unlock()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f := &nodeBundleFlight{done: make(chan struct{})}
|
||||||
|
c.flights[key] = f
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
var cacheOK bool
|
||||||
|
var finished bool
|
||||||
|
finish := func() {
|
||||||
|
if finished {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
finished = true
|
||||||
|
out := body
|
||||||
|
c.mu.Lock()
|
||||||
|
delete(c.flights, key)
|
||||||
|
if err == nil && cacheOK {
|
||||||
|
if stored, ok := c.storeLocked(key, out); ok {
|
||||||
|
out = stored
|
||||||
|
}
|
||||||
|
}
|
||||||
|
f.body = out
|
||||||
|
f.err = err
|
||||||
|
close(f.done)
|
||||||
|
c.mu.Unlock()
|
||||||
|
body = out
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
if rec := recover(); rec != nil {
|
||||||
|
body = nil
|
||||||
|
cacheOK = false
|
||||||
|
err = fmt.Errorf("httpapi: node bundle build: %v", rec)
|
||||||
|
}
|
||||||
|
finish()
|
||||||
|
}()
|
||||||
|
|
||||||
|
// Detach from the leader's cancel so one disconnected agent does not fail the shared build.
|
||||||
|
body, cacheOK, err = build(context.WithoutCancel(ctx))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) {
|
||||||
|
select {
|
||||||
|
case <-f.done:
|
||||||
|
return f.body, f.err
|
||||||
|
case <-ctx.Done():
|
||||||
|
select {
|
||||||
|
case <-f.done:
|
||||||
|
return f.body, f.err
|
||||||
|
default:
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// storeLocked retains body when it fits under both caps, evicting the oldest idle
|
||||||
|
// entries first. body is copied. The stored slice is returned.
|
||||||
|
func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) {
|
||||||
|
n := len(body)
|
||||||
|
if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
if e, ok := c.items[key]; ok {
|
||||||
|
c.total -= len(e.body)
|
||||||
|
delete(c.items, key)
|
||||||
|
c.removeLRULocked(key)
|
||||||
|
}
|
||||||
|
for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) {
|
||||||
|
if len(c.lru) == 0 {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
c.deleteOldestLocked()
|
||||||
|
}
|
||||||
|
if len(c.items) >= c.maxN || c.total+n > c.maxBytes {
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
stored := make([]byte, n)
|
||||||
|
copy(stored, body)
|
||||||
|
c.items[key] = &nodeBundleEntry{body: stored}
|
||||||
|
c.total += n
|
||||||
|
c.lru = append(c.lru, key)
|
||||||
|
return stored, true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodeBundleCache) touchLocked(key string) {
|
||||||
|
c.removeLRULocked(key)
|
||||||
|
c.lru = append(c.lru, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodeBundleCache) deleteOldestLocked() {
|
||||||
|
if len(c.lru) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key := c.lru[0]
|
||||||
|
c.lru = c.lru[1:]
|
||||||
|
if e, ok := c.items[key]; ok {
|
||||||
|
c.total -= len(e.body)
|
||||||
|
delete(c.items, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *nodeBundleCache) removeLRULocked(key string) {
|
||||||
|
for i, k := range c.lru {
|
||||||
|
if k != key {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
copy(c.lru[i:], c.lru[i+1:])
|
||||||
|
c.lru = c.lru[:len(c.lru)-1]
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) {
|
||||||
|
if s != nil && s.nodeBundleBuild != nil {
|
||||||
|
return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv)
|
||||||
|
}
|
||||||
|
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv)
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip.
|
||||||
|
// The bool is false when overlay inputs moved during the load, so the caller must not
|
||||||
|
// cache the body under the key captured before the load. Failed loads are not cacheable.
|
||||||
|
func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||||
|
}
|
||||||
|
rev, err := s.store.GetRevision(tenantID, revisionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, errNodeBundleRevisionMissing
|
||||||
|
}
|
||||||
|
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||||
|
}
|
||||||
|
fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||||
|
}
|
||||||
|
body, err := s.buildNodeBundle(revisionID, speakerID, overlaid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, &nodeBundleOpError{op: "internal", err: err}
|
||||||
|
}
|
||||||
|
cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0
|
||||||
|
return body, cacheable, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,791 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/tar"
|
||||||
|
"bytes"
|
||||||
|
"compress/gzip"
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
"evobgp/internal/signing"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) {
|
||||||
|
a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay")
|
||||||
|
if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") {
|
||||||
|
t.Fatal("tenants collided")
|
||||||
|
}
|
||||||
|
if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") {
|
||||||
|
t.Fatal("revisions collided")
|
||||||
|
}
|
||||||
|
if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") {
|
||||||
|
t.Fatal("speakers collided")
|
||||||
|
}
|
||||||
|
if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") {
|
||||||
|
t.Fatal("overlay inputs collided")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(8, 1<<20)
|
||||||
|
var builds atomic.Int32
|
||||||
|
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
return []byte("signed-body"), true, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
return []byte("other"), true, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, []byte("signed-body")) {
|
||||||
|
t.Fatalf("hit body = %q", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("singleflight", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(8, 1<<20)
|
||||||
|
var builds atomic.Int32
|
||||||
|
release := make(chan struct{})
|
||||||
|
const n = 8
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
bodies := make([][]byte, n)
|
||||||
|
errs := make([]error, n)
|
||||||
|
wg.Add(n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
<-release
|
||||||
|
return []byte("signed-bundle"), true, nil
|
||||||
|
})
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
c.mu.Lock()
|
||||||
|
joined := 0
|
||||||
|
if f := c.flights["same"]; f != nil {
|
||||||
|
joined = f.joined
|
||||||
|
}
|
||||||
|
c.mu.Unlock()
|
||||||
|
if builds.Load() > 1 {
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
t.Fatalf("builds = %d before followers joined", builds.Load())
|
||||||
|
}
|
||||||
|
if builds.Load() == 1 && joined == n-1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
t.Fatalf("builds=%d joined=%d", builds.Load(), joined)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||||
|
}
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
if errs[i] != nil {
|
||||||
|
t.Fatalf("waiter %d: %v", i, errs[i])
|
||||||
|
}
|
||||||
|
if !bytes.Equal(bodies[i], []byte("signed-bundle")) {
|
||||||
|
t.Fatalf("waiter %d body = %q", i, bodies[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("canceled waiter does not drop the build", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(4, 1024)
|
||||||
|
release := make(chan struct{})
|
||||||
|
var builds atomic.Int32
|
||||||
|
leaderErr := make(chan error, 1)
|
||||||
|
go func() {
|
||||||
|
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
<-release
|
||||||
|
return []byte("body"), true, nil
|
||||||
|
})
|
||||||
|
leaderErr <- err
|
||||||
|
}()
|
||||||
|
deadline := time.Now().Add(2 * time.Second)
|
||||||
|
for {
|
||||||
|
c.mu.Lock()
|
||||||
|
_, ok := c.flights["k"]
|
||||||
|
c.mu.Unlock()
|
||||||
|
if ok || time.Now().After(deadline) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
_, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
return []byte("nope"), true, nil
|
||||||
|
})
|
||||||
|
if !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("follower err = %v", err)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
if err := <-leaderErr; err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatalf("builds = %d", builds.Load())
|
||||||
|
}
|
||||||
|
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
return []byte("other"), true, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatal("successful build was not cached after a canceled waiter")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("errors are not cached", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(4, 1024)
|
||||||
|
calls := 0
|
||||||
|
boom := errors.New("httpapi: bundle build failed")
|
||||||
|
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return nil, true, boom
|
||||||
|
})
|
||||||
|
if !errors.Is(err, boom) {
|
||||||
|
t.Fatalf("err = %v", err)
|
||||||
|
}
|
||||||
|
_, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return nil, false, errNodeBundleRevisionMissing
|
||||||
|
})
|
||||||
|
if !errors.Is(err, errNodeBundleRevisionMissing) {
|
||||||
|
t.Fatalf("err = %v", err)
|
||||||
|
}
|
||||||
|
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return []byte("ok"), true, nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if string(got) != "ok" || calls != 3 {
|
||||||
|
t.Fatalf("calls=%d body=%q", calls, got)
|
||||||
|
}
|
||||||
|
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return []byte("other"), true, nil
|
||||||
|
})
|
||||||
|
if err != nil || calls != 3 {
|
||||||
|
t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("unstorable success is not cached", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(4, 1024)
|
||||||
|
calls := 0
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return []byte("fresh"), false, nil
|
||||||
|
})
|
||||||
|
if err != nil || string(got) != "fresh" {
|
||||||
|
t.Fatalf("i=%d err=%v body=%q", i, err, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Fatalf("calls = %d, want 2", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) {
|
||||||
|
t.Run("entry cap evicts oldest idle", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(2, 100)
|
||||||
|
put := func(key, body string) {
|
||||||
|
t.Helper()
|
||||||
|
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||||
|
return []byte(body), true, nil
|
||||||
|
})
|
||||||
|
if err != nil || string(got) != body {
|
||||||
|
t.Fatalf("put %s: %v %q", key, err, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put("a", "A")
|
||||||
|
put("b", "B")
|
||||||
|
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||||
|
t.Fatal("recent entry was rebuilt")
|
||||||
|
return nil, false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
put("c", "C")
|
||||||
|
assertBundleCacheAccounting(t, c)
|
||||||
|
if _, ok := c.items["b"]; ok {
|
||||||
|
t.Fatal("idle entry b was kept")
|
||||||
|
}
|
||||||
|
if _, ok := c.items["a"]; !ok || c.items["c"] == nil {
|
||||||
|
t.Fatalf("kept %v", cacheKeys(c))
|
||||||
|
}
|
||||||
|
if len(c.items) > 2 {
|
||||||
|
t.Fatalf("entry cap exceeded: %d", len(c.items))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("byte cap", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(8, 10)
|
||||||
|
put := func(key string, n int) {
|
||||||
|
t.Helper()
|
||||||
|
body := bytes.Repeat([]byte("x"), n)
|
||||||
|
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||||
|
return body, true, nil
|
||||||
|
})
|
||||||
|
if err != nil || !bytes.Equal(got, body) {
|
||||||
|
t.Fatalf("put %s: %v len=%d", key, err, len(got))
|
||||||
|
}
|
||||||
|
assertBundleCacheAccounting(t, c)
|
||||||
|
}
|
||||||
|
put("a", 4)
|
||||||
|
put("b", 4)
|
||||||
|
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||||
|
t.Fatal("rebuilt a")
|
||||||
|
return nil, false, nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
put("c", 4)
|
||||||
|
assertBundleCacheAccounting(t, c)
|
||||||
|
if _, ok := c.items["b"]; ok {
|
||||||
|
t.Fatal("byte cap kept idle entry b")
|
||||||
|
}
|
||||||
|
if len(c.items) != 2 || c.total != 8 {
|
||||||
|
t.Fatalf("n=%d total=%d", len(c.items), c.total)
|
||||||
|
}
|
||||||
|
put("d", 6)
|
||||||
|
put("e", 6)
|
||||||
|
assertBundleCacheAccounting(t, c)
|
||||||
|
if len(c.items) != 1 || c.total != 6 {
|
||||||
|
t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("entry larger than the byte cap is not stored", func(t *testing.T) {
|
||||||
|
c := newNodeBundleCache(4, 8)
|
||||||
|
calls := 0
|
||||||
|
big := bytes.Repeat([]byte("y"), 9)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) {
|
||||||
|
calls++
|
||||||
|
return big, true, nil
|
||||||
|
})
|
||||||
|
if err != nil || !bytes.Equal(got, big) {
|
||||||
|
t.Fatalf("i=%d err=%v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Fatalf("oversize body was cached, calls=%d", calls)
|
||||||
|
}
|
||||||
|
if len(c.items) != 0 || c.total != 0 {
|
||||||
|
t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNodeBundleOverlayFingerprintInputs(t *testing.T) {
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, rev, speaker := m.DemoIDs()
|
||||||
|
|
||||||
|
base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if base != again {
|
||||||
|
t.Fatal("fingerprint changed without an overlay input change")
|
||||||
|
}
|
||||||
|
|
||||||
|
sp, err := m.GetSpeaker(tenant, speaker)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{
|
||||||
|
LastDispatchAt: "2026-01-01T00:00:00Z",
|
||||||
|
LastDispatchStatus: "ok",
|
||||||
|
SyncStatus: "synced",
|
||||||
|
})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||||
|
t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
peers := m.ListPeers(tenant)
|
||||||
|
if len(peers) == 0 {
|
||||||
|
t.Fatal("expected demo peers")
|
||||||
|
}
|
||||||
|
state := "Active"
|
||||||
|
if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||||
|
t.Fatalf("session state changed fingerprint: %v %s", err, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if routed == base {
|
||||||
|
t.Fatal("router id setting did not change the overlay fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if nodeFP == routed {
|
||||||
|
t.Fatal("node ipv4 did not change the overlay fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
NodeIPv4: "198.51.100.9",
|
||||||
|
BirdBgpSourceIPv4: "203.0.113.40",
|
||||||
|
})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if sourceFP == nodeFP {
|
||||||
|
t.Fatal("source ipv4 did not change the overlay fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
NodeIPv4: "198.51.100.8",
|
||||||
|
BirdBgpSourceIPv4: "203.0.113.40",
|
||||||
|
})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP {
|
||||||
|
t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP)
|
||||||
|
}
|
||||||
|
|
||||||
|
v6 := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
NodeIPv4: "198.51.100.8",
|
||||||
|
BirdBgpSourceIPv4: "203.0.113.40",
|
||||||
|
BirdBgpSourceIPv6: "2001:db8::40",
|
||||||
|
})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Source v6 changes local v6. When no peer policy overrides it, the rendered
|
||||||
|
// peers fragment can stay the same. A difference means the overlay emitted it;
|
||||||
|
// equality means a rebuild would not change the bundle.
|
||||||
|
_ = v6FP
|
||||||
|
|
||||||
|
if _, err := m.CreatePeer(tenant, &store.BGPPeer{
|
||||||
|
Name: "cache-extra",
|
||||||
|
Neighbor: "203.0.113.50",
|
||||||
|
RemoteASN: 64512,
|
||||||
|
Enabled: true,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if withPeer == v6FP {
|
||||||
|
t.Fatal("added peer did not change the overlay fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
sp2, err := m.CreateSpeaker(tenant, &store.Speaker{
|
||||||
|
Role: "replica",
|
||||||
|
Endpoint: "https://203.0.113.2",
|
||||||
|
MetaJSON: v6,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if other == withPeer {
|
||||||
|
t.Fatal("speakers with the same addresses shared an overlay fingerprint")
|
||||||
|
}
|
||||||
|
|
||||||
|
otherRev, err := m.CreateRollbackRevision(tenant, rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if revFP == withPeer {
|
||||||
|
t.Fatal("revision id did not change the overlay fingerprint")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleNodeBundleCache(t *testing.T) {
|
||||||
|
srv, ts, ids := startNodeBundleServer(t)
|
||||||
|
if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes {
|
||||||
|
t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes)
|
||||||
|
}
|
||||||
|
var builds atomic.Int32
|
||||||
|
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||||
|
}
|
||||||
|
client := ts.Client()
|
||||||
|
|
||||||
|
status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099")
|
||||||
|
if status != http.StatusNotFound {
|
||||||
|
t.Fatalf("missing revision status %d: %s", status, raw)
|
||||||
|
}
|
||||||
|
if builds.Load() != 0 {
|
||||||
|
t.Fatal("missing revision built a bundle")
|
||||||
|
}
|
||||||
|
|
||||||
|
body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||||
|
}
|
||||||
|
if !bytes.Equal(body1, body2) {
|
||||||
|
t.Fatal("cache hit was not byte-for-byte identical")
|
||||||
|
}
|
||||||
|
verifyNodeBundle(t, srv, body2)
|
||||||
|
|
||||||
|
sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{
|
||||||
|
Role: "replica",
|
||||||
|
Endpoint: "https://203.0.113.2",
|
||||||
|
MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev)
|
||||||
|
if builds.Load() != 2 {
|
||||||
|
t.Fatalf("second speaker builds=%d", builds.Load())
|
||||||
|
}
|
||||||
|
if bytes.Equal(body1, bodyB) {
|
||||||
|
t.Fatal("different speakers returned the same bundle")
|
||||||
|
}
|
||||||
|
if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") {
|
||||||
|
t.Fatal("speaker B bundle missing its router id")
|
||||||
|
}
|
||||||
|
body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 2 || !bytes.Equal(body1, body3) {
|
||||||
|
t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3))
|
||||||
|
}
|
||||||
|
|
||||||
|
meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||||
|
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 3 {
|
||||||
|
t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load())
|
||||||
|
}
|
||||||
|
if bytes.Equal(body1, body4) {
|
||||||
|
t.Fatal("changed overlay input reused the previous bundle")
|
||||||
|
}
|
||||||
|
conf := bundleFile(t, body4, "bird.conf")
|
||||||
|
if !strings.Contains(conf, "198.51.100.9") {
|
||||||
|
t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf)
|
||||||
|
}
|
||||||
|
verifyNodeBundle(t, srv, body4)
|
||||||
|
body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 3 || !bytes.Equal(body4, body5) {
|
||||||
|
t.Fatal("rebuilt bundle was not cached")
|
||||||
|
}
|
||||||
|
|
||||||
|
source := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
NodeIPv4: "198.51.100.9",
|
||||||
|
BirdBgpSourceIPv4: "203.0.113.40",
|
||||||
|
})
|
||||||
|
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 4 {
|
||||||
|
t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load())
|
||||||
|
}
|
||||||
|
conf = bundleFile(t, body6, "bird.conf")
|
||||||
|
if !strings.Contains(conf, "203.0.113.40") {
|
||||||
|
t.Fatalf("source ipv4 not applied: %s", conf)
|
||||||
|
}
|
||||||
|
verifyNodeBundle(t, srv, body6)
|
||||||
|
|
||||||
|
otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev)
|
||||||
|
if builds.Load() != 5 {
|
||||||
|
t.Fatalf("other revision builds=%d", builds.Load())
|
||||||
|
}
|
||||||
|
if bytes.Equal(body6, bodyRev) {
|
||||||
|
t.Fatal("revisions shared a bundle body")
|
||||||
|
}
|
||||||
|
cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
if builds.Load() != 5 || !bytes.Equal(body6, cached) {
|
||||||
|
t.Fatal("original revision was evicted or rebuilt")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHandleNodeBundleSingleflight(t *testing.T) {
|
||||||
|
srv, ts, ids := startNodeBundleServer(t)
|
||||||
|
var builds atomic.Int32
|
||||||
|
release := make(chan struct{})
|
||||||
|
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||||
|
builds.Add(1)
|
||||||
|
<-release
|
||||||
|
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||||
|
}
|
||||||
|
client := ts.Client()
|
||||||
|
const n = 2
|
||||||
|
bodies := make([][]byte, n)
|
||||||
|
errs := make([]error, n)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
wg.Add(n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
go func(i int) {
|
||||||
|
defer wg.Done()
|
||||||
|
bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev)
|
||||||
|
}(i)
|
||||||
|
}
|
||||||
|
deadline := time.Now().Add(3 * time.Second)
|
||||||
|
joined := 0
|
||||||
|
for {
|
||||||
|
if builds.Load() > 1 {
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
t.Fatalf("concurrent pulls built %d times", builds.Load())
|
||||||
|
}
|
||||||
|
srv.bundleCache.mu.Lock()
|
||||||
|
joined = 0
|
||||||
|
for _, f := range srv.bundleCache.flights {
|
||||||
|
joined += f.joined
|
||||||
|
}
|
||||||
|
srv.bundleCache.mu.Unlock()
|
||||||
|
if builds.Load() == 1 && joined >= 1 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined)
|
||||||
|
}
|
||||||
|
time.Sleep(time.Millisecond)
|
||||||
|
}
|
||||||
|
close(release)
|
||||||
|
wg.Wait()
|
||||||
|
if builds.Load() != 1 {
|
||||||
|
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||||
|
}
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
if errs[i] != nil {
|
||||||
|
t.Fatal(errs[i])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !bytes.Equal(bodies[0], bodies[1]) {
|
||||||
|
t.Fatal("singleflight returned different bodies")
|
||||||
|
}
|
||||||
|
verifyNodeBundle(t, srv, bodies[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
type nodeBundleIDs struct {
|
||||||
|
tenant, rev, speaker string
|
||||||
|
}
|
||||||
|
|
||||||
|
func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) {
|
||||||
|
t.Helper()
|
||||||
|
srv, err := New(Options{
|
||||||
|
InsecureDev: true,
|
||||||
|
SeedDemo: true,
|
||||||
|
BundleSeedHex: testBundleSeed,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
tenant, _, _, rev, speaker := srv.Store().DemoIDs()
|
||||||
|
mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node")
|
||||||
|
ts := httptest.NewServer(srv.Handler())
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker}
|
||||||
|
}
|
||||||
|
|
||||||
|
func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
status, body := pullNodeBundle(t, client, base, speaker, rev)
|
||||||
|
if status != http.StatusOK {
|
||||||
|
t.Fatalf("status %d: %s", status, body)
|
||||||
|
}
|
||||||
|
return body
|
||||||
|
}
|
||||||
|
|
||||||
|
func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer nodekey")
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return resp.StatusCode, body
|
||||||
|
}
|
||||||
|
|
||||||
|
func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) {
|
||||||
|
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", "Bearer nodekey")
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, errors.New(resp.Status + ": " + string(body))
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) {
|
||||||
|
t.Helper()
|
||||||
|
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bundleFile(t *testing.T, gz []byte, name string) string {
|
||||||
|
t.Helper()
|
||||||
|
gr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = gr.Close() }()
|
||||||
|
tr := tar.NewReader(gr)
|
||||||
|
for {
|
||||||
|
hdr, err := tr.Next()
|
||||||
|
if errors.Is(err, io.EOF) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if hdr.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var b strings.Builder
|
||||||
|
if _, err := io.Copy(&b, tr); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
t.Fatalf("archive missing %s", name)
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) {
|
||||||
|
t.Helper()
|
||||||
|
if len(c.items) != len(c.lru) {
|
||||||
|
t.Fatalf("items %d lru %d", len(c.items), len(c.lru))
|
||||||
|
}
|
||||||
|
sum := 0
|
||||||
|
seen := make(map[string]bool, len(c.lru))
|
||||||
|
for _, k := range c.lru {
|
||||||
|
e, ok := c.items[k]
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("lru key %q missing from items", k)
|
||||||
|
}
|
||||||
|
if seen[k] {
|
||||||
|
t.Fatalf("duplicate lru key %q", k)
|
||||||
|
}
|
||||||
|
seen[k] = true
|
||||||
|
sum += len(e.body)
|
||||||
|
}
|
||||||
|
if sum != c.total {
|
||||||
|
t.Fatalf("total=%d sum=%d", c.total, sum)
|
||||||
|
}
|
||||||
|
if c.maxN > 0 && len(c.items) > c.maxN {
|
||||||
|
t.Fatalf("entries %d over cap %d", len(c.items), c.maxN)
|
||||||
|
}
|
||||||
|
if c.maxBytes > 0 && c.total > c.maxBytes {
|
||||||
|
t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func cacheKeys(c *nodeBundleCache) []string {
|
||||||
|
keys := make([]string, 0, len(c.items))
|
||||||
|
for k := range c.items {
|
||||||
|
keys = append(keys, k)
|
||||||
|
}
|
||||||
|
return keys
|
||||||
|
}
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/ed25519"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) {
|
||||||
|
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer srv.Close()
|
||||||
|
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||||
|
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node")
|
||||||
|
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
handler := srv.Handler()
|
||||||
|
call := func(path, token string) (int, []byte, string) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := http.NewRequest(http.MethodGet, path, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if token != "" {
|
||||||
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
handler.ServeHTTP(w, req)
|
||||||
|
resp := w.Result()
|
||||||
|
defer func() { _ = resp.Body.Close() }()
|
||||||
|
body, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return resp.StatusCode, body, resp.Header.Get("Content-Type")
|
||||||
|
}
|
||||||
|
latestPath := "/v1/speakers/" + speaker + "/revisions/latest"
|
||||||
|
bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision
|
||||||
|
for _, path := range []string{latestPath, bundlePath} {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
token string
|
||||||
|
want int
|
||||||
|
}{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} {
|
||||||
|
status, _, _ := call(path, tc.token)
|
||||||
|
if status != tc.want {
|
||||||
|
t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
status, raw, _ := call(latestPath, "legacy-node")
|
||||||
|
if status != http.StatusOK {
|
||||||
|
t.Fatalf("legacy latest: status %d: %s", status, raw)
|
||||||
|
}
|
||||||
|
var latest struct {
|
||||||
|
RevisionID string `json:"revision_id"`
|
||||||
|
PublishedAt string `json:"published_at"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &latest); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if latest.RevisionID != revision || latest.PublishedAt == "" {
|
||||||
|
t.Fatalf("legacy latest response: %+v", latest)
|
||||||
|
}
|
||||||
|
status, raw, contentType := call("/v1/bundle/signing-public-key", "operator")
|
||||||
|
if status != http.StatusOK || contentType != "application/json" {
|
||||||
|
t.Fatalf("public key status %d, type %q: %s", status, contentType, raw)
|
||||||
|
}
|
||||||
|
var key struct {
|
||||||
|
PublicKeyBase64 string `json:"public_key_base64"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &key); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64)
|
||||||
|
if err != nil || len(pub) != ed25519.PublicKeySize {
|
||||||
|
t.Fatalf("legacy public key: len=%d err=%v", len(pub), err)
|
||||||
|
}
|
||||||
|
status, raw, contentType = call(bundlePath, "legacy-node")
|
||||||
|
if status != http.StatusOK || contentType != "application/gzip" {
|
||||||
|
t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw)
|
||||||
|
}
|
||||||
|
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("legacy signed bundle verification: %v", err)
|
||||||
|
}
|
||||||
|
if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 {
|
||||||
|
t.Fatalf("legacy bundle manifest: %+v", verified.Manifest)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,105 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type expensiveOperation int
|
||||||
|
|
||||||
|
const (
|
||||||
|
lookupOperation expensiveOperation = iota
|
||||||
|
cdnPreviewOperation
|
||||||
|
revisionPreviewOperation
|
||||||
|
)
|
||||||
|
|
||||||
|
type tenantWork struct {
|
||||||
|
start time.Time
|
||||||
|
counts [3]int
|
||||||
|
active int
|
||||||
|
}
|
||||||
|
|
||||||
|
// operationLimiter is deliberately local to one API process. It never queues
|
||||||
|
// waiters or evicts active tenants to admit new identities.
|
||||||
|
type operationLimiter struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
tenants map[string]*tenantWork
|
||||||
|
active, maxActive, maxTenantActive, maxTenants int
|
||||||
|
rates [3]int
|
||||||
|
nextSweep time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func newOperationLimiter() *operationLimiter {
|
||||||
|
return &operationLimiter{
|
||||||
|
tenants: make(map[string]*tenantWork), maxTenants: 4096,
|
||||||
|
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
|
||||||
|
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
|
||||||
|
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func operationEnvLimit(name string, fallback, maximum int) int {
|
||||||
|
n, err := strconv.Atoi(os.Getenv(name))
|
||||||
|
if err != nil || n < 1 || n > maximum {
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
if !now.Before(l.nextSweep) {
|
||||||
|
for key, entry := range l.tenants {
|
||||||
|
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
|
||||||
|
delete(l.tenants, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
l.nextSweep = now.Add(time.Second)
|
||||||
|
}
|
||||||
|
if l.active >= l.maxActive {
|
||||||
|
return nil, 1
|
||||||
|
}
|
||||||
|
entry := l.tenants[tenant]
|
||||||
|
if entry == nil {
|
||||||
|
if len(l.tenants) >= l.maxTenants {
|
||||||
|
return nil, 1
|
||||||
|
}
|
||||||
|
entry = &tenantWork{start: now}
|
||||||
|
l.tenants[tenant] = entry
|
||||||
|
}
|
||||||
|
if !now.Before(entry.start.Add(time.Minute)) {
|
||||||
|
entry.start = now
|
||||||
|
entry.counts = [3]int{}
|
||||||
|
}
|
||||||
|
if entry.active >= l.maxTenantActive {
|
||||||
|
return nil, 1
|
||||||
|
}
|
||||||
|
if entry.counts[op] >= l.rates[op] {
|
||||||
|
wait := entry.start.Add(time.Minute).Sub(now)
|
||||||
|
return nil, max(1, int((wait+time.Second-1)/time.Second))
|
||||||
|
}
|
||||||
|
entry.counts[op]++
|
||||||
|
entry.active++
|
||||||
|
l.active++
|
||||||
|
var once sync.Once
|
||||||
|
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
|
||||||
|
if r.Context().Err() != nil {
|
||||||
|
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
|
||||||
|
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
|
||||||
|
if release == nil {
|
||||||
|
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
||||||
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
|
||||||
|
return nil, false
|
||||||
|
}
|
||||||
|
return release, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) {
|
||||||
|
l := newOperationLimiter()
|
||||||
|
l.maxActive = 2
|
||||||
|
l.maxTenantActive = 1
|
||||||
|
l.rates = [3]int{1, 2, 2}
|
||||||
|
now := time.Now()
|
||||||
|
a, _ := l.acquire("a", lookupOperation, now)
|
||||||
|
if a == nil {
|
||||||
|
t.Fatal("first denied")
|
||||||
|
}
|
||||||
|
if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil {
|
||||||
|
t.Fatal("tenant concurrency bypass")
|
||||||
|
}
|
||||||
|
b, _ := l.acquire("b", revisionPreviewOperation, now)
|
||||||
|
if b == nil {
|
||||||
|
t.Fatal("other tenant denied")
|
||||||
|
}
|
||||||
|
if r, _ := l.acquire("c", lookupOperation, now); r != nil {
|
||||||
|
t.Fatal("global concurrency bypass")
|
||||||
|
}
|
||||||
|
a()
|
||||||
|
a()
|
||||||
|
b()
|
||||||
|
if l.active != 0 {
|
||||||
|
t.Fatal("release not idempotent")
|
||||||
|
}
|
||||||
|
if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 {
|
||||||
|
t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry)
|
||||||
|
}
|
||||||
|
c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second))
|
||||||
|
if c == nil {
|
||||||
|
t.Fatal("independent operation denied")
|
||||||
|
}
|
||||||
|
c()
|
||||||
|
d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute))
|
||||||
|
if d == nil {
|
||||||
|
t.Fatal("expired window not reset")
|
||||||
|
}
|
||||||
|
d()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) {
|
||||||
|
l := newOperationLimiter()
|
||||||
|
l.maxTenants = 1
|
||||||
|
now := time.Now()
|
||||||
|
a, _ := l.acquire("a", lookupOperation, now)
|
||||||
|
if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil {
|
||||||
|
t.Fatal("active entry evicted")
|
||||||
|
}
|
||||||
|
a()
|
||||||
|
b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute))
|
||||||
|
if b == nil || len(l.tenants) != 1 {
|
||||||
|
t.Fatal("idle expiry did not free bounded state")
|
||||||
|
}
|
||||||
|
b()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestOperationConcurrentAdmissionBound(t *testing.T) {
|
||||||
|
l := newOperationLimiter()
|
||||||
|
l.maxActive = 4
|
||||||
|
l.maxTenantActive = 4
|
||||||
|
l.rates = [3]int{100, 100, 100}
|
||||||
|
results := make(chan func(), 32)
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
for range 32 {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(results)
|
||||||
|
accepted := 0
|
||||||
|
for r := range results {
|
||||||
|
if r != nil {
|
||||||
|
accepted++
|
||||||
|
r()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if accepted != 4 || l.active != 0 {
|
||||||
|
t.Fatalf("accepted=%d active=%d", accepted, l.active)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type waitingRevisionStore struct {
|
||||||
|
store.Backend
|
||||||
|
entered chan struct{}
|
||||||
|
unblock chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) {
|
||||||
|
s.entered <- struct{}{}
|
||||||
|
<-s.unblock
|
||||||
|
return s.Backend.GetRevision(tenant, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1")
|
||||||
|
t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, revision, _ := m.DemoIDs()
|
||||||
|
entered := make(chan struct{}, 2)
|
||||||
|
unblock := make(chan struct{})
|
||||||
|
s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}}
|
||||||
|
request := func() *http.Request {
|
||||||
|
r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil)
|
||||||
|
r.SetPathValue("revision_id", revision)
|
||||||
|
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(request().Context())
|
||||||
|
defer cancel()
|
||||||
|
first := httptest.NewRecorder()
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }()
|
||||||
|
<-entered
|
||||||
|
cancel() // Backend deliberately ignores cancellation: capacity stays reserved.
|
||||||
|
second := httptest.NewRecorder()
|
||||||
|
s.handleRevisionPreview(second, request())
|
||||||
|
if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" {
|
||||||
|
close(unblock)
|
||||||
|
<-done
|
||||||
|
t.Fatalf("quota: %d %s", second.Code, second.Body.String())
|
||||||
|
}
|
||||||
|
close(unblock)
|
||||||
|
<-done
|
||||||
|
third := httptest.NewRecorder()
|
||||||
|
s.handleRevisionPreview(third, request())
|
||||||
|
if third.Code != http.StatusOK {
|
||||||
|
t.Fatalf("release failed: %d %s", third.Code, third.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -6,7 +6,6 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions.
|
|
||||||
func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) {
|
|
||||||
if s.store == nil || tenantID == "" {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
knownNeighbors := map[string]struct{}{}
|
|
||||||
for _, p := range s.store.ListPeers(tenantID) {
|
|
||||||
if p == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok {
|
|
||||||
knownNeighbors[n] = struct{}{}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
now := time.Now().UTC()
|
|
||||||
for _, v := range views {
|
|
||||||
for _, sess := range v.Sessions {
|
|
||||||
if !birdfmt.IsDynamicDiscoverySession(sess.Name) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, known := knownNeighbors[neighbor]; known {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
_, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{
|
|
||||||
SpeakerID: v.SpeakerID,
|
|
||||||
NeighborID: strings.TrimSpace(sess.NeighborID),
|
|
||||||
Neighbor: neighbor,
|
|
||||||
RemoteASN: sess.NeighborAS,
|
|
||||||
ProtocolName: sess.Name,
|
|
||||||
SessionState: sess.State,
|
|
||||||
SeenAt: now,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
|
||||||
a, ok := authFromContext(r.Context())
|
a, ok := authFromContext(r.Context())
|
||||||
if !ok {
|
if !ok {
|
||||||
@@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques
|
|||||||
if status == "" {
|
if status == "" {
|
||||||
status = store.PeerDiscoveryPending
|
status = store.PeerDiscoveryPending
|
||||||
}
|
}
|
||||||
// Refresh live discovery when listing pending.
|
|
||||||
if status == store.PeerDiscoveryPending || status == "all" {
|
|
||||||
views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true)
|
|
||||||
s.syncPeerDiscoveriesFromLive(a.TenantID, views)
|
|
||||||
}
|
|
||||||
listStatus := status
|
listStatus := status
|
||||||
if status == "all" {
|
if status == "all" {
|
||||||
listStatus = ""
|
listStatus = ""
|
||||||
@@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque
|
|||||||
writeStoreErr(w, err)
|
writeStoreErr(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve")
|
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve")
|
||||||
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
|
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
|
||||||
"peer_id": peer.ID,
|
"peer_id": peer.ID,
|
||||||
"discovery_id": disc.ID,
|
"discovery_id": disc.ID,
|
||||||
|
|||||||
+164
-95
@@ -1,47 +1,98 @@
|
|||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/nodedispatch"
|
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
const peerLiveCacheTTL = 15 * time.Second
|
|
||||||
|
|
||||||
type speakerBGPLive struct {
|
type speakerBGPLive struct {
|
||||||
SpeakerID string
|
SpeakerID string
|
||||||
Label string
|
Label string
|
||||||
Sessions []birdfmt.BGPSession
|
Sessions []birdfmt.BGPSession
|
||||||
Error string
|
Error string
|
||||||
|
Polled bool
|
||||||
|
ObservedAt time.Time
|
||||||
|
ReceivedAt time.Time
|
||||||
|
LastSuccessAt time.Time
|
||||||
|
Source string
|
||||||
|
ObservationOverride *liveObservation
|
||||||
}
|
}
|
||||||
|
|
||||||
type peerLiveCacheEntry struct {
|
type liveObservation struct {
|
||||||
at time.Time
|
ObservedAt *string `json:"observed_at"`
|
||||||
views []speakerBGPLive
|
ReceivedAt *string `json:"received_at"`
|
||||||
|
LastSuccessAt *string `json:"last_success_at"`
|
||||||
|
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
Quality string `json:"quality"`
|
||||||
|
ErrorCode string `json:"error_code,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry
|
func liveTime(t time.Time) *string {
|
||||||
|
if t.IsZero() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := t.UTC().Format(time.RFC3339Nano)
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v speakerBGPLive) observed() bool {
|
||||||
|
return v.Polled || v.Error != "" || len(v.Sessions) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v speakerBGPLive) successful() bool {
|
||||||
|
return v.observed() && v.Error == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v speakerBGPLive) observation() liveObservation {
|
||||||
|
if v.ObservationOverride != nil {
|
||||||
|
return *v.ObservationOverride
|
||||||
|
}
|
||||||
|
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
|
||||||
|
if o.Source == "" {
|
||||||
|
o.Source = "none"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case v.successful():
|
||||||
|
o.Quality = "ok"
|
||||||
|
case v.Error != "" && !v.LastSuccessAt.IsZero():
|
||||||
|
o.Quality = "stale"
|
||||||
|
o.ErrorCode = liveErrorCode(v.Error)
|
||||||
|
case v.Error != "":
|
||||||
|
o.Quality = "unreachable"
|
||||||
|
o.ErrorCode = liveErrorCode(v.Error)
|
||||||
|
default:
|
||||||
|
o.ErrorCode = "not_polled"
|
||||||
|
}
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
|
func liveErrorCode(err string) string {
|
||||||
|
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
|
||||||
|
return "not_configured"
|
||||||
|
}
|
||||||
|
return "poll_failed"
|
||||||
|
}
|
||||||
|
|
||||||
type peerSessionOnSpeaker struct {
|
type peerSessionOnSpeaker struct {
|
||||||
SpeakerID string `json:"speaker_id"`
|
SpeakerID string `json:"speaker_id"`
|
||||||
Label string `json:"label"`
|
Label string `json:"label"`
|
||||||
State string `json:"state"`
|
State string `json:"state"`
|
||||||
PollError string `json:"poll_error,omitempty"`
|
PollError string `json:"poll_error,omitempty"`
|
||||||
|
Observation liveObservation `json:"observation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type liveSpeakerPoll struct {
|
type liveSpeakerPoll struct {
|
||||||
SpeakerID string `json:"speaker_id"`
|
SpeakerID string `json:"speaker_id"`
|
||||||
Label string `json:"label"`
|
Label string `json:"label"`
|
||||||
OK bool `json:"ok"`
|
OK bool `json:"ok"`
|
||||||
SessionCount int `json:"session_count"`
|
SessionCount *int `json:"session_count"`
|
||||||
PollError string `json:"poll_error,omitempty"`
|
PollError string `json:"poll_error,omitempty"`
|
||||||
|
Observation liveObservation `json:"observation"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func speakerDisplayLabel(sp *store.Speaker) string {
|
func speakerDisplayLabel(sp *store.Speaker) string {
|
||||||
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
|
|||||||
return sp.ID
|
return sp.ID
|
||||||
}
|
}
|
||||||
|
|
||||||
func masterSpeakerID(speakers []*store.Speaker) string {
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
|
||||||
return sp.ID
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
|
|
||||||
if !fresh {
|
|
||||||
if v, ok := peerLiveCache.Load(tenantID); ok {
|
|
||||||
ent := v.(peerLiveCacheEntry)
|
|
||||||
if time.Since(ent.at) < peerLiveCacheTTL {
|
|
||||||
return ent.views
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
speakers := s.store.ListSpeakersForTenant(tenantID)
|
|
||||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
|
||||||
|
|
||||||
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
|
|
||||||
v := speakerBGPLive{Label: "CP (local BIRD)"}
|
|
||||||
if mid := masterSpeakerID(speakers); mid != "" {
|
|
||||||
v.SpeakerID = mid
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if sp != nil && sp.ID == mid {
|
|
||||||
v.Label = speakerDisplayLabel(sp)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
|
||||||
if err != nil {
|
|
||||||
v.Error = err.Error()
|
|
||||||
} else {
|
|
||||||
v.Sessions = birdfmt.ParseBGPSessions(out)
|
|
||||||
}
|
|
||||||
views = append(views, v)
|
|
||||||
}
|
|
||||||
|
|
||||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
|
||||||
type resWrap struct {
|
|
||||||
sp *store.Speaker
|
|
||||||
res nodedispatch.BirdProtocolsResult
|
|
||||||
}
|
|
||||||
ch := make(chan resWrap, len(speakers))
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if sp == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
|
||||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
wg.Add(1)
|
|
||||||
go func(speaker *store.Speaker) {
|
|
||||||
defer wg.Done()
|
|
||||||
ch <- resWrap{
|
|
||||||
sp: speaker,
|
|
||||||
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
|
|
||||||
}
|
|
||||||
}(sp)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(ch)
|
|
||||||
for rw := range ch {
|
|
||||||
views = append(views, speakerBGPLive{
|
|
||||||
SpeakerID: rw.sp.ID,
|
|
||||||
Label: speakerDisplayLabel(rw.sp),
|
|
||||||
Sessions: rw.res.Sessions,
|
|
||||||
Error: rw.res.Error,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
|
|
||||||
return views
|
|
||||||
}
|
|
||||||
|
|
||||||
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
|
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
|
||||||
out := make([]liveSpeakerPoll, 0, len(views))
|
out := make([]liveSpeakerPoll, 0, len(views))
|
||||||
for _, v := range views {
|
for _, v := range views {
|
||||||
|
var sessionCount *int
|
||||||
|
if v.successful() {
|
||||||
|
count := len(v.Sessions)
|
||||||
|
sessionCount = &count
|
||||||
|
}
|
||||||
out = append(out, liveSpeakerPoll{
|
out = append(out, liveSpeakerPoll{
|
||||||
SpeakerID: v.SpeakerID,
|
SpeakerID: v.SpeakerID,
|
||||||
Label: v.Label,
|
Label: v.Label,
|
||||||
OK: v.Error == "",
|
OK: v.successful(),
|
||||||
SessionCount: len(v.Sessions),
|
SessionCount: sessionCount,
|
||||||
PollError: v.Error,
|
PollError: v.Error,
|
||||||
|
Observation: v.observation(),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -183,11 +160,12 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
|||||||
protoName := birdfmt.PeerProtocolName(peer.ID)
|
protoName := birdfmt.PeerProtocolName(peer.ID)
|
||||||
|
|
||||||
for _, v := range views {
|
for _, v := range views {
|
||||||
if v.Error != "" && len(v.Sessions) == 0 {
|
if !v.successful() {
|
||||||
on = append(on, peerSessionOnSpeaker{
|
on = append(on, peerSessionOnSpeaker{
|
||||||
SpeakerID: v.SpeakerID,
|
SpeakerID: v.SpeakerID,
|
||||||
Label: v.Label,
|
Label: v.Label,
|
||||||
PollError: v.Error,
|
PollError: v.Error,
|
||||||
|
Observation: v.observation(),
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -197,6 +175,7 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
|||||||
SpeakerID: v.SpeakerID,
|
SpeakerID: v.SpeakerID,
|
||||||
Label: v.Label,
|
Label: v.Label,
|
||||||
State: "absent",
|
State: "absent",
|
||||||
|
Observation: v.observation(),
|
||||||
})
|
})
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
@@ -204,6 +183,7 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
|||||||
SpeakerID: v.SpeakerID,
|
SpeakerID: v.SpeakerID,
|
||||||
Label: v.Label,
|
Label: v.Label,
|
||||||
State: sess.State,
|
State: sess.State,
|
||||||
|
Observation: v.observation(),
|
||||||
}
|
}
|
||||||
on = append(on, hit)
|
on = append(on, hit)
|
||||||
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
|
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
|
||||||
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
|
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
|
||||||
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
expected := expectedPeerViews(peer, views)
|
||||||
|
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
|
||||||
|
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||||
row["session_on_speakers"] = on
|
row["session_on_speakers"] = on
|
||||||
row["established_on_speakers"] = establishedOn
|
row["established_on_speakers"] = establishedOn
|
||||||
row["session_conflict"] = false
|
row["session_conflict"] = false
|
||||||
row["session_mismatch"] = mismatch
|
row["session_mismatch"] = mismatch
|
||||||
|
row["last_known_session_state"] = row["session_state"]
|
||||||
|
row["session_state"] = nil
|
||||||
if state != "" {
|
if state != "" {
|
||||||
row["session_state"] = state
|
row["session_state"] = state
|
||||||
}
|
}
|
||||||
|
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
|
||||||
|
row["session_state"] = "absent"
|
||||||
|
}
|
||||||
|
row["live_observation"] = peerLiveObservation(expected)
|
||||||
if connLabel != "" {
|
if connLabel != "" {
|
||||||
row["connected_speaker_label"] = connLabel
|
row["connected_speaker_label"] = connLabel
|
||||||
}
|
}
|
||||||
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
|
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
|
||||||
|
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
|
||||||
|
return views
|
||||||
|
}
|
||||||
|
want := strings.TrimSpace(*peer.SpeakerID)
|
||||||
|
expected := make([]speakerBGPLive, 0, 1)
|
||||||
|
for _, v := range views {
|
||||||
|
if strings.EqualFold(v.SpeakerID, want) {
|
||||||
|
expected = append(expected, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return expected
|
||||||
|
}
|
||||||
|
|
||||||
|
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
|
||||||
|
for _, v := range views {
|
||||||
|
if !v.successful() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func peerLiveObservation(views []speakerBGPLive) liveObservation {
|
||||||
|
if len(views) == 0 {
|
||||||
|
return speakerBGPLive{}.observation()
|
||||||
|
}
|
||||||
|
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
|
||||||
|
if result.Source == "" {
|
||||||
|
result.Source = "none"
|
||||||
|
}
|
||||||
|
var latestObserved, latestReceived, latestSuccess time.Time
|
||||||
|
allOK, allFailed, anyStale := true, true, false
|
||||||
|
for _, v := range views {
|
||||||
|
o := v.observation()
|
||||||
|
if v.ObservedAt.After(latestObserved) {
|
||||||
|
latestObserved = v.ObservedAt
|
||||||
|
}
|
||||||
|
if v.ReceivedAt.After(latestReceived) {
|
||||||
|
latestReceived = v.ReceivedAt
|
||||||
|
}
|
||||||
|
if v.LastSuccessAt.After(latestSuccess) {
|
||||||
|
latestSuccess = v.LastSuccessAt
|
||||||
|
}
|
||||||
|
allOK = allOK && v.successful()
|
||||||
|
allFailed = allFailed && o.Quality == "unreachable"
|
||||||
|
anyStale = anyStale || o.Quality == "stale"
|
||||||
|
if result.ErrorCode == "" && o.ErrorCode != "" {
|
||||||
|
result.ErrorCode = o.ErrorCode
|
||||||
|
}
|
||||||
|
if o.ExpiresAt != nil {
|
||||||
|
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
|
||||||
|
if err == nil {
|
||||||
|
previous := time.Time{}
|
||||||
|
if result.ExpiresAt != nil {
|
||||||
|
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
|
||||||
|
}
|
||||||
|
if previous.IsZero() || expires.Before(previous) {
|
||||||
|
result.ExpiresAt = o.ExpiresAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
result.ObservedAt = liveTime(latestObserved)
|
||||||
|
result.ReceivedAt = liveTime(latestReceived)
|
||||||
|
result.LastSuccessAt = liveTime(latestSuccess)
|
||||||
|
if len(views) > 1 {
|
||||||
|
result.Source = "mixed"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case allOK:
|
||||||
|
result.Quality = "ok"
|
||||||
|
result.ErrorCode = ""
|
||||||
|
case anyStale:
|
||||||
|
result.Quality = "stale"
|
||||||
|
case allFailed:
|
||||||
|
result.Quality = "unreachable"
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
func peerLiveSpeakerIDOrNull(id string) any {
|
func peerLiveSpeakerIDOrNull(id string) any {
|
||||||
if strings.TrimSpace(id) == "" {
|
if strings.TrimSpace(id) == "" {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -2,11 +2,84 @@ package httpapi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestApplyPeerLiveFieldsObservationStates(t *testing.T) {
|
||||||
|
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||||
|
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
views []speakerBGPLive
|
||||||
|
wantState any
|
||||||
|
wantQuality string
|
||||||
|
}{
|
||||||
|
{"no collector", nil, nil, "unknown"},
|
||||||
|
{"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"},
|
||||||
|
{"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"},
|
||||||
|
{"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
row := map[string]any{"session_state": "Established"}
|
||||||
|
applyPeerLiveFields(row, peer, tc.views)
|
||||||
|
if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" {
|
||||||
|
t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"])
|
||||||
|
}
|
||||||
|
observation := row["live_observation"].(liveObservation)
|
||||||
|
if observation.Quality != tc.wantQuality {
|
||||||
|
t.Fatalf("observation=%+v", observation)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) {
|
||||||
|
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||||
|
assigned := "speaker-a"
|
||||||
|
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned}
|
||||||
|
viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"}
|
||||||
|
viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"}
|
||||||
|
viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||||
|
viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
assigned bool
|
||||||
|
views []speakerBGPLive
|
||||||
|
wantState any
|
||||||
|
wantQuality string
|
||||||
|
wantObserved string
|
||||||
|
}{
|
||||||
|
{"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)},
|
||||||
|
{"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)},
|
||||||
|
{"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)},
|
||||||
|
{"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
candidate := *peer
|
||||||
|
if !tc.assigned {
|
||||||
|
candidate.SpeakerID = nil
|
||||||
|
}
|
||||||
|
row := map[string]any{"session_state": "Established"}
|
||||||
|
applyPeerLiveFields(row, &candidate, tc.views)
|
||||||
|
obs := row["live_observation"].(liveObservation)
|
||||||
|
if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality {
|
||||||
|
t.Fatalf("state=%v observation=%+v", row["session_state"], obs)
|
||||||
|
}
|
||||||
|
if tc.wantObserved == "" && obs.ObservedAt != nil {
|
||||||
|
t.Fatalf("unexpected timestamp %v", *obs.ObservedAt)
|
||||||
|
}
|
||||||
|
if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) {
|
||||||
|
t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
|
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
|
||||||
peer := &store.BGPPeer{
|
peer := &store.BGPPeer{
|
||||||
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||||
|
|||||||
@@ -9,23 +9,33 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints.
|
const maxAuthRateLimitClients = 4096
|
||||||
|
|
||||||
|
type authRateHit struct {
|
||||||
|
start time.Time
|
||||||
|
count int
|
||||||
|
}
|
||||||
|
|
||||||
|
// authRateLimiter keeps a bounded fixed-window counter per client IP.
|
||||||
type authRateLimiter struct {
|
type authRateLimiter struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
hits map[string][]time.Time
|
hits map[string]authRateHit
|
||||||
limit int
|
limit int
|
||||||
window time.Duration
|
window time.Duration
|
||||||
|
maxClients int
|
||||||
|
nextSweep time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
func newAuthRateLimiter() *authRateLimiter {
|
func newAuthRateLimiter() *authRateLimiter {
|
||||||
limit := 60
|
limit := 60
|
||||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 {
|
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 {
|
||||||
limit = n
|
limit = n
|
||||||
}
|
}
|
||||||
return &authRateLimiter{
|
return &authRateLimiter{
|
||||||
hits: make(map[string][]time.Time),
|
hits: make(map[string]authRateHit),
|
||||||
limit: limit,
|
limit: limit,
|
||||||
window: time.Minute,
|
window: time.Minute,
|
||||||
|
maxClients: maxAuthRateLimitClients,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool {
|
|||||||
if l == nil {
|
if l == nil {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
now := time.Now()
|
return l.allowAt(ip, time.Now())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *authRateLimiter) allowAt(ip string, now time.Time) bool {
|
||||||
l.mu.Lock()
|
l.mu.Lock()
|
||||||
defer l.mu.Unlock()
|
defer l.mu.Unlock()
|
||||||
cut := now.Add(-l.window)
|
if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) {
|
||||||
arr := l.hits[ip]
|
l.nextSweep = time.Time{}
|
||||||
kept := arr[:0]
|
for key, hit := range l.hits {
|
||||||
for _, t := range arr {
|
expiry := hit.start.Add(l.window)
|
||||||
if t.After(cut) {
|
if !now.Before(expiry) {
|
||||||
kept = append(kept, t)
|
delete(l.hits, key)
|
||||||
|
} else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||||
|
l.nextSweep = expiry
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(kept) >= l.limit {
|
}
|
||||||
l.hits[ip] = kept
|
hit, exists := l.hits[ip]
|
||||||
|
if !exists && len(l.hits) >= l.maxClients {
|
||||||
|
// Never evict an active counter: a new identity must wait for expiry.
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
kept = append(kept, now)
|
if exists && hit.count >= l.limit {
|
||||||
l.hits[ip] = kept
|
return false
|
||||||
|
}
|
||||||
|
if !exists {
|
||||||
|
hit.start = now
|
||||||
|
expiry := now.Add(l.window)
|
||||||
|
if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||||
|
l.nextSweep = expiry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hit.count++
|
||||||
|
l.hits[ip] = hit
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,100 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const requestBodyLimit int64 = 8 << 20
|
||||||
|
|
||||||
|
// withRequestBodyLimit must run inside authMiddleware. Reading the complete
|
||||||
|
// bounded body before dispatch prevents a valid JSON prefix from committing a
|
||||||
|
// mutation before a later size/read error is discovered. Content-Type is not a
|
||||||
|
// security boundary; CSV retains its additional row/field checks in its handler.
|
||||||
|
func withRequestBodyLimit(next http.Handler) http.Handler {
|
||||||
|
return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next)
|
||||||
|
}
|
||||||
|
|
||||||
|
func withBodyLimit(limit int64, next http.Handler) http.Handler {
|
||||||
|
return withBodyLimitAndConcurrency(limit, 4, next)
|
||||||
|
}
|
||||||
|
|
||||||
|
func requestBodyConcurrency() int {
|
||||||
|
n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT")))
|
||||||
|
if err == nil && n >= 1 && n <= 64 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
return 4
|
||||||
|
}
|
||||||
|
|
||||||
|
func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler {
|
||||||
|
slots := make(chan struct{}, concurrency)
|
||||||
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
original := r.Body
|
||||||
|
if original != nil {
|
||||||
|
defer func() { _ = original.Close() }()
|
||||||
|
}
|
||||||
|
if r.ContentLength > limit {
|
||||||
|
writeRequestBodyTooLarge(w)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Context().Err() != nil {
|
||||||
|
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if r.Body == nil || r.Body == http.NoBody {
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case slots <- struct{}{}:
|
||||||
|
defer func() { <-slots }()
|
||||||
|
default:
|
||||||
|
w.Header().Set("Retry-After", "1")
|
||||||
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bounded := http.MaxBytesReader(w, r.Body, limit)
|
||||||
|
data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded})
|
||||||
|
var tooLarge *http.MaxBytesError
|
||||||
|
if errors.As(err, &tooLarge) {
|
||||||
|
writeRequestBodyTooLarge(w)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var timeout net.Error
|
||||||
|
if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
|
||||||
|
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.Body = io.NopCloser(bytes.NewReader(data))
|
||||||
|
r.ContentLength = int64(len(data))
|
||||||
|
next.ServeHTTP(w, r)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeRequestBodyTooLarge(w http.ResponseWriter) {
|
||||||
|
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit")
|
||||||
|
}
|
||||||
|
|
||||||
|
type requestContextReader struct {
|
||||||
|
ctx context.Context
|
||||||
|
reader io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r requestContextReader) Read(p []byte) (int, error) {
|
||||||
|
if err := r.ctx.Err(); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return r.reader.Read(p)
|
||||||
|
}
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
|
||||||
|
const limit = 32
|
||||||
|
for _, unknown := range []bool{false, true} {
|
||||||
|
for _, contentType := range []string{"", "application/json", "text/plain"} {
|
||||||
|
for _, extra := range []int{0, 1} {
|
||||||
|
label := "known"
|
||||||
|
if unknown {
|
||||||
|
label = "unknown"
|
||||||
|
}
|
||||||
|
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
|
||||||
|
mutations := 0
|
||||||
|
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var input map[string]any
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
mutations++
|
||||||
|
w.WriteHeader(http.StatusCreated)
|
||||||
|
}))
|
||||||
|
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||||
|
if unknown {
|
||||||
|
req.ContentLength = -1
|
||||||
|
}
|
||||||
|
if contentType != "" {
|
||||||
|
req.Header.Set("Content-Type", contentType)
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
if extra == 0 {
|
||||||
|
if w.Code != http.StatusCreated || mutations != 1 {
|
||||||
|
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
|
||||||
|
}
|
||||||
|
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
|
||||||
|
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type checkedRequestBody struct {
|
||||||
|
reads int
|
||||||
|
closed bool
|
||||||
|
reader io.Reader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
|
||||||
|
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
type failingRequestReader struct{}
|
||||||
|
|
||||||
|
func (failingRequestReader) Read([]byte) (int, error) {
|
||||||
|
return 0, errors.New("untrusted detailed read failure")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
|
||||||
|
for _, cancelled := range []bool{false, true} {
|
||||||
|
calls := 0
|
||||||
|
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||||
|
req.Body = b
|
||||||
|
req.ContentLength = -1
|
||||||
|
if cancelled {
|
||||||
|
ctx, cancel := context.WithCancel(req.Context())
|
||||||
|
cancel()
|
||||||
|
req = req.WithContext(ctx)
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
|
||||||
|
want := http.StatusBadRequest
|
||||||
|
if cancelled {
|
||||||
|
want = http.StatusRequestTimeout
|
||||||
|
}
|
||||||
|
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
|
||||||
|
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
|
||||||
|
}
|
||||||
|
if cancelled && b.reads != 0 {
|
||||||
|
t.Fatal("read body after cancellation")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
|
||||||
|
b := &checkedRequestBody{reader: failingRequestReader{}}
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||||
|
req.Body = b
|
||||||
|
req.ContentLength = requestBodyLimit + 1
|
||||||
|
s := &Server{}
|
||||||
|
called := false
|
||||||
|
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
|
||||||
|
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
|
||||||
|
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
|
||||||
|
req.ContentLength = length
|
||||||
|
called := false
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
|
||||||
|
if w.Code != http.StatusRequestEntityTooLarge || called {
|
||||||
|
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
|
||||||
|
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s.Close()
|
||||||
|
tenant, _, _, _, _ := s.store.DemoIDs()
|
||||||
|
before := len(s.store.ListModules(tenant))
|
||||||
|
h := s.Handler()
|
||||||
|
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
|
||||||
|
for _, extra := range []int{1, 0} {
|
||||||
|
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||||
|
req.ContentLength = -1
|
||||||
|
req.Header.Set("Authorization", "Bearer dev")
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
if extra == 1 {
|
||||||
|
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
|
||||||
|
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
|
||||||
|
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
|
||||||
|
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
f, ok := w.(http.Flusher)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("response lost Flusher")
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
|
_, _ = io.WriteString(w, "data: ready\n\n")
|
||||||
|
f.Flush()
|
||||||
|
})).ServeHTTP(w, req)
|
||||||
|
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
|
||||||
|
t.Fatal("stream response buffered")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockedAdmissionBody struct {
|
||||||
|
entered chan struct{}
|
||||||
|
resume chan struct{}
|
||||||
|
sent bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
|
||||||
|
if b.sent {
|
||||||
|
return 0, io.EOF
|
||||||
|
}
|
||||||
|
b.sent = true
|
||||||
|
close(b.entered)
|
||||||
|
<-b.resume
|
||||||
|
p[0] = 'x'
|
||||||
|
return 1, nil
|
||||||
|
}
|
||||||
|
func (*blockedAdmissionBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
|
||||||
|
readEntered, readResume := make(chan struct{}), make(chan struct{})
|
||||||
|
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
|
||||||
|
var once sync.Once
|
||||||
|
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.URL.Path != "/empty" {
|
||||||
|
once.Do(func() { close(handlerEntered) })
|
||||||
|
<-handlerResume
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}))
|
||||||
|
first := httptest.NewRequest(http.MethodPost, "/first", nil)
|
||||||
|
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
|
||||||
|
first.ContentLength = 0 // A claimed zero must not bypass body admission.
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
|
||||||
|
select {
|
||||||
|
case <-readEntered:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("first body did not start")
|
||||||
|
}
|
||||||
|
reject := func() {
|
||||||
|
t.Helper()
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
|
||||||
|
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
reject()
|
||||||
|
// An empty legacy GET remains available while all body slots are occupied.
|
||||||
|
empty := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
|
||||||
|
if empty.Code != http.StatusNoContent {
|
||||||
|
t.Fatal("empty GET consumed body slot")
|
||||||
|
}
|
||||||
|
close(readResume)
|
||||||
|
select {
|
||||||
|
case <-handlerEntered:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("handler did not start")
|
||||||
|
}
|
||||||
|
reject() // The fully read buffer is still retained by the running handler.
|
||||||
|
close(handlerResume)
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("first request did not finish")
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
|
||||||
|
if w.Code != http.StatusNoContent {
|
||||||
|
t.Fatalf("body slot not released: %d", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
|
||||||
|
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
|
||||||
|
for range 2 {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
|
||||||
|
req.Body = io.NopCloser(failingRequestReader{})
|
||||||
|
req.ContentLength = -1
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
h.ServeHTTP(w, req)
|
||||||
|
if w.Code != http.StatusBadRequest {
|
||||||
|
t.Fatalf("read failure retained slot: %d", w.Code)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
raw string
|
||||||
|
want int
|
||||||
|
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
|
||||||
|
t.Run(tc.raw, func(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
|
||||||
|
if got := requestBodyConcurrency(); got != tc.want {
|
||||||
|
t.Fatalf("concurrency=%d want%d", got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
+147
-79
@@ -8,14 +8,12 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/bundle"
|
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/pipeline"
|
"evobgp/internal/pipeline"
|
||||||
@@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler {
|
|||||||
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
|
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
|
||||||
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
|
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
|
||||||
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
|
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
|
||||||
s.mux.Handle("/v1/", s.authMiddleware(wrappedV1))
|
s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1)))
|
||||||
return s.withCORS(observability.HTTPMiddleware(s.mux))
|
return s.withCORS(observability.HTTPMiddleware(s.mux))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
|||||||
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
|
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
|
||||||
m.HandleFunc("GET /peers", s.handleListPeers)
|
m.HandleFunc("GET /peers", s.handleListPeers)
|
||||||
m.HandleFunc("GET /speakers", s.handleListSpeakers)
|
m.HandleFunc("GET /speakers", s.handleListSpeakers)
|
||||||
|
m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh)
|
||||||
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
|
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
|
||||||
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
|
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
|
||||||
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
|
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
|
||||||
@@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
|||||||
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
|
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
|
||||||
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
|
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
|
||||||
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
|
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
|
||||||
|
m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob)
|
||||||
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
|
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
|
||||||
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
|
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
|
||||||
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
|
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
|
||||||
@@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
|||||||
allPeers := s.store.ListPeers(a.TenantID)
|
allPeers := s.store.ListPeers(a.TenantID)
|
||||||
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
|
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
|
||||||
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
|
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
|
||||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID))
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
if err != nil {
|
||||||
defer cancel()
|
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||||
liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh)
|
return
|
||||||
if fresh {
|
|
||||||
s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews)
|
|
||||||
}
|
}
|
||||||
items := make([]map[string]any, 0, len(page))
|
items := make([]map[string]any, 0, len(page))
|
||||||
for _, p := range page {
|
for _, p := range page {
|
||||||
@@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string {
|
|
||||||
if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") {
|
|
||||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
|
||||||
}
|
|
||||||
if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok {
|
|
||||||
return cached
|
|
||||||
}
|
|
||||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string {
|
|
||||||
sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
|
|
||||||
if sock == "" {
|
|
||||||
return map[string]string{}
|
|
||||||
}
|
|
||||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
|
||||||
if err != nil {
|
|
||||||
return map[string]string{}
|
|
||||||
}
|
|
||||||
states := birdfmt.ParseBGPProtocolStates(out)
|
|
||||||
observability.SetBirdProtocolStates(states)
|
|
||||||
return states
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
|
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
|
||||||
func parseBGPProtocolStates(output string) map[string]string {
|
func parseBGPProtocolStates(output string) map[string]string {
|
||||||
out := make(map[string]string)
|
out := make(map[string]string)
|
||||||
@@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) {
|
|||||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
||||||
var liveByID map[string]map[string]any
|
var liveByID map[string]map[string]any
|
||||||
if fresh {
|
if fresh {
|
||||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
var err error
|
||||||
defer cancel()
|
_, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers)
|
||||||
liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers)
|
if err != nil {
|
||||||
|
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||||
|
return
|
||||||
|
}
|
||||||
}
|
}
|
||||||
items := make([]map[string]any, 0, len(speakers))
|
items := make([]map[string]any, 0, len(speakers))
|
||||||
for _, sp := range speakers {
|
for _, sp := range speakers {
|
||||||
@@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) {
|
|||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
mid := mod.ID
|
mid := mod.ID
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
||||||
"module_id": moduleID,
|
"module_id": moduleID,
|
||||||
"trigger": "api",
|
"trigger": "api",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
moduleIDs := body.ModuleIDs
|
moduleIDs := body.ModuleIDs
|
||||||
|
sort.Strings(body.ModuleIDs)
|
||||||
|
requestIDs := append([]string{}, body.ModuleIDs...)
|
||||||
|
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||||
|
var idemPtr *string
|
||||||
|
if idem != "" {
|
||||||
|
idemPtr = &idem
|
||||||
|
}
|
||||||
|
request := map[string]any{"module_ids": requestIDs, "trigger": "api"}
|
||||||
|
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) {
|
||||||
|
return
|
||||||
|
}
|
||||||
if len(moduleIDs) == 0 {
|
if len(moduleIDs) == 0 {
|
||||||
now := time.Now().UTC()
|
now := time.Now().UTC()
|
||||||
for _, mod := range s.store.ListModules(a.TenantID) {
|
for _, mod := range s.store.ListModules(a.TenantID) {
|
||||||
@@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if len(moduleIDs) == 0 {
|
if len(moduleIDs) == 0 && idem == "" {
|
||||||
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
|
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
||||||
var idemPtr *string
|
|
||||||
if idem != "" {
|
|
||||||
idemPtr = &idem
|
|
||||||
}
|
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
|
||||||
"module_ids": moduleIDs,
|
"module_ids": moduleIDs,
|
||||||
"trigger": "api",
|
"trigger": "api",
|
||||||
})
|
}, request)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||||
"job_id": snap["job_id"],
|
"job_id": snap["job_id"],
|
||||||
"status": snap["status"],
|
"status": snap["status"],
|
||||||
"module_ids": moduleIDs,
|
"module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"],
|
||||||
"modules_due": len(moduleIDs),
|
"modules_due": len(moduleIDs),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any {
|
|||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error).
|
// enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation.
|
||||||
func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) {
|
func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) {
|
||||||
if s.jobs == nil {
|
if s.jobs == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
mid := moduleID
|
mid := moduleID
|
||||||
key := "module_refresh:" + moduleID
|
// Each mutation is a new logical refresh. Reusing a module-wide key would
|
||||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{
|
// incorrectly replay an older terminal job after later configuration changes.
|
||||||
|
_, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
|
||||||
"module_id": moduleID,
|
"module_id": moduleID,
|
||||||
"trigger": trigger,
|
"trigger": trigger,
|
||||||
})
|
})
|
||||||
|
s.reportAutomaticEnqueue(w, r, trigger, moduleID, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
|
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
|
||||||
func (s *Server) enqueuePeerReconcile(tenantID, trigger string) {
|
func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) {
|
||||||
if s.jobs == nil {
|
if s.jobs == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
_, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
||||||
"trigger": trigger,
|
"trigger": trigger,
|
||||||
"job_title": "Обновление BGP пиров",
|
"job_title": "Обновление BGP пиров",
|
||||||
})
|
})
|
||||||
|
s.reportAutomaticEnqueue(w, r, trigger, "", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) {
|
||||||
|
if err == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed")
|
||||||
|
if a, ok := authFromContext(r.Context()); ok {
|
||||||
|
s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
|
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||||
@@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request)
|
|||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
||||||
"source_revision_id": revID,
|
"source_revision_id": revID,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) {
|
|||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||||
"revision_id": revID,
|
"revision_id": revID,
|
||||||
"strategy": body.Strategy,
|
"strategy": body.Strategy,
|
||||||
"trigger": "api",
|
"trigger": "api",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) {
|
|||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||||
"revision_id": revID,
|
"revision_id": revID,
|
||||||
"speaker_id": spkID,
|
"speaker_id": spkID,
|
||||||
"trigger": "api",
|
"trigger": "api",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) {
|
|||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) {
|
|||||||
status := r.URL.Query().Get("status")
|
status := r.URL.Query().Get("status")
|
||||||
kind := r.URL.Query().Get("kind")
|
kind := r.URL.Query().Get("kind")
|
||||||
cursor := r.URL.Query().Get("cursor")
|
cursor := r.URL.Query().Get("cursor")
|
||||||
list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit)
|
list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit)
|
||||||
|
if err != nil {
|
||||||
|
writeInternalError(w, "jobs_list", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
items := make([]map[string]any, 0, len(list))
|
items := make([]map[string]any, 0, len(list))
|
||||||
for _, j := range list {
|
for _, j := range list {
|
||||||
items = append(items, j.Snapshot())
|
items = append(items, j.Snapshot())
|
||||||
@@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
writeJobReadError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, j.Snapshot())
|
writeJSON(w, http.StatusOK, j.Snapshot())
|
||||||
@@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
writeJobReadError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
snap := j.Snapshot()
|
snap := j.Snapshot()
|
||||||
@@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
|||||||
out["revision_id"] = v
|
out["revision_id"] = v
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
sourceJobID := ""
|
||||||
|
if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" {
|
||||||
|
// Resolve through the tenant-scoped registry before exposing linked progress.
|
||||||
|
if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil {
|
||||||
|
writeJobReadError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sourceJobID = source
|
||||||
|
out["source_deployment_job_id"] = source
|
||||||
|
}
|
||||||
|
rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID)
|
||||||
|
if err == nil && len(rows) == 0 && sourceJobID != "" {
|
||||||
|
rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
progress := make([]map[string]any, 0, len(rows))
|
||||||
|
for _, d := range rows {
|
||||||
|
progress = append(progress, map[string]any{
|
||||||
|
"job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID,
|
||||||
|
"generation": d.Generation, "state": d.State, "confirmation": d.Confirmation,
|
||||||
|
"error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound,
|
||||||
|
"health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID,
|
||||||
|
"health_source_at": d.HealthSourceAt, "health_error": d.HealthError,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
out["deployment_progress"] = progress
|
||||||
writeJSON(w, http.StatusOK, out)
|
writeJSON(w, http.StatusOK, out)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
|
|||||||
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id"))
|
j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
writeJobReadError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
|
||||||
|
a, ok := authFromContext(r.Context())
|
||||||
|
if !ok {
|
||||||
|
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, store.ErrJobQueueFull) {
|
||||||
|
writeJobEnqueueError(w, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if errors.Is(err, store.ErrJobConflict) {
|
||||||
|
writeProblem(w, http.StatusConflict, "Conflict", "job is not failed")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJobReadError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||||
@@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
|
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rev, err := s.store.GetRevision(a.TenantID, rid)
|
overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid)
|
||||||
if err != nil {
|
|
||||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
frags := rev.PreviewFragments
|
|
||||||
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "bundle overlay", err)
|
writeInternalError(w, "bundle overlay", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
frags = overlaid
|
// Finished gzip, including manifest and bundle.sig. Revision bytes are immutable;
|
||||||
tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv)
|
// the overlay fingerprint keeps speaker-specific renders from colliding.
|
||||||
|
tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) {
|
||||||
|
return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP)
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "internal", err)
|
if errors.Is(err, errNodeBundleRevisionMissing) {
|
||||||
|
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
op := "internal"
|
||||||
|
var opErr *nodeBundleOpError
|
||||||
|
if errors.As(err, &opErr) && opErr.op != "" {
|
||||||
|
op = opErr.op
|
||||||
|
}
|
||||||
|
writeInternalError(w, op, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Content-Type", "application/gzip")
|
w.Header().Set("Content-Type", "application/gzip")
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package httpapi
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"log"
|
"log"
|
||||||
"net"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
@@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func clientIP(r *http.Request) string {
|
func clientIP(r *http.Request) string {
|
||||||
if r == nil {
|
return clientIPWithTrustedProxies(r, trustedProxyPrefixes())
|
||||||
return ""
|
|
||||||
}
|
|
||||||
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
|
|
||||||
parts := strings.Split(xff, ",")
|
|
||||||
if len(parts) > 0 {
|
|
||||||
return strings.TrimSpace(parts[0])
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
|
|
||||||
return xrip
|
|
||||||
}
|
|
||||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
|
||||||
if err != nil {
|
|
||||||
return strings.TrimSpace(r.RemoteAddr)
|
|
||||||
}
|
|
||||||
return host
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// initAuditPusher wires portal push when URL and secret are configured.
|
// initAuditPusher wires portal push when URL and secret are configured.
|
||||||
|
|||||||
@@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
|||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation)
|
||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
if !ok {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
defer release()
|
||||||
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeStoreErr(w, err)
|
writeStoreErr(w, err)
|
||||||
@@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
|||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||||
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||||
|
return
|
||||||
|
}
|
||||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
|
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
|
||||||
@@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
|||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||||
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
|
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
raw, err := pipeline.ReadCDNBody(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeBadGateway(w, "cdn preview read body", err)
|
writeBadGateway(w, "cdn preview read body", err)
|
||||||
return
|
return
|
||||||
@@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
|
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create")
|
||||||
writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
|
writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
|
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch")
|
||||||
writeJSON(w, http.StatusOK, cdnSourceJSON(x))
|
writeJSON(w, http.StatusOK, cdnSourceJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
|
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create")
|
||||||
writeJSON(w, http.StatusCreated, asEntryJSON(x))
|
writeJSON(w, http.StatusCreated, asEntryJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch")
|
||||||
writeJSON(w, http.StatusOK, asEntryJSON(x))
|
writeJSON(w, http.StatusOK, asEntryJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create")
|
||||||
writeJSON(w, http.StatusCreated, domainEntryJSON(x))
|
writeJSON(w, http.StatusCreated, domainEntryJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch")
|
||||||
writeJSON(w, http.StatusOK, domainEntryJSON(x))
|
writeJSON(w, http.StatusOK, domainEntryJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create")
|
||||||
writeJSON(w, http.StatusCreated, ipRangeJSON(x))
|
writeJSON(w, http.StatusCreated, ipRangeJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch")
|
||||||
writeJSON(w, http.StatusOK, ipRangeJSON(x))
|
writeJSON(w, http.StatusOK, ipRangeJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
moduleID := r.PathValue("module_id")
|
moduleID := r.PathValue("module_id")
|
||||||
res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body)
|
res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, importer.ErrCSVTooLarge) {
|
||||||
|
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests")
|
||||||
|
return
|
||||||
|
}
|
||||||
if errors.Is(err, store.ErrInvalidInput) {
|
if errors.Is(err, store.ErrInvalidInput) {
|
||||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
|
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
|
||||||
return
|
return
|
||||||
@@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
|||||||
if res.Imported > 0 {
|
if res.Imported > 0 {
|
||||||
switch res.ModuleType {
|
switch res.ModuleType {
|
||||||
case "AS_PREFIXES":
|
case "AS_PREFIXES":
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv")
|
||||||
case "DOMAINS":
|
case "DOMAINS":
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv")
|
||||||
case "IP_RANGES":
|
case "IP_RANGES":
|
||||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv")
|
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
writeJSON(w, http.StatusOK, map[string]any{
|
writeJSON(w, http.StatusOK, map[string]any{
|
||||||
@@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||||
s.enqueuePeerReconcile(a.TenantID, "peer_create")
|
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create")
|
||||||
writeJSON(w, http.StatusCreated, peerJSON(x))
|
writeJSON(w, http.StatusCreated, peerJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||||
s.enqueuePeerReconcile(a.TenantID, "peer_patch")
|
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch")
|
||||||
writeJSON(w, http.StatusOK, peerJSON(x))
|
writeJSON(w, http.StatusOK, peerJSON(x))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
|
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
|
||||||
s.enqueuePeerReconcile(a.TenantID, "peer_delete")
|
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete")
|
||||||
w.WriteHeader(http.StatusNoContent)
|
w.WriteHeader(http.StatusNoContent)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
if peerDiscoverySettingsChanged(body) {
|
if peerDiscoverySettingsChanged(body) {
|
||||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings")
|
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings")
|
||||||
}
|
}
|
||||||
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
|
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
|
||||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
@@ -8,8 +9,60 @@ import (
|
|||||||
"net/http/httptest"
|
"net/http/httptest"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/importer"
|
||||||
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestCSVImportLimitsBeforeMutations(t *testing.T) {
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := m.DemoIDs()
|
||||||
|
mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
module := mod.ID
|
||||||
|
s := &Server{store: m}
|
||||||
|
before, err := m.ListASEntries(tenant, module)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
header := "asn,community\n"
|
||||||
|
validPrefix := header + "64512,\n"
|
||||||
|
oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n"
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, body string
|
||||||
|
unknownLength bool
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK},
|
||||||
|
{"oversize", oversized, false, http.StatusRequestEntityTooLarge},
|
||||||
|
{"unknown_length", oversized, true, http.StatusRequestEntityTooLarge},
|
||||||
|
{"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge},
|
||||||
|
{"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body))
|
||||||
|
if tc.unknownLength {
|
||||||
|
req.ContentLength = -1
|
||||||
|
req.TransferEncoding = []string{"chunked"}
|
||||||
|
}
|
||||||
|
req.SetPathValue("module_id", module)
|
||||||
|
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
s.handleImportModuleEntriesCSV(w, req)
|
||||||
|
if w.Code != tc.status {
|
||||||
|
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
after, e := m.ListASEntries(tenant, module)
|
||||||
|
if e != nil || len(after) != len(before) {
|
||||||
|
t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
|
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
|
||||||
srv, err := New(Options{
|
srv, err := New(Options{
|
||||||
InsecureDev: true,
|
InsecureDev: true,
|
||||||
|
|||||||
@@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
|
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
|
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, store.ErrInvalidInput) {
|
if errors.Is(err, store.ErrInvalidInput) {
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
@@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{
|
if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{
|
||||||
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
|
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
kind := "maintenance_policy_run"
|
kind := "maintenance_policy_run"
|
||||||
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
|
||||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||||
var idemPtr *string
|
var idemPtr *string
|
||||||
if idem != "" {
|
if idem != "" {
|
||||||
@@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
|||||||
if dryRun {
|
if dryRun {
|
||||||
title = "Maintenance policy dry-run"
|
title = "Maintenance policy dry-run"
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
||||||
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||||
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
|
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "maintenance_policy_enqueue", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import (
|
|||||||
|
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
"evobgp/internal/pgmonitor"
|
"evobgp/internal/pgmonitor"
|
||||||
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a
|
|||||||
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
|
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
|
||||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||||
var idemPtr *string
|
var idemPtr *string
|
||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
meta["actor_prefix"] = actorPrefix(a)
|
meta["actor_prefix"] = actorPrefix(a)
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta)
|
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||||
|
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
@@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
|
|||||||
if idem != "" {
|
if idem != "" {
|
||||||
idemPtr = &idem
|
idemPtr = &idem
|
||||||
}
|
}
|
||||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||||
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
|
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
|
||||||
"job_title": "PostgreSQL cleanup (deprecated path)",
|
"job_title": "PostgreSQL cleanup (deprecated path)",
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
writeJobEnqueueError(w, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||||
|
|||||||
@@ -8,15 +8,17 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/audit"
|
"evobgp/internal/audit"
|
||||||
"evobgp/internal/jobs"
|
"evobgp/internal/jobs"
|
||||||
|
"evobgp/internal/logging"
|
||||||
"evobgp/internal/maintenance"
|
"evobgp/internal/maintenance"
|
||||||
"evobgp/internal/pgmonitor"
|
"evobgp/internal/pgmonitor"
|
||||||
"evobgp/internal/repository"
|
|
||||||
"evobgp/internal/runtimelogs"
|
"evobgp/internal/runtimelogs"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
|
"evobgp/internal/telemetry"
|
||||||
|
|
||||||
"github.com/jackc/pgx/v5/pgxpool"
|
"github.com/jackc/pgx/v5/pgxpool"
|
||||||
)
|
)
|
||||||
@@ -30,7 +32,11 @@ type Server struct {
|
|||||||
maintStats *maintenance.DBStatsProvider
|
maintStats *maintenance.DBStatsProvider
|
||||||
jobs *jobs.Registry
|
jobs *jobs.Registry
|
||||||
authLimiter *authRateLimiter
|
authLimiter *authRateLimiter
|
||||||
|
operationOnce sync.Once
|
||||||
|
operationLimiter *operationLimiter
|
||||||
bundlePriv ed25519.PrivateKey
|
bundlePriv ed25519.PrivateKey
|
||||||
|
bundleCache *nodeBundleCache
|
||||||
|
nodeBundleBuild nodeBundleBuilder
|
||||||
keyResolver *apiKeyResolver
|
keyResolver *apiKeyResolver
|
||||||
firewallResolver *firewallTokenResolver
|
firewallResolver *firewallTokenResolver
|
||||||
bundleSeedHex string
|
bundleSeedHex string
|
||||||
@@ -39,6 +45,11 @@ type Server struct {
|
|||||||
runtimeLogs *runtimelogs.Service
|
runtimeLogs *runtimelogs.Service
|
||||||
runtimeLogsPolicyTenant string
|
runtimeLogsPolicyTenant string
|
||||||
mux *http.ServeMux
|
mux *http.ServeMux
|
||||||
|
backgroundMu sync.Mutex
|
||||||
|
backgroundStarted bool
|
||||||
|
backgroundClosed bool
|
||||||
|
backgroundCancel context.CancelFunc
|
||||||
|
telemetryCollector *telemetry.Collector
|
||||||
|
|
||||||
// Portal / dual-auth (JWT) configuration.
|
// Portal / dual-auth (JWT) configuration.
|
||||||
jwtSecret string
|
jwtSecret string
|
||||||
@@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) {
|
|||||||
maintStats: maintStats,
|
maintStats: maintStats,
|
||||||
jobs: reg,
|
jobs: reg,
|
||||||
bundlePriv: priv,
|
bundlePriv: priv,
|
||||||
|
bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes),
|
||||||
keyResolver: resolver,
|
keyResolver: resolver,
|
||||||
firewallResolver: fwResolver,
|
firewallResolver: fwResolver,
|
||||||
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
|
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
|
||||||
@@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) {
|
|||||||
|
|
||||||
// Close releases database resources.
|
// Close releases database resources.
|
||||||
func (s *Server) Close() {
|
func (s *Server) Close() {
|
||||||
|
if s == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.backgroundMu.Lock()
|
||||||
|
if s.backgroundClosed {
|
||||||
|
s.backgroundMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.backgroundClosed = true
|
||||||
|
cancel := s.backgroundCancel
|
||||||
|
collector := s.telemetryCollector
|
||||||
|
s.backgroundMu.Unlock()
|
||||||
|
if cancel != nil {
|
||||||
|
cancel()
|
||||||
|
}
|
||||||
|
if collector != nil {
|
||||||
|
_ = collector.Close(context.Background())
|
||||||
|
}
|
||||||
|
// Do not close the pool while a worker is still persisting its final state.
|
||||||
|
if s.jobs != nil {
|
||||||
|
s.jobs.RequestCancelAll()
|
||||||
|
_ = s.jobs.Drain(context.Background())
|
||||||
|
}
|
||||||
if s.pgPool != nil {
|
if s.pgPool != nil {
|
||||||
s.pgPool.Close()
|
s.pgPool.Close()
|
||||||
}
|
}
|
||||||
@@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store }
|
|||||||
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
|
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
|
||||||
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
|
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
|
||||||
|
|
||||||
// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled.
|
// StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled.
|
||||||
func (s *Server) StartBackground(ctx context.Context) {
|
func (s *Server) StartBackground(ctx context.Context) {
|
||||||
if s != nil && s.pgPool != nil {
|
if s == nil {
|
||||||
pgmonitor.StartScheduler(ctx, s.pgPool)
|
return
|
||||||
|
}
|
||||||
|
s.backgroundMu.Lock()
|
||||||
|
defer s.backgroundMu.Unlock()
|
||||||
|
if s.backgroundClosed || s.backgroundStarted {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.backgroundStarted = true
|
||||||
|
backgroundCtx, cancel := context.WithCancel(ctx)
|
||||||
|
s.backgroundCancel = cancel
|
||||||
|
if s.store != nil {
|
||||||
|
s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv())
|
||||||
|
}
|
||||||
|
if s.pgPool != nil {
|
||||||
|
pgmonitor.StartScheduler(backgroundCtx, s.pgPool)
|
||||||
if s.jobs != nil {
|
if s.jobs != nil {
|
||||||
jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool))
|
jobs.StartDurableQueueLoop(backgroundCtx, s.jobs)
|
||||||
|
jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if s != nil && s.maintConfig != nil && s.jobs != nil {
|
if s != nil && s.maintConfig != nil && s.jobs != nil {
|
||||||
maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
||||||
key := idem
|
key := idem
|
||||||
_, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
_, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
||||||
"policy_id": policyID,
|
"policy_id": policyID,
|
||||||
"dry_run": dryRun,
|
"dry_run": dryRun,
|
||||||
"trigger": "scheduler",
|
"trigger": "scheduler",
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err)
|
||||||
|
}
|
||||||
}, 30*time.Second)
|
}, 30*time.Second)
|
||||||
}
|
}
|
||||||
if s != nil && s.runtimeLogs != nil && s.store != nil {
|
if s != nil && s.runtimeLogs != nil && s.store != nil {
|
||||||
runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{
|
runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{
|
||||||
Service: s.runtimeLogs,
|
Service: s.runtimeLogs,
|
||||||
Store: s.store,
|
Store: s.store,
|
||||||
PolicyTenant: s.runtimeLogsPolicyTenant,
|
PolicyTenant: s.runtimeLogsPolicyTenant,
|
||||||
|
|||||||
@@ -6,7 +6,11 @@ import (
|
|||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"net/url"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -165,24 +169,65 @@ func publicControlPlaneURL(r *http.Request, override string) string {
|
|||||||
if s := strings.TrimSpace(override); s != "" {
|
if s := strings.TrimSpace(override); s != "" {
|
||||||
return strings.TrimRight(s, "/")
|
return strings.TrimRight(s, "/")
|
||||||
}
|
}
|
||||||
if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") {
|
if r == nil {
|
||||||
return strings.TrimRight(origin, "/")
|
|
||||||
}
|
|
||||||
proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
|
|
||||||
if proto == "" {
|
|
||||||
proto = "https"
|
|
||||||
}
|
|
||||||
host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
|
|
||||||
if i := strings.Index(host, ","); i >= 0 {
|
|
||||||
host = strings.TrimSpace(host[:i])
|
|
||||||
}
|
|
||||||
if host == "" {
|
|
||||||
host = strings.TrimSpace(r.Host)
|
|
||||||
}
|
|
||||||
if host == "" {
|
|
||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) {
|
||||||
|
protos := r.Header.Values("X-Forwarded-Proto")
|
||||||
|
hosts := r.Header.Values("X-Forwarded-Host")
|
||||||
|
if len(protos) == 1 && len(hosts) == 1 {
|
||||||
|
proto := strings.TrimSpace(protos[0])
|
||||||
|
host := strings.TrimSpace(hosts[0])
|
||||||
|
if (proto == "http" || proto == "https") && validControlPlaneHost(host) {
|
||||||
return proto + "://" + host
|
return proto + "://" + host
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !validControlPlaneHost(r.Host) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
proto := "http"
|
||||||
|
if r.TLS != nil {
|
||||||
|
proto = "https"
|
||||||
|
}
|
||||||
|
return proto + "://" + r.Host
|
||||||
|
}
|
||||||
|
|
||||||
|
func validControlPlaneHost(host string) bool {
|
||||||
|
if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
u, err := url.Parse("http://" + host)
|
||||||
|
if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
name := u.Hostname()
|
||||||
|
if name == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if strings.HasPrefix(host, "[") {
|
||||||
|
if _, err := netip.ParseAddr(name); err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
} else if net.ParseIP(name) == nil {
|
||||||
|
for _, label := range strings.Split(name, ".") {
|
||||||
|
if label == "" || label[0] == '-' || label[len(label)-1] == '-' {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, c := range label {
|
||||||
|
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if port := u.Port(); port != "" {
|
||||||
|
n, err := strconv.Atoi(port)
|
||||||
|
if err != nil || n < 1 || n > 65535 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
|
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
package httpapi
|
package httpapi
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
@@ -11,6 +9,34 @@ import (
|
|||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type agentHealthPoll struct {
|
||||||
|
result nodedispatch.AgentHealthResult
|
||||||
|
at time.Time
|
||||||
|
lastSuccessAt time.Time
|
||||||
|
observationOverride *liveObservation
|
||||||
|
}
|
||||||
|
|
||||||
|
func agentHealthObservation(poll *agentHealthPoll) liveObservation {
|
||||||
|
if poll == nil {
|
||||||
|
return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"}
|
||||||
|
}
|
||||||
|
if poll.observationOverride != nil {
|
||||||
|
return *poll.observationOverride
|
||||||
|
}
|
||||||
|
o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"}
|
||||||
|
if !poll.result.OK {
|
||||||
|
o.Quality = "unreachable"
|
||||||
|
if !poll.lastSuccessAt.IsZero() {
|
||||||
|
o.Quality = "stale"
|
||||||
|
}
|
||||||
|
o.ErrorCode = liveErrorCode(poll.result.Error)
|
||||||
|
if poll.result.Error == "agent reported ok=false" {
|
||||||
|
o.ErrorCode = "reported_unhealthy"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return o
|
||||||
|
}
|
||||||
|
|
||||||
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
||||||
total = len(sessions)
|
total = len(sessions)
|
||||||
for _, s := range sessions {
|
for _, s := range sessions {
|
||||||
@@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
|
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
|
||||||
|
var poll *agentHealthPoll
|
||||||
|
if health != nil {
|
||||||
|
poll = &agentHealthPoll{result: *health}
|
||||||
|
}
|
||||||
|
return speakerLiveStatusWithPoll(sp, view, poll)
|
||||||
|
}
|
||||||
|
|
||||||
|
func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any {
|
||||||
total, established := countBGPSessions(view.Sessions)
|
total, established := countBGPSessions(view.Sessions)
|
||||||
|
bgpObservation := view.observation()
|
||||||
|
agentObservation := agentHealthObservation(health)
|
||||||
|
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil {
|
||||||
|
agentObservation = bgpObservation
|
||||||
|
}
|
||||||
m := map[string]any{
|
m := map[string]any{
|
||||||
"label": view.Label,
|
"label": view.Label,
|
||||||
"bgp_poll_ok": view.Error == "",
|
"bgp_poll_ok": view.successful(),
|
||||||
"bgp_sessions_total": total,
|
"bgp_sessions_total": nil,
|
||||||
"bgp_established": established,
|
"bgp_established": nil,
|
||||||
|
"bgp_observation": bgpObservation,
|
||||||
|
"agent_observation": agentObservation,
|
||||||
|
}
|
||||||
|
if view.successful() {
|
||||||
|
m["bgp_sessions_total"] = total
|
||||||
|
m["bgp_established"] = established
|
||||||
}
|
}
|
||||||
if view.Error != "" {
|
if view.Error != "" {
|
||||||
m["bgp_poll_error"] = view.Error
|
m["bgp_poll_error"] = view.Error
|
||||||
}
|
}
|
||||||
if health != nil {
|
if health != nil {
|
||||||
m["agent_ok"] = health.OK
|
m["agent_ok"] = health.result.OK
|
||||||
if health.Error != "" {
|
if health.result.Error != "" {
|
||||||
m["agent_error"] = health.Error
|
m["agent_error"] = health.result.Error
|
||||||
}
|
}
|
||||||
if health.LastSyncAt != "" {
|
if health.result.LastSyncAt != "" {
|
||||||
m["agent_last_sync_at"] = health.LastSyncAt
|
m["agent_last_sync_at"] = health.result.LastSyncAt
|
||||||
}
|
}
|
||||||
if health.LastAppliedRevisionID != "" {
|
if health.result.LastAppliedRevisionID != "" {
|
||||||
m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID
|
m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID
|
||||||
}
|
}
|
||||||
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||||
m["agent_ok"] = view.Error == ""
|
m["agent_ok"] = view.successful()
|
||||||
if view.Error != "" {
|
if view.Error != "" {
|
||||||
m["agent_error"] = view.Error
|
m["agent_error"] = view.Error
|
||||||
}
|
}
|
||||||
@@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
|||||||
m["agent_ok"] = false
|
m["agent_ok"] = false
|
||||||
m["agent_error"] = "agent health not polled"
|
m["agent_error"] = "agent health not polled"
|
||||||
}
|
}
|
||||||
if len(view.Sessions) > 0 {
|
if view.successful() {
|
||||||
sess := make([]map[string]any, 0, len(view.Sessions))
|
sess := make([]map[string]any, 0, len(view.Sessions))
|
||||||
for _, s := range view.Sessions {
|
for _, s := range view.Sessions {
|
||||||
row := map[string]any{
|
row := map[string]any{
|
||||||
@@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
|||||||
}
|
}
|
||||||
return m
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any {
|
|
||||||
views := s.collectSpeakerBGPLive(ctx, tenantID, fresh)
|
|
||||||
viewByID := make(map[string]speakerBGPLive, len(views))
|
|
||||||
for _, v := range views {
|
|
||||||
if v.SpeakerID != "" {
|
|
||||||
viewByID[v.SpeakerID] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
|
||||||
type healthWrap struct {
|
|
||||||
id string
|
|
||||||
h nodedispatch.AgentHealthResult
|
|
||||||
}
|
|
||||||
healthCh := make(chan healthWrap, len(speakers))
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if sp == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
|
||||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
wg.Add(1)
|
|
||||||
go func(speaker *store.Speaker) {
|
|
||||||
defer wg.Done()
|
|
||||||
healthCh <- healthWrap{
|
|
||||||
id: speaker.ID,
|
|
||||||
h: nodedispatch.FetchAgentHealth(ctx, speaker, opts),
|
|
||||||
}
|
|
||||||
}(sp)
|
|
||||||
}
|
|
||||||
wg.Wait()
|
|
||||||
close(healthCh)
|
|
||||||
healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers))
|
|
||||||
for hw := range healthCh {
|
|
||||||
healthByID[hw.id] = hw.h
|
|
||||||
}
|
|
||||||
|
|
||||||
out := make(map[string]map[string]any, len(speakers))
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if sp == nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
view, ok := viewByID[sp.ID]
|
|
||||||
if !ok {
|
|
||||||
view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)}
|
|
||||||
}
|
|
||||||
var hp *nodedispatch.AgentHealthResult
|
|
||||||
if h, ok := healthByID[sp.ID]; ok {
|
|
||||||
hCopy := h
|
|
||||||
hp = &hCopy
|
|
||||||
}
|
|
||||||
out[sp.ID] = speakerLiveStatusJSON(sp, view, hp)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
@@ -2,12 +2,29 @@ package httpapi
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/nodedispatch"
|
"evobgp/internal/nodedispatch"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) {
|
||||||
|
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||||
|
unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil)
|
||||||
|
if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil {
|
||||||
|
t.Fatalf("unpolled=%#v", unpolled)
|
||||||
|
}
|
||||||
|
if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" {
|
||||||
|
t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"])
|
||||||
|
}
|
||||||
|
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||||
|
empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil)
|
||||||
|
if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 {
|
||||||
|
t.Fatalf("empty=%#v", empty)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCountBGPSessions(t *testing.T) {
|
func TestCountBGPSessions(t *testing.T) {
|
||||||
total, est := countBGPSessions([]birdfmt.BGPSession{
|
total, est := countBGPSessions([]birdfmt.BGPSession{
|
||||||
{Name: "p1", State: "Established"},
|
{Name: "p1", State: "Established"},
|
||||||
|
|||||||
@@ -0,0 +1,154 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
"evobgp/internal/telemetry"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
|
||||||
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
snapshots, err := s.store.ListTelemetry(ctx, tenant)
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
|
||||||
|
return views, statuses, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshotObservation uses control-plane receipt time, never the remote clock.
|
||||||
|
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
|
||||||
|
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
|
||||||
|
if !o.ReceivedAt.IsZero() {
|
||||||
|
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
|
||||||
|
}
|
||||||
|
if v.Source == "" {
|
||||||
|
v.Source = "none"
|
||||||
|
}
|
||||||
|
switch {
|
||||||
|
case invalid != "":
|
||||||
|
v.ErrorCode = invalid
|
||||||
|
case o.ReceivedAt.IsZero():
|
||||||
|
v.ErrorCode = "not_polled"
|
||||||
|
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
|
||||||
|
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
|
||||||
|
case o.Error != "":
|
||||||
|
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
|
||||||
|
if !o.LastSuccessAt.IsZero() {
|
||||||
|
v.Quality = "stale"
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
v.Quality = "ok"
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
|
||||||
|
byID := make(map[string]*store.Speaker, len(speakers))
|
||||||
|
for _, sp := range speakers {
|
||||||
|
if sp != nil {
|
||||||
|
byID[sp.ID] = sp
|
||||||
|
}
|
||||||
|
}
|
||||||
|
grouped := make(map[string][]store.TelemetrySnapshot)
|
||||||
|
for _, snapshot := range snapshots {
|
||||||
|
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
|
||||||
|
}
|
||||||
|
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||||
|
statuses := make(map[string]map[string]any, len(speakers))
|
||||||
|
appendView := func(id string, sp *store.Speaker) {
|
||||||
|
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
|
||||||
|
if sp == nil {
|
||||||
|
view.Label = "CP (local)"
|
||||||
|
}
|
||||||
|
var health *agentHealthPoll
|
||||||
|
rows := grouped[id]
|
||||||
|
// A restarted local collector may leave an expired authority behind.
|
||||||
|
// Only simultaneously current authorities conflict; never prefer one
|
||||||
|
// of two current BIRD nodes arbitrarily.
|
||||||
|
if len(rows) > 1 {
|
||||||
|
current := make([]store.TelemetrySnapshot, 0, len(rows))
|
||||||
|
for _, row := range rows {
|
||||||
|
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
|
||||||
|
current = append(current, row)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(current) > 0 {
|
||||||
|
rows = current
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(rows) > 1 {
|
||||||
|
view.Error = "ambiguous telemetry authorities"
|
||||||
|
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
|
||||||
|
} else if len(rows) == 1 {
|
||||||
|
row := rows[0]
|
||||||
|
invalid := ""
|
||||||
|
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
|
||||||
|
invalid = "configuration_changed"
|
||||||
|
}
|
||||||
|
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
|
||||||
|
view.ObservationOverride = &bgp
|
||||||
|
view.Polled = !row.BGP.ReceivedAt.IsZero()
|
||||||
|
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
|
||||||
|
if bgp.Quality == "ok" {
|
||||||
|
view.Sessions = row.BGP.Sessions
|
||||||
|
} else {
|
||||||
|
view.Error = bgp.ErrorCode
|
||||||
|
}
|
||||||
|
if row.Kind == "remote_agent" {
|
||||||
|
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
|
||||||
|
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
|
||||||
|
if h.Quality != "ok" {
|
||||||
|
result.Error = h.ErrorCode
|
||||||
|
}
|
||||||
|
if h.Quality == "ok" && !result.OK {
|
||||||
|
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
|
||||||
|
}
|
||||||
|
health = &agentHealthPoll{result: result, observationOverride: &h}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
views = append(views, view)
|
||||||
|
if sp != nil {
|
||||||
|
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, sp := range speakers {
|
||||||
|
if sp != nil {
|
||||||
|
appendView(sp.ID, sp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(grouped[""]) > 0 {
|
||||||
|
appendView("", nil)
|
||||||
|
}
|
||||||
|
return views, statuses
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
|
||||||
|
a, ok := authFromContext(r.Context())
|
||||||
|
if !ok {
|
||||||
|
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.requirePerm(w, a, "bgp:network:read") {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
|
||||||
|
if err != nil {
|
||||||
|
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type telemetryHTTPStore struct {
|
||||||
|
store.Backend
|
||||||
|
discoveryWrites int
|
||||||
|
snapshotErr error
|
||||||
|
refreshCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) {
|
||||||
|
s.discoveryWrites++
|
||||||
|
return s.Backend.UpsertPeerDiscovery(tenant, in)
|
||||||
|
}
|
||||||
|
func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) {
|
||||||
|
if s.snapshotErr != nil {
|
||||||
|
return nil, s.snapshotErr
|
||||||
|
}
|
||||||
|
return s.Backend.ListTelemetry(ctx, tenant)
|
||||||
|
}
|
||||||
|
func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) {
|
||||||
|
s.refreshCalls++
|
||||||
|
return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown)
|
||||||
|
}
|
||||||
|
func telemetryRequest(method, path, tenant string) *http.Request {
|
||||||
|
r := httptest.NewRequest(method, path, nil)
|
||||||
|
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"}))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) {
|
||||||
|
var calls atomic.Int64
|
||||||
|
agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
calls.Add(1)
|
||||||
|
_, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`))
|
||||||
|
}))
|
||||||
|
defer agent.Close()
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1")
|
||||||
|
mem := store.NewMemory()
|
||||||
|
mem.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := mem.DemoIDs()
|
||||||
|
sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Positive control: the legacy adapter reaches the fake configured agent.
|
||||||
|
if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK {
|
||||||
|
t.Fatal(result)
|
||||||
|
}
|
||||||
|
calls.Store(0)
|
||||||
|
st := &telemetryHTTPStore{Backend: mem}
|
||||||
|
server := &Server{store: st}
|
||||||
|
claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "")
|
||||||
|
if err != nil || len(claims) != 1 {
|
||||||
|
t.Fatalf("claim: %v %v", claims, err)
|
||||||
|
}
|
||||||
|
for i := 0; i < 20; i++ {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
path string
|
||||||
|
handler http.HandlerFunc
|
||||||
|
}{
|
||||||
|
{"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers},
|
||||||
|
{"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries},
|
||||||
|
} {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant))
|
||||||
|
if w.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant))
|
||||||
|
if w.Code != http.StatusAccepted {
|
||||||
|
t.Fatalf("refresh: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
want := `"requested":false`
|
||||||
|
if i == 0 {
|
||||||
|
want = `"requested":true`
|
||||||
|
}
|
||||||
|
if !strings.Contains(w.Body.String(), want) {
|
||||||
|
t.Fatalf("refresh coalescing: %s", w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if calls.Load() != 0 || st.discoveryWrites != 0 {
|
||||||
|
t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) {
|
||||||
|
mem := store.NewMemory()
|
||||||
|
mem.SeedDemo()
|
||||||
|
tenant, _, _, _, _ := mem.DemoIDs()
|
||||||
|
st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")}
|
||||||
|
server := &Server{store: st}
|
||||||
|
for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} {
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant))
|
||||||
|
if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {
|
||||||
|
t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil)
|
||||||
|
w := httptest.NewRecorder()
|
||||||
|
server.handleTelemetryRefresh(w, r)
|
||||||
|
if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 {
|
||||||
|
t.Fatal("unauthenticated refresh admitted")
|
||||||
|
}
|
||||||
|
r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT}))
|
||||||
|
w = httptest.NewRecorder()
|
||||||
|
server.handleTelemetryRefresh(w, r)
|
||||||
|
if w.Code != http.StatusForbidden || st.refreshCalls != 0 {
|
||||||
|
t.Fatal("unprivileged refresh admitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) {
|
||||||
|
s, err := New(Options{SeedDemo: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
s.StartBackground(ctx)
|
||||||
|
first := s.telemetryCollector
|
||||||
|
if first == nil {
|
||||||
|
t.Fatal("telemetry collector not started")
|
||||||
|
}
|
||||||
|
s.StartBackground(ctx)
|
||||||
|
if s.telemetryCollector != first {
|
||||||
|
t.Fatal("duplicate telemetry collector")
|
||||||
|
}
|
||||||
|
s.Close()
|
||||||
|
s.StartBackground(ctx)
|
||||||
|
if s.telemetryCollector != first {
|
||||||
|
t.Fatal("collector started after close")
|
||||||
|
}
|
||||||
|
s.Close()
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"evobgp/internal/birdfmt"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"}
|
||||||
|
observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"}
|
||||||
|
base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}}
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
mutate func(*store.TelemetrySnapshot)
|
||||||
|
bgp, health bool
|
||||||
|
quality string
|
||||||
|
}{
|
||||||
|
{"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"},
|
||||||
|
{"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"},
|
||||||
|
{"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"},
|
||||||
|
{"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"},
|
||||||
|
{"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"},
|
||||||
|
{"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
snapshot := base
|
||||||
|
tc.mutate(&snapshot)
|
||||||
|
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute)
|
||||||
|
status := statuses[sp.ID]
|
||||||
|
if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality {
|
||||||
|
t.Fatalf("status=%+v observation=%+v", status, views[0].observation())
|
||||||
|
}
|
||||||
|
if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) {
|
||||||
|
t.Fatal("stale BGP exposed live sessions")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||||
|
for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} {
|
||||||
|
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute)
|
||||||
|
if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true {
|
||||||
|
t.Fatalf("unexpected healthy master: %+v", statuses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||||
|
fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}}
|
||||||
|
old := fresh
|
||||||
|
old.LocalAuthority = "old"
|
||||||
|
old.BGP.ReceivedAt = now.Add(-2 * time.Minute)
|
||||||
|
_, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||||
|
if status[sp.ID]["bgp_poll_ok"] != true {
|
||||||
|
t.Fatal("expired authority hides current local snapshot")
|
||||||
|
}
|
||||||
|
old.BGP.ReceivedAt = now
|
||||||
|
_, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||||
|
if status[sp.ID]["bgp_poll_ok"] != false {
|
||||||
|
t.Fatal("two current local authorities are ambiguous")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) {
|
||||||
|
now := time.Now()
|
||||||
|
sp := &store.Speaker{ID: "replica", Role: "replica"}
|
||||||
|
row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}}
|
||||||
|
views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute)
|
||||||
|
observation := peerLiveObservation(views)
|
||||||
|
if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" {
|
||||||
|
t.Fatalf("changed config is not a failed current poll: %+v", observation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) {
|
||||||
|
now := time.Now().UTC()
|
||||||
|
old := now.Add(-30 * time.Second)
|
||||||
|
a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "")
|
||||||
|
b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "")
|
||||||
|
want := old.Add(5 * time.Minute).Format(time.RFC3339Nano)
|
||||||
|
if a.ExpiresAt == nil || *a.ExpiresAt != want {
|
||||||
|
t.Fatalf("expiry ignores configured interval: %+v", a)
|
||||||
|
}
|
||||||
|
views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}}
|
||||||
|
combined := peerLiveObservation(views)
|
||||||
|
if combined.ExpiresAt == nil || *combined.ExpiresAt != want {
|
||||||
|
t.Fatalf("aggregate extends oldest source: %+v", combined)
|
||||||
|
}
|
||||||
|
missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "")
|
||||||
|
if missing.ExpiresAt != nil {
|
||||||
|
t.Fatal("unobserved source has expiry")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid.
|
||||||
|
// An empty configuration deliberately treats every connection as direct.
|
||||||
|
func trustedProxyPrefixes() []netip.Prefix {
|
||||||
|
raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS"))
|
||||||
|
if raw == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
parts := strings.Split(raw, ",")
|
||||||
|
prefixes := make([]netip.Prefix, 0, len(parts))
|
||||||
|
for _, part := range parts {
|
||||||
|
prefix, err := netip.ParsePrefix(strings.TrimSpace(part))
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if prefix.Addr().Is4In6() {
|
||||||
|
if prefix.Bits() < 96 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
|
||||||
|
}
|
||||||
|
prefixes = append(prefixes, prefix.Masked())
|
||||||
|
}
|
||||||
|
return prefixes
|
||||||
|
}
|
||||||
|
|
||||||
|
func peerAddress(r *http.Request) (netip.Addr, bool) {
|
||||||
|
if r == nil {
|
||||||
|
return netip.Addr{}, false
|
||||||
|
}
|
||||||
|
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||||
|
if err != nil {
|
||||||
|
host = strings.TrimSpace(r.RemoteAddr)
|
||||||
|
}
|
||||||
|
ip, err := netip.ParseAddr(host)
|
||||||
|
if err != nil {
|
||||||
|
return netip.Addr{}, false
|
||||||
|
}
|
||||||
|
return ip.WithZone("").Unmap(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool {
|
||||||
|
for _, prefix := range prefixes {
|
||||||
|
if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string {
|
||||||
|
peer, ok := peerAddress(r)
|
||||||
|
if !ok {
|
||||||
|
if r == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(r.RemoteAddr)
|
||||||
|
}
|
||||||
|
if !isTrustedProxy(peer, prefixes) {
|
||||||
|
return peer.String()
|
||||||
|
}
|
||||||
|
values := r.Header.Values("X-Forwarded-For")
|
||||||
|
if len(values) != 1 || strings.TrimSpace(values[0]) == "" {
|
||||||
|
return peer.String()
|
||||||
|
}
|
||||||
|
hops := strings.Split(values[0], ",")
|
||||||
|
current := peer
|
||||||
|
for i := len(hops) - 1; i >= 0; i-- {
|
||||||
|
if !isTrustedProxy(current, prefixes) {
|
||||||
|
return current.String()
|
||||||
|
}
|
||||||
|
next, err := netip.ParseAddr(strings.TrimSpace(hops[i]))
|
||||||
|
if err != nil {
|
||||||
|
return peer.String()
|
||||||
|
}
|
||||||
|
if next.Zone() != "" {
|
||||||
|
return peer.String()
|
||||||
|
}
|
||||||
|
current = next.Unmap()
|
||||||
|
}
|
||||||
|
return current.String()
|
||||||
|
}
|
||||||
@@ -0,0 +1,129 @@
|
|||||||
|
package httpapi
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http/httptest"
|
||||||
|
"net/netip"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestClientIPTrustedProxyChain(t *testing.T) {
|
||||||
|
prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
remote string
|
||||||
|
xff string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"},
|
||||||
|
{"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"},
|
||||||
|
{"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"},
|
||||||
|
{"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"},
|
||||||
|
{"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"},
|
||||||
|
{"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"},
|
||||||
|
{"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||||
|
r.RemoteAddr = tt.remote
|
||||||
|
r.Header.Set("X-Forwarded-For", tt.xff)
|
||||||
|
if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want {
|
||||||
|
t.Fatalf("client IP = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestClientIPConfigurationFailsClosed(t *testing.T) {
|
||||||
|
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||||
|
r.RemoteAddr = "10.0.0.2:1234"
|
||||||
|
r.Header.Set("X-Forwarded-For", "203.0.113.9")
|
||||||
|
r.Header.Set("X-Real-IP", "203.0.113.8")
|
||||||
|
if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" {
|
||||||
|
t.Fatalf("direct client IP = %q", got)
|
||||||
|
}
|
||||||
|
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid")
|
||||||
|
if got := clientIP(r); got != "10.0.0.2" {
|
||||||
|
t.Fatalf("invalid CIDR client IP = %q", got)
|
||||||
|
}
|
||||||
|
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128")
|
||||||
|
if got := clientIP(r); got != "203.0.113.9" {
|
||||||
|
t.Fatalf("mapped CIDR client IP = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPublicControlPlaneURLTrust(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
|
||||||
|
r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil)
|
||||||
|
r.Header.Set("Origin", "https://evil.example")
|
||||||
|
r.Header.Set("X-Forwarded-Proto", "https")
|
||||||
|
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||||
|
r.RemoteAddr = "198.51.100.8:1234"
|
||||||
|
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||||
|
t.Fatalf("direct URL = %q", got)
|
||||||
|
}
|
||||||
|
r.RemoteAddr = "10.0.0.2:1234"
|
||||||
|
if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" {
|
||||||
|
t.Fatalf("proxied URL = %q", got)
|
||||||
|
}
|
||||||
|
if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" {
|
||||||
|
t.Fatalf("override URL = %q", got)
|
||||||
|
}
|
||||||
|
for _, bad := range []struct{ proto, host string }{
|
||||||
|
{"javascript", "bgp.example.test"},
|
||||||
|
{"https,http", "bgp.example.test"},
|
||||||
|
{"https", "evil.example/path"},
|
||||||
|
{"https", "[email protected]"},
|
||||||
|
{"https", "evil.example,bgp.example.test"},
|
||||||
|
{"https", "bgp.example.test:99999"},
|
||||||
|
{"https", "bgp.example.test:"},
|
||||||
|
} {
|
||||||
|
r.Header.Set("X-Forwarded-Proto", bad.proto)
|
||||||
|
r.Header.Set("X-Forwarded-Host", bad.host)
|
||||||
|
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||||
|
t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r.Header.Set("X-Forwarded-Proto", "https")
|
||||||
|
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||||
|
r.Header.Add("X-Forwarded-Host", "evil.example")
|
||||||
|
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||||
|
t.Fatalf("duplicate forwarded host = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthRateLimiterManyIdentities(t *testing.T) {
|
||||||
|
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32}
|
||||||
|
now := time.Unix(1000, 0)
|
||||||
|
for i := 0; i < 1000; i++ {
|
||||||
|
allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now)
|
||||||
|
if allowed != (i < l.maxClients) {
|
||||||
|
t.Fatalf("identity %d allowed = %t", i, allowed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) {
|
||||||
|
t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) {
|
||||||
|
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2}
|
||||||
|
now := time.Unix(1000, 0)
|
||||||
|
if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) {
|
||||||
|
t.Fatal("initial requests rejected")
|
||||||
|
}
|
||||||
|
if l.allowAt("a", now) || l.allowAt("c", now) {
|
||||||
|
t.Fatal("active limit or capacity bypassed")
|
||||||
|
}
|
||||||
|
if len(l.hits) != 2 {
|
||||||
|
t.Fatalf("map size = %d", len(l.hits))
|
||||||
|
}
|
||||||
|
if !l.allowAt("c", now.Add(time.Minute)) {
|
||||||
|
t.Fatal("expired entries did not release capacity")
|
||||||
|
}
|
||||||
|
if len(l.hits) != 1 {
|
||||||
|
t.Fatalf("expired entries remained: %d", len(l.hits))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrResponseTooLarge means the complete decoded HTTP response exceeded its budget.
|
||||||
|
var ErrResponseTooLarge = errors.New("httpclient: response body exceeds limit")
|
||||||
|
|
||||||
|
// ReadResponseBody reads a complete bounded body, including one extra byte to
|
||||||
|
// distinguish an exact limit from truncation. Standard Transport gzip decoding
|
||||||
|
// occurs before this reader, so the limit covers decompressed data as well.
|
||||||
|
func ReadResponseBody(resp *http.Response, limit int64) ([]byte, error) {
|
||||||
|
if limit <= 0 || limit == int64(^uint64(0)>>1) {
|
||||||
|
return nil, fmt.Errorf("httpclient: invalid response limit")
|
||||||
|
}
|
||||||
|
if resp.ContentLength > limit {
|
||||||
|
return nil, ErrResponseTooLarge
|
||||||
|
}
|
||||||
|
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if int64(len(body)) > limit {
|
||||||
|
return nil, ErrResponseTooLarge
|
||||||
|
}
|
||||||
|
return body, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestReadResponseBodyDoesNotAcceptValidPrefixTruncation(t *testing.T) {
|
||||||
|
for _, length := range []int64{-1, 9} {
|
||||||
|
resp := &http.Response{ContentLength: length, Body: io.NopCloser(strings.NewReader("{} "))}
|
||||||
|
if _, err := ReadResponseBody(resp, 8); !errors.Is(err, ErrResponseTooLarge) {
|
||||||
|
t.Fatalf("length=%d got %v", length, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resp := &http.Response{ContentLength: -1, Body: io.NopCloser(strings.NewReader("{} "))}
|
||||||
|
if b, err := ReadResponseBody(resp, 8); err != nil || len(b) != 8 {
|
||||||
|
t.Fatalf("exact limit failed: %q %v", b, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,8 +1,13 @@
|
|||||||
package httpclient
|
package httpclient
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/observability"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -16,14 +21,77 @@ type hostBreaker struct {
|
|||||||
openUntil time.Time
|
openUntil time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
var hostBreakers sync.Map // string -> *hostBreaker
|
// ErrBreakerCapacity rejects new upstream identities when the bounded cache is full.
|
||||||
|
var ErrBreakerCapacity = errors.New("httpclient: upstream circuit capacity exhausted")
|
||||||
|
|
||||||
func breakerForHost(host string) *hostBreaker {
|
type breakerEntry struct {
|
||||||
|
breaker *hostBreaker
|
||||||
|
lastUsed time.Time
|
||||||
|
active int
|
||||||
|
}
|
||||||
|
|
||||||
|
type breakerCache struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
entries map[string]*breakerEntry
|
||||||
|
maximum int
|
||||||
|
idle time.Duration
|
||||||
|
nextSweep time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
var hostBreakers = breakerCache{entries: make(map[string]*breakerEntry), maximum: 1024, idle: 10 * time.Minute}
|
||||||
|
|
||||||
|
func (c *breakerCache) sweep(now time.Time) {
|
||||||
|
for host, e := range c.entries {
|
||||||
|
if e.active != 0 || now.Before(e.lastUsed.Add(c.idle)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
e.breaker.mu.Lock()
|
||||||
|
open := now.Before(e.breaker.openUntil)
|
||||||
|
e.breaker.mu.Unlock()
|
||||||
|
if !open {
|
||||||
|
delete(c.entries, host)
|
||||||
|
observability.RemoveUpstreamBreaker(host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.nextSweep = now.Add(time.Second)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *breakerCache) acquire(host string, now time.Time) (*hostBreaker, string, func(), error) {
|
||||||
|
if len(host) > 254 {
|
||||||
|
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
|
||||||
|
}
|
||||||
|
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||||
|
if len(host) > 253 {
|
||||||
|
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
|
||||||
|
}
|
||||||
if host == "" {
|
if host == "" {
|
||||||
host = "_"
|
host = "_"
|
||||||
}
|
}
|
||||||
v, _ := hostBreakers.LoadOrStore(host, &hostBreaker{})
|
c.mu.Lock()
|
||||||
return v.(*hostBreaker)
|
defer c.mu.Unlock()
|
||||||
|
if !now.Before(c.nextSweep) {
|
||||||
|
c.sweep(now)
|
||||||
|
}
|
||||||
|
e := c.entries[host]
|
||||||
|
if e == nil {
|
||||||
|
if len(c.entries) >= c.maximum {
|
||||||
|
return nil, host, nil, ErrBreakerCapacity
|
||||||
|
}
|
||||||
|
e = &breakerEntry{breaker: &hostBreaker{}}
|
||||||
|
c.entries[host] = e
|
||||||
|
}
|
||||||
|
e.active++
|
||||||
|
e.lastUsed = now
|
||||||
|
var once sync.Once
|
||||||
|
release := func() {
|
||||||
|
once.Do(func() {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
e.active--
|
||||||
|
e.lastUsed = time.Now()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return e.breaker, host, release, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *hostBreaker) allow() bool {
|
func (b *hostBreaker) allow() bool {
|
||||||
@@ -51,23 +119,31 @@ func (b *hostBreaker) recordFailure() {
|
|||||||
|
|
||||||
// SnapshotBreakers returns whether each known host breaker is currently open.
|
// SnapshotBreakers returns whether each known host breaker is currently open.
|
||||||
func SnapshotBreakers() map[string]bool {
|
func SnapshotBreakers() map[string]bool {
|
||||||
|
hostBreakers.mu.Lock()
|
||||||
|
defer hostBreakers.mu.Unlock()
|
||||||
|
now := time.Now()
|
||||||
|
hostBreakers.sweep(now)
|
||||||
out := map[string]bool{}
|
out := map[string]bool{}
|
||||||
hostBreakers.Range(func(key, value any) bool {
|
for host, e := range hostBreakers.entries {
|
||||||
host, _ := key.(string)
|
b := e.breaker
|
||||||
b, _ := value.(*hostBreaker)
|
|
||||||
if b == nil {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
open := time.Now().Before(b.openUntil)
|
open := time.Now().Before(b.openUntil)
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
out[host] = open
|
out[host] = open
|
||||||
return true
|
}
|
||||||
})
|
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// ResetHostBreakers clears all circuit breakers (tests only).
|
// ResetHostBreakers clears idle circuit breakers (tests only). In-flight
|
||||||
|
// references are retained so their completions cannot recreate orphan metrics.
|
||||||
func ResetHostBreakers() {
|
func ResetHostBreakers() {
|
||||||
hostBreakers = sync.Map{}
|
hostBreakers.mu.Lock()
|
||||||
|
defer hostBreakers.mu.Unlock()
|
||||||
|
for host, e := range hostBreakers.entries {
|
||||||
|
if e.active == 0 {
|
||||||
|
delete(hostBreakers.entries, host)
|
||||||
|
observability.RemoveUpstreamBreaker(host)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
hostBreakers.nextSweep = time.Time{}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/observability"
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBreakerCanonicalHostAndActiveRetention(t *testing.T) {
|
||||||
|
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
|
||||||
|
now := time.Now()
|
||||||
|
first, key, release, err := c.acquire("Cache-Test.Example.", now)
|
||||||
|
if err != nil || key != "cache-test.example" {
|
||||||
|
t.Fatalf("canonical key=%s err=%v", key, err)
|
||||||
|
}
|
||||||
|
second, _, release2, err := c.acquire("cache-test.example", now)
|
||||||
|
if err != nil || first != second || len(c.entries) != 1 {
|
||||||
|
t.Fatal("alias bypassed shared breaker")
|
||||||
|
}
|
||||||
|
observability.SetUpstreamBreakerOpen(key, true)
|
||||||
|
first.openUntil = now.Add(3 * time.Minute)
|
||||||
|
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute)); !errors.Is(err, ErrBreakerCapacity) {
|
||||||
|
t.Fatalf("active entry evicted: %v", err)
|
||||||
|
}
|
||||||
|
release()
|
||||||
|
release()
|
||||||
|
release2()
|
||||||
|
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute+2*time.Second)); !errors.Is(err, ErrBreakerCapacity) {
|
||||||
|
t.Fatalf("open breaker evicted: %v", err)
|
||||||
|
}
|
||||||
|
_, _, done, err := c.acquire("other.example", now.Add(4*time.Minute))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer done()
|
||||||
|
if len(c.entries) != 1 || c.entries[key] != nil {
|
||||||
|
t.Fatal("idle closed entry retained")
|
||||||
|
}
|
||||||
|
families, err := prometheus.DefaultGatherer.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, family := range families {
|
||||||
|
if strings.HasSuffix(family.GetName(), "upstream_breaker_open") {
|
||||||
|
for _, metric := range family.Metric {
|
||||||
|
for _, label := range metric.Label {
|
||||||
|
if label.GetName() == "host" && label.GetValue() == key {
|
||||||
|
t.Fatal("evicted host metric retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type breakerTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f breakerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestBreakerCapacityRejectsBeforeDial(t *testing.T) {
|
||||||
|
ResetHostBreakers()
|
||||||
|
var releases []func()
|
||||||
|
defer func() {
|
||||||
|
for _, release := range releases {
|
||||||
|
release()
|
||||||
|
}
|
||||||
|
ResetHostBreakers()
|
||||||
|
}()
|
||||||
|
for i := 0; i < hostBreakers.maximum; i++ {
|
||||||
|
_, _, release, err := hostBreakers.acquire(fmt.Sprintf("capacity-%d.example", i), time.Now())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
releases = append(releases, release)
|
||||||
|
}
|
||||||
|
called := false
|
||||||
|
hc := &http.Client{Transport: breakerTransport(func(*http.Request) (*http.Response, error) { called = true; return nil, errors.New("unexpected dial") })}
|
||||||
|
req, _ := http.NewRequest(http.MethodGet, "https://overflow.example/", nil)
|
||||||
|
_, err := DoWithBreaker(context.Background(), hc, req, 1)
|
||||||
|
if !errors.Is(err, ErrBreakerCapacity) || called {
|
||||||
|
t.Fatalf("capacity err=%v dial=%t", err, called)
|
||||||
|
}
|
||||||
|
if len(SnapshotBreakers()) != hostBreakers.maximum {
|
||||||
|
t.Fatal("cache capacity changed")
|
||||||
|
}
|
||||||
|
ResetHostBreakers()
|
||||||
|
if len(SnapshotBreakers()) != hostBreakers.maximum {
|
||||||
|
t.Fatal("reset discarded in-flight references")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBreakerRejectsOversizedIdentity(t *testing.T) {
|
||||||
|
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
|
||||||
|
if _, _, _, err := c.acquire(strings.Repeat("x", 254), time.Now()); err == nil || len(c.entries) != 0 {
|
||||||
|
t.Fatal("oversized identity retained")
|
||||||
|
}
|
||||||
|
name := strings.Repeat("a", 253)
|
||||||
|
first, key, release, err := c.acquire(name, time.Now())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
second, alias, release2, err := c.acquire(strings.ToUpper(name)+".", time.Now())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer release2()
|
||||||
|
if first != second || key != alias {
|
||||||
|
t.Fatal("maximum length trailing-dot alias differs")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -54,7 +54,9 @@ func DoWithRetry(ctx context.Context, hc *http.Client, req *http.Request, maxAtt
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if resp.StatusCode >= 500 {
|
if resp.StatusCode >= 500 {
|
||||||
_, _ = io.Copy(io.Discard, resp.Body)
|
// Draining is only an optimization for connection reuse, never an
|
||||||
|
// invitation to consume an unbounded error stream before retrying.
|
||||||
|
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status)
|
lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status)
|
||||||
continue
|
continue
|
||||||
@@ -72,8 +74,11 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
|
|||||||
if req == nil || req.URL == nil {
|
if req == nil || req.URL == nil {
|
||||||
return nil, fmt.Errorf("httpclient: nil request")
|
return nil, fmt.Errorf("httpclient: nil request")
|
||||||
}
|
}
|
||||||
host := req.URL.Hostname()
|
br, host, release, err := hostBreakers.acquire(req.URL.Hostname(), time.Now())
|
||||||
br := breakerForHost(host)
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
if !br.allow() {
|
if !br.allow() {
|
||||||
observability.SetUpstreamBreakerOpen(host, true)
|
observability.SetUpstreamBreakerOpen(host, true)
|
||||||
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
|
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
|
||||||
|
|||||||
@@ -0,0 +1,44 @@
|
|||||||
|
package httpclient
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
type endlessRetryBody struct {
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *endlessRetryBody) Read(p []byte) (int, error) {
|
||||||
|
for i := range p {
|
||||||
|
p[i] = 'x'
|
||||||
|
}
|
||||||
|
b.read += len(p)
|
||||||
|
return len(p), nil
|
||||||
|
}
|
||||||
|
func (b *endlessRetryBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
type retryTransport func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f retryTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestRetryDrainBoundsEndlessErrorBody(t *testing.T) {
|
||||||
|
b := &endlessRetryBody{}
|
||||||
|
c := &http.Client{Transport: retryTransport(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: 503, Status: "503 Service Unavailable", Body: b, Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
req, err := http.NewRequest(http.MethodGet, "http://agent.example/bundle", nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err = DoWithRetry(context.Background(), c, req, 1); err == nil {
|
||||||
|
t.Fatal("upstream failure accepted")
|
||||||
|
}
|
||||||
|
if b.read != 64<<10 || !b.closed {
|
||||||
|
t.Fatalf("retry consumed %d bytes, closed=%v", b.read, b.closed)
|
||||||
|
}
|
||||||
|
var _ io.ReadCloser = b
|
||||||
|
}
|
||||||
+106
-8
@@ -1,7 +1,10 @@
|
|||||||
package importer
|
package importer
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
"encoding/csv"
|
"encoding/csv"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -10,6 +13,18 @@ import (
|
|||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// MaxCSVBytes is the complete request limit, including the optional header.
|
||||||
|
const MaxCSVBytes = 8 << 20
|
||||||
|
|
||||||
|
// MaxCSVRows bounds database work for a single import, excluding its header.
|
||||||
|
const MaxCSVRows = 10000
|
||||||
|
|
||||||
|
// MaxCSVFields bounds CSV parser allocations for delimiter-heavy records.
|
||||||
|
const MaxCSVFields = 64
|
||||||
|
|
||||||
|
// ErrCSVTooLarge means the entire import was rejected before creating entries.
|
||||||
|
var ErrCSVTooLarge = errors.New("importer: csv exceeds byte, row or field limit")
|
||||||
|
|
||||||
// Result of importing module entries from CSV.
|
// Result of importing module entries from CSV.
|
||||||
type Result struct {
|
type Result struct {
|
||||||
Imported int
|
Imported int
|
||||||
@@ -17,19 +32,51 @@ type Result struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules.
|
// ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules.
|
||||||
func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
func ImportModuleEntriesCSV(ctx context.Context, st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
mod, err := st.GetModule(tenantID, moduleID)
|
mod, err := st.GetModule(tenantID, moduleID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
cr := csv.NewReader(io.LimitReader(body, 8<<20))
|
// Read one byte beyond the cap: a valid prefix must never be imported as if
|
||||||
|
// it were the complete request. Check all resource limits before mutations.
|
||||||
|
raw, err := io.ReadAll(io.LimitReader(body, MaxCSVBytes+1))
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("importer: read csv: %w", err)
|
||||||
|
}
|
||||||
|
if len(raw) > MaxCSVBytes {
|
||||||
|
return nil, ErrCSVTooLarge
|
||||||
|
}
|
||||||
|
if err := checkCSVFields(ctx, raw); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cr := csv.NewReader(bytes.NewReader(raw))
|
||||||
cr.TrimLeadingSpace = true
|
cr.TrimLeadingSpace = true
|
||||||
cr.FieldsPerRecord = -1
|
cr.FieldsPerRecord = -1
|
||||||
rows, err := cr.ReadAll()
|
rows := make([][]string, 0)
|
||||||
|
headerRows := 0
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
record, err := cr.Read()
|
||||||
|
if err == io.EOF {
|
||||||
|
break
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
||||||
}
|
}
|
||||||
|
if len(rows) == 0 && isCSVHeader(record) {
|
||||||
|
headerRows = 1
|
||||||
|
}
|
||||||
|
if len(rows)-headerRows >= MaxCSVRows {
|
||||||
|
return nil, ErrCSVTooLarge
|
||||||
|
}
|
||||||
|
rows = append(rows, record)
|
||||||
|
}
|
||||||
if len(rows) == 0 {
|
if len(rows) == 0 {
|
||||||
return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput)
|
return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput)
|
||||||
}
|
}
|
||||||
@@ -63,18 +110,17 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
|||||||
}
|
}
|
||||||
|
|
||||||
start := 0
|
start := 0
|
||||||
if len(rows[0]) >= 2 {
|
if isCSVHeader(rows[0]) {
|
||||||
key := strings.ToLower(strings.TrimSpace(rows[0][0]))
|
|
||||||
switch key {
|
|
||||||
case "asn", "domain", "iprange":
|
|
||||||
start = 1
|
start = 1
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
imported := 0
|
imported := 0
|
||||||
switch mod.Type {
|
switch mod.Type {
|
||||||
case "AS_PREFIXES":
|
case "AS_PREFIXES":
|
||||||
for i := start; i < len(rows); i++ {
|
for i := start; i < len(rows); i++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
rec := rows[i]
|
rec := rows[i]
|
||||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||||
continue
|
continue
|
||||||
@@ -97,6 +143,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
|||||||
}
|
}
|
||||||
case "DOMAINS":
|
case "DOMAINS":
|
||||||
for i := start; i < len(rows); i++ {
|
for i := start; i < len(rows); i++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
rec := rows[i]
|
rec := rows[i]
|
||||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||||
continue
|
continue
|
||||||
@@ -119,6 +168,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
|||||||
}
|
}
|
||||||
case "IP_RANGES":
|
case "IP_RANGES":
|
||||||
for i := start; i < len(rows); i++ {
|
for i := start; i < len(rows); i++ {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
rec := rows[i]
|
rec := rows[i]
|
||||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||||
continue
|
continue
|
||||||
@@ -145,3 +197,49 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
|||||||
|
|
||||||
return &Result{Imported: imported, ModuleType: mod.Type}, nil
|
return &Result{Imported: imported, ModuleType: mod.Type}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func isCSVHeader(record []string) bool {
|
||||||
|
if len(record) < 2 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
switch strings.ToLower(strings.TrimSpace(record[0])) {
|
||||||
|
case "asn", "domain", "iprange":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkCSVFields runs before encoding/csv allocates a string/index for each field.
|
||||||
|
// Quoted commas, escaped quotes and multiline fields don't start new fields.
|
||||||
|
// Syntax validation remains the responsibility of encoding/csv below.
|
||||||
|
func checkCSVFields(ctx context.Context, raw []byte) error {
|
||||||
|
quoted, fields := false, 1
|
||||||
|
for i := 0; i < len(raw); i++ {
|
||||||
|
if i%4096 == 0 {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
switch raw[i] {
|
||||||
|
case '"':
|
||||||
|
if quoted && i+1 < len(raw) && raw[i+1] == '"' {
|
||||||
|
i++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
quoted = !quoted
|
||||||
|
case ',':
|
||||||
|
if !quoted {
|
||||||
|
fields++
|
||||||
|
if fields > MaxCSVFields {
|
||||||
|
return ErrCSVTooLarge
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case '\n':
|
||||||
|
if !quoted {
|
||||||
|
fields = 1
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
package importer
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type importBackend struct {
|
||||||
|
store.Backend
|
||||||
|
created int
|
||||||
|
onCreate func()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *importBackend) GetModule(string, string) (*store.Module, error) {
|
||||||
|
return &store.Module{Type: "AS_PREFIXES"}, nil
|
||||||
|
}
|
||||||
|
func (s *importBackend) ListCommunities(string) ([]*store.Community, error) { return nil, nil }
|
||||||
|
func (s *importBackend) CreateASEntry(_, _ string, e *store.ASEntry) (*store.ASEntry, error) {
|
||||||
|
s.created++
|
||||||
|
if s.onCreate != nil {
|
||||||
|
s.onCreate()
|
||||||
|
}
|
||||||
|
return e, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVByteLimitRejectsValidTruncatedPrefix(t *testing.T) {
|
||||||
|
header := "asn,community\n"
|
||||||
|
for _, extra := range []int{0, 1} {
|
||||||
|
s := &importBackend{}
|
||||||
|
body := header + strings.Repeat("\n", MaxCSVBytes-len(header)) + strings.Repeat("\n", extra)
|
||||||
|
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(body))
|
||||||
|
if extra == 0 && err != nil {
|
||||||
|
t.Fatalf("exact limit: %v", err)
|
||||||
|
}
|
||||||
|
if extra == 1 && !errors.Is(err, ErrCSVTooLarge) {
|
||||||
|
t.Fatalf("limit+1: %v", err)
|
||||||
|
}
|
||||||
|
if s.created != 0 {
|
||||||
|
t.Fatal("oversized import created entries")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVRowLimitCheckedBeforeWrites(t *testing.T) {
|
||||||
|
for _, header := range []string{"", "asn,community\n"} {
|
||||||
|
for _, extra := range []int{0, 1} {
|
||||||
|
s := &importBackend{}
|
||||||
|
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(header+strings.Repeat("64512,\n", MaxCSVRows+extra)))
|
||||||
|
if extra == 0 && (err != nil || s.created != MaxCSVRows) {
|
||||||
|
t.Fatalf("exact rows: created=%d err=%v", s.created, err)
|
||||||
|
}
|
||||||
|
if extra == 1 && (!errors.Is(err, ErrCSVTooLarge) || s.created != 0) {
|
||||||
|
t.Fatalf("rows+1: created=%d err=%v", s.created, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVCancellationStopsFurtherWrites(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
s := &importBackend{onCreate: cancel}
|
||||||
|
_, err := ImportModuleEntriesCSV(ctx, s, "tenant", "module", strings.NewReader("asn,community\n64512,\n64513,\n"))
|
||||||
|
if !errors.Is(err, context.Canceled) || s.created != 1 {
|
||||||
|
t.Fatalf("created=%d err=%v", s.created, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCSVFieldLimitAndQuotedFields(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, input string
|
||||||
|
oversized bool
|
||||||
|
}{
|
||||||
|
{"exact", "64512," + strings.Repeat(",", MaxCSVFields-2) + "\n", false},
|
||||||
|
{"over", "64512," + strings.Repeat(",", MaxCSVFields-1) + "\n", true},
|
||||||
|
{"many_fields", strings.Repeat(",", 1<<20) + "\n", true},
|
||||||
|
{"quoted_extra", "64512,,\"" + strings.Repeat(",", 128) + "\nquoted \"\"text\"\"\"\n", false},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
s := &importBackend{}
|
||||||
|
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(tc.input))
|
||||||
|
if tc.oversized {
|
||||||
|
if !errors.Is(err, ErrCSVTooLarge) || s.created != 0 {
|
||||||
|
t.Fatalf("created=%d err=%v", s.created, err)
|
||||||
|
}
|
||||||
|
} else if err != nil || s.created != 1 {
|
||||||
|
t.Fatalf("created=%d err=%v", s.created, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,231 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
type deployRoundTrip func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f deployRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestDeployApplyConfirmation(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, response, confirmation, status string
|
||||||
|
dispatch bool
|
||||||
|
stale bool
|
||||||
|
wantApplied bool
|
||||||
|
}{
|
||||||
|
{name: "dispatch disabled", confirmation: "unconfirmed", status: StatusSucceeded},
|
||||||
|
{name: "bare success", dispatch: true, response: `{"ok":true}`, confirmation: "unconfirmed", status: StatusSucceeded},
|
||||||
|
{name: "matching ACK", dispatch: true, response: "matching", confirmation: "agent_reported", status: StatusSucceeded, wantApplied: true},
|
||||||
|
{name: "mismatched ACK", dispatch: true, response: `{"ok":true,"applied_revision_id":"other"}`, confirmation: "unconfirmed", status: StatusFailed},
|
||||||
|
{name: "stale desired", dispatch: true, response: "matching", confirmation: "unconfirmed", status: StatusFailed, stale: true},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||||
|
if tc.dispatch {
|
||||||
|
// A remote-only job must not touch the control plane BIRD tree.
|
||||||
|
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", t.TempDir())
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||||
|
} else {
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||||
|
}
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, module, previous, speakerID := m.DemoIDs()
|
||||||
|
if err := m.SetLastAppliedRevision(tenant, speakerID, previous); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
revision := uuid.NewString()
|
||||||
|
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newer := uuid.NewString()
|
||||||
|
if err := m.CreateRenderRevision(context.Background(), newer, tenant, module, &revision, "newer-hash", map[string]string{"bird.conf": "# newer\n"}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, speakerID, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||||
|
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" || r.Header.Get("Authorization") != "Bearer secret" {
|
||||||
|
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
|
||||||
|
}
|
||||||
|
var body map[string]string
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["revision_id"] != revision {
|
||||||
|
t.Errorf("legacy sync body: %#v, %v", body, err)
|
||||||
|
}
|
||||||
|
if tc.stale {
|
||||||
|
if err := m.PublishRevisionForSpeaker(speakerID, newer); err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
response := tc.response
|
||||||
|
if response == "matching" {
|
||||||
|
response = `{"ok":true,"applied_revision_id":"` + revision + `"}`
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision, "speaker_id": speakerID}}
|
||||||
|
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||||
|
if j.Status != tc.status {
|
||||||
|
t.Fatalf("job status = %s, want %s; error=%v", j.Status, tc.status, j.Error)
|
||||||
|
}
|
||||||
|
sp, err := m.GetSpeaker(tenant, speakerID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := previous
|
||||||
|
if tc.wantApplied {
|
||||||
|
want = revision
|
||||||
|
}
|
||||||
|
if sp.LastAppliedRevisionID == nil || *sp.LastAppliedRevisionID != want {
|
||||||
|
t.Fatalf("applied = %v, want %s", sp.LastAppliedRevisionID, want)
|
||||||
|
}
|
||||||
|
published, _, err := m.LatestPublishedRevision(speakerID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantPublished := revision
|
||||||
|
if tc.stale {
|
||||||
|
wantPublished = newer
|
||||||
|
}
|
||||||
|
if published != wantPublished {
|
||||||
|
t.Fatalf("published = %s, want %s", published, wantPublished)
|
||||||
|
}
|
||||||
|
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||||
|
if len(results) != 1 || results[0].Confirmation != tc.confirmation {
|
||||||
|
t.Fatalf("results = %+v", results)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeployApplyPartialResult(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, module, previous, first := m.DemoIDs()
|
||||||
|
revision := uuid.NewString()
|
||||||
|
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
goodMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "good.example.test", AgentSecret: "secret"})
|
||||||
|
if _, err := m.UpdateSpeaker(tenant, first, &store.SpeakerPatch{MetaJSON: &goodMeta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
badMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "bad.example.test", AgentSecret: "secret"})
|
||||||
|
second, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: badMeta})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||||
|
response := `{"ok":true,"applied_revision_id":"` + revision + `"}`
|
||||||
|
if strings.HasPrefix(r.URL.Host, "bad.") {
|
||||||
|
response = `{"ok":false,"error":"apply failed"}`
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
|
||||||
|
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||||
|
if j.Status != StatusFailed {
|
||||||
|
t.Fatalf("job status = %s, want failed", j.Status)
|
||||||
|
}
|
||||||
|
summary := j.Meta["apply_summary"].(map[string]any)
|
||||||
|
if summary["applied_count"] != 1 || summary["failed_count"] != 1 || summary["published_count"] != 2 {
|
||||||
|
t.Fatalf("partial summary = %#v", summary)
|
||||||
|
}
|
||||||
|
firstSp, _ := m.GetSpeaker(tenant, first)
|
||||||
|
secondSp, _ := m.GetSpeaker(tenant, second.ID)
|
||||||
|
if firstSp.LastAppliedRevisionID == nil || *firstSp.LastAppliedRevisionID != revision || secondSp.LastAppliedRevisionID != nil {
|
||||||
|
t.Fatalf("partial applied state: first=%v second=%v", firstSp.LastAppliedRevisionID, secondSp.LastAppliedRevisionID)
|
||||||
|
}
|
||||||
|
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||||
|
got := map[string]bool{}
|
||||||
|
for _, res := range results {
|
||||||
|
got[res.SpeakerID] = true
|
||||||
|
}
|
||||||
|
if len(results) != 2 || !got[first] || !got[second.ID] {
|
||||||
|
t.Fatalf("partial results=%+v", results)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeployApplyCancellationStopsFurtherPublication(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, module, previous, seeded := m.DemoIDs()
|
||||||
|
if err := m.DeleteSpeaker(tenant, seeded); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
revision := uuid.NewString()
|
||||||
|
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||||
|
speakerCount := speakerWakeLimit + 2
|
||||||
|
for range speakerCount {
|
||||||
|
if _, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: meta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
speakers := m.ListSpeakersForTenant(tenant)
|
||||||
|
if len(speakers) != speakerCount {
|
||||||
|
t.Fatalf("speakers=%d, want %d", len(speakers), speakerCount)
|
||||||
|
}
|
||||||
|
var j *Job
|
||||||
|
var calls atomic.Int32
|
||||||
|
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||||
|
// Cancel only once the in-flight wake limit is busy, so later speakers stay unpublished.
|
||||||
|
if calls.Add(1) == int32(speakerWakeLimit) {
|
||||||
|
j.RequestCancel()
|
||||||
|
}
|
||||||
|
<-r.Context().Done()
|
||||||
|
return nil, r.Context().Err()
|
||||||
|
})}
|
||||||
|
j = &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
|
||||||
|
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||||
|
if j.Status != StatusCancelled || int(calls.Load()) != speakerWakeLimit {
|
||||||
|
t.Fatalf("cancelled job status=%s dispatch calls=%d", j.Status, calls.Load())
|
||||||
|
}
|
||||||
|
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||||
|
gotResult := map[string]bool{}
|
||||||
|
for _, res := range results {
|
||||||
|
gotResult[res.SpeakerID] = res.Status == "cancelled"
|
||||||
|
}
|
||||||
|
if len(results) != speakerWakeLimit {
|
||||||
|
t.Fatalf("cancelled results=%+v", results)
|
||||||
|
}
|
||||||
|
for i, sp := range speakers {
|
||||||
|
got, _, err := m.LatestPublishedRevision(sp.ID)
|
||||||
|
if i < speakerWakeLimit {
|
||||||
|
if err != nil || got != revision || !gotResult[sp.ID] {
|
||||||
|
t.Fatalf("speaker %s published=%s err=%v result=%v", sp.ID, got, err, gotResult[sp.ID])
|
||||||
|
}
|
||||||
|
if sp.LastAppliedRevisionID != nil {
|
||||||
|
t.Fatalf("cancelled speaker %s marked applied", sp.ID)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err == nil {
|
||||||
|
t.Fatalf("speaker %s published %s after cancellation", sp.ID, got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if j.Meta["apply_summary"].(map[string]any)["published_count"] != speakerWakeLimit {
|
||||||
|
t.Fatalf("cancelled summary = %#v", j.Meta["apply_summary"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/logging"
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func deploymentFenceForJob(j *Job) store.JobFence {
|
||||||
|
generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
|
||||||
|
round, _ := strconv.Atoi(j.metaString("reconcile_round"))
|
||||||
|
return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
|
||||||
|
}
|
||||||
|
|
||||||
|
func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
|
||||||
|
for {
|
||||||
|
d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||||
|
if !errors.Is(err, store.ErrDeploymentBusy) {
|
||||||
|
return d, err
|
||||||
|
}
|
||||||
|
timer := time.NewTimer(time.Second)
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
timer.Stop()
|
||||||
|
return store.DeploymentSpeaker{}, ctx.Err()
|
||||||
|
case <-timer.C:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReconcileDeployments schedules at most three recovery rounds for current desired
|
||||||
|
// generations. Stable keys coalesce API replicas; the worker CAS checks generation
|
||||||
|
// again before publishing, and preserves the original desired request sequence.
|
||||||
|
func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
|
||||||
|
rows, err := st.ListPendingDeployments(ctx, 100)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, d := range rows {
|
||||||
|
key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
|
||||||
|
_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
|
||||||
|
// ingest/deploy/scheduler processes must not start this job-producing loop.
|
||||||
|
func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
|
||||||
|
if r == nil || r.queue == nil || st == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.durableMu.Lock()
|
||||||
|
if r.durableStop {
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.durableLoopWG.Add(1)
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
go func() {
|
||||||
|
defer r.durableLoopWG.Done()
|
||||||
|
ticker := time.NewTicker(10 * time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
}
|
||||||
|
r.durableMu.Lock()
|
||||||
|
stopped := r.durableStop
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
if stopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
|
||||||
|
err := ObserveDeployments(qctx, st, nil)
|
||||||
|
if err == nil {
|
||||||
|
err = ReconcileDeployments(qctx, r, st)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if err != nil && ctx.Err() == nil {
|
||||||
|
logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
// ObserveDeployments samples compatible health before reconciling drift. The CP
|
||||||
|
// receipt time and optional agent source time are stored separately; missing
|
||||||
|
// legacy revision fields never manufacture an applied/verified confirmation.
|
||||||
|
func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
|
||||||
|
rows, err := st.ListDeploymentObservations(ctx, 10)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, d := range rows {
|
||||||
|
sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
|
||||||
|
if e != nil {
|
||||||
|
return e
|
||||||
|
}
|
||||||
|
observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
|
||||||
|
if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
|
||||||
|
hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||||
|
h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
|
||||||
|
cancel()
|
||||||
|
observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
|
||||||
|
if h.OK && observation.RevisionID == "" {
|
||||||
|
observation.Error = "legacy health did not report an applied revision"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,140 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||||
|
ctx := context.Background()
|
||||||
|
r := NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(m)
|
||||||
|
initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
claim := func(owner string) *Job {
|
||||||
|
t.Helper()
|
||||||
|
rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||||
|
if e != nil || len(rows) != 1 {
|
||||||
|
t.Fatalf("claim: %+v %v", rows, e)
|
||||||
|
}
|
||||||
|
return jobFromRecord(rows[0])
|
||||||
|
}
|
||||||
|
first := claim("initial")
|
||||||
|
if first.ID != initial.ID {
|
||||||
|
t.Fatal("wrong initial job")
|
||||||
|
}
|
||||||
|
(&Worker{Store: m}).Process(first)
|
||||||
|
if err = m.FinishJob(ctx, recordFromJob(first)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// Enqueue an operator request before the synthetic recovery request. The latter
|
||||||
|
// must retain the earlier desired order despite its own larger queue sequence.
|
||||||
|
operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for range 2 {
|
||||||
|
if err = ReconcileDeployments(ctx, r, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100)
|
||||||
|
if err != nil || len(all) != 3 {
|
||||||
|
t.Fatalf("reconcile dedupe: %d %v", len(all), err)
|
||||||
|
}
|
||||||
|
newer := claim("operator")
|
||||||
|
if newer.ID != operator.ID {
|
||||||
|
t.Fatal("wrong operator job")
|
||||||
|
}
|
||||||
|
recovery := claim("recovery")
|
||||||
|
f := deploymentFenceForJob(recovery)
|
||||||
|
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if d.RequestSequence != first.RequestSequence {
|
||||||
|
t.Fatal("recovery lost original desired order")
|
||||||
|
}
|
||||||
|
if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err)
|
||||||
|
}
|
||||||
|
if newD.RequestSequence != newer.RequestSequence {
|
||||||
|
t.Fatal("operator desired sequence changed")
|
||||||
|
}
|
||||||
|
if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil {
|
||||||
|
t.Fatal("stale recovery generation reused")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||||
|
m := store.NewMemory()
|
||||||
|
m.SeedDemo()
|
||||||
|
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||||
|
ctx := context.Background()
|
||||||
|
r := NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(m)
|
||||||
|
_, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
|
||||||
|
if err != nil || len(rows) != 1 {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
j := jobFromRecord(rows[0])
|
||||||
|
(&Worker{Store: m}).Process(j)
|
||||||
|
if err = m.FinishJob(ctx, recordFromJob(j)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||||
|
if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||||
|
response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}`
|
||||||
|
client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) {
|
||||||
|
if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" {
|
||||||
|
t.Error("health must be authenticated GET")
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
if err = ObserveDeployments(ctx, m, client); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
|
||||||
|
if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" {
|
||||||
|
t.Fatalf("legacy health confirmation: %+v %v", progress, err)
|
||||||
|
}
|
||||||
|
response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}`
|
||||||
|
if err = ObserveDeployments(ctx, m, client); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID)
|
||||||
|
if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" {
|
||||||
|
t.Fatalf("late drift hidden: %+v", progress)
|
||||||
|
}
|
||||||
|
if err = ReconcileDeployments(ctx, r, m); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100)
|
||||||
|
if len(jobs) != 1 {
|
||||||
|
t.Fatalf("drift recovery jobs: %d", len(jobs))
|
||||||
|
}
|
||||||
|
}
|
||||||
+193
-113
@@ -2,136 +2,216 @@ package jobs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"os"
|
"errors"
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"evobgp/internal/logging"
|
"evobgp/internal/logging"
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
"evobgp/internal/repository"
|
"evobgp/internal/store"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Adopt registers a durable job_audit claim into the in-process registry and starts the worker.
|
const durableLease = time.Minute
|
||||||
// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart.
|
|
||||||
func (r *Registry) Adopt(j *Job) bool {
|
// StartDurableQueueLoop claims work only after reserving local capacity.
|
||||||
if r == nil || j == nil || j.ID == "" {
|
func StartDurableQueueLoop(ctx context.Context, r *Registry) {
|
||||||
return false
|
if r == nil || r.queue == nil {
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
log := logging.With("component", "jobs.durable")
|
||||||
|
r.durableMu.Lock()
|
||||||
|
if r.durableStop {
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.durableLoopWG.Add(2)
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
go r.reapExpiredJobsLoop(ctx)
|
||||||
|
go func() {
|
||||||
|
defer r.durableLoopWG.Done()
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
r.durableMu.Lock()
|
||||||
|
stopped := r.durableStop
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
if stopped {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||||
|
if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil {
|
||||||
|
observability.RecordJobAuditDepth(counts)
|
||||||
|
}
|
||||||
|
metricCancel()
|
||||||
|
free := 0
|
||||||
|
reserve:
|
||||||
|
for free < cap(r.workerSem) {
|
||||||
|
select {
|
||||||
|
case r.workerSem <- struct{}{}:
|
||||||
|
free++
|
||||||
|
default:
|
||||||
|
break reserve
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if free > 0 {
|
||||||
|
qctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("claim failed", "err", err)
|
||||||
|
}
|
||||||
|
for i := len(claimed); i < free; i++ {
|
||||||
|
<-r.workerSem
|
||||||
|
}
|
||||||
|
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
|
||||||
|
for _, c := range claimed {
|
||||||
|
r.durableMu.Lock()
|
||||||
|
stopped = r.durableStop
|
||||||
|
if stopped || ctx.Err() != nil {
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil {
|
||||||
|
log.Warn("release during shutdown failed", "job_id", c.ID, "err", err)
|
||||||
|
}
|
||||||
|
releaseCancel()
|
||||||
|
<-r.workerSem
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
j := jobFromRecord(c)
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
if _, exists := r.byID[j.ID]; exists {
|
|
||||||
r.mu.Unlock()
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
if j.Status == "" {
|
|
||||||
j.Status = StatusQueued
|
|
||||||
}
|
|
||||||
if j.Meta == nil {
|
|
||||||
j.Meta = map[string]any{}
|
|
||||||
}
|
|
||||||
r.byID[j.ID] = j
|
r.byID[j.ID] = j
|
||||||
if j.IdempotencyKey != nil && *j.IdempotencyKey != "" {
|
|
||||||
r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j
|
|
||||||
}
|
|
||||||
if isRefreshKind(j.Kind) {
|
if isRefreshKind(j.Kind) {
|
||||||
r.inflightRefresh[j.TenantID]++
|
r.inflightRefresh[j.TenantID]++
|
||||||
}
|
}
|
||||||
workerStart := r.workerStart
|
|
||||||
r.mu.Unlock()
|
r.mu.Unlock()
|
||||||
|
r.durableWG.Add(1)
|
||||||
if workerStart != nil {
|
go r.runClaimed(j)
|
||||||
go func() {
|
r.durableMu.Unlock()
|
||||||
r.workerSem <- struct{}{}
|
|
||||||
active := len(r.workerSem)
|
|
||||||
capacity := cap(r.workerSem)
|
|
||||||
observability.RecordJobQueueDepth(active, capacity)
|
|
||||||
defer func() {
|
|
||||||
<-r.workerSem
|
|
||||||
observability.RecordJobQueueDepth(len(r.workerSem), capacity)
|
|
||||||
}()
|
|
||||||
workerStart(j)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED).
|
|
||||||
// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m).
|
|
||||||
func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) {
|
|
||||||
if ctx == nil || reg == nil || audit == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
interval := 20 * time.Second
|
|
||||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" {
|
|
||||||
if d, err := time.ParseDuration(s); err == nil && d > 0 {
|
|
||||||
interval = d
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
staleAfter := 15 * time.Minute
|
|
||||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" {
|
|
||||||
if d, err := time.ParseDuration(s); err == nil && d > 0 {
|
|
||||||
staleAfter = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
limit := 8
|
|
||||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 {
|
|
||||||
limit = n
|
|
||||||
}
|
|
||||||
grace := 30 * time.Second
|
|
||||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" {
|
|
||||||
if d, err := time.ParseDuration(s); err == nil && d >= 0 {
|
|
||||||
grace = d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
log := logging.With("component", "jobs.durable")
|
|
||||||
run := func() {
|
|
||||||
cctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
n, err := audit.ReclaimStaleRunning(cctx, staleAfter)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn("reclaim stale running failed", "err", err)
|
|
||||||
} else if n > 0 {
|
|
||||||
log.Info("reclaimed stale running jobs", "count", n)
|
|
||||||
}
|
|
||||||
claimed, err := audit.ClaimQueued(cctx, limit, grace)
|
|
||||||
if err != nil {
|
|
||||||
log.Warn("claim queued failed", "err", err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
for i := range claimed {
|
|
||||||
c := claimed[i]
|
|
||||||
j := &Job{
|
|
||||||
ID: c.ID,
|
|
||||||
TenantID: c.TenantID,
|
|
||||||
Kind: c.Kind,
|
|
||||||
Status: StatusQueued,
|
|
||||||
IdempotencyKey: c.IdempotencyKey,
|
|
||||||
ModuleID: c.ModuleID,
|
|
||||||
CreatedAt: c.CreatedAt,
|
|
||||||
Meta: c.Meta,
|
|
||||||
}
|
|
||||||
if !reg.Adopt(j) {
|
|
||||||
// Already local — leave DB running; local worker owns it.
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID)
|
|
||||||
}
|
|
||||||
counts, err := audit.CountByStatus(cctx)
|
|
||||||
if err == nil {
|
|
||||||
observability.RecordJobAuditDepth(counts)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
run()
|
|
||||||
t := time.NewTicker(interval)
|
|
||||||
defer t.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
return
|
return
|
||||||
case <-t.C:
|
case <-ticker.C:
|
||||||
run()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim.
|
||||||
|
func (r *Registry) reapExpiredJobsLoop(ctx context.Context) {
|
||||||
|
defer r.durableLoopWG.Done()
|
||||||
|
r.reapExpiredJobs(ctx)
|
||||||
|
reap := time.NewTicker(15 * time.Second)
|
||||||
|
stop := time.NewTicker(time.Second)
|
||||||
|
defer reap.Stop()
|
||||||
|
defer stop.Stop()
|
||||||
|
for {
|
||||||
|
if r.durableStopped() || ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
case <-stop.C:
|
||||||
|
if r.durableStopped() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case <-reap.C:
|
||||||
|
r.reapExpiredJobs(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) durableStopped() bool {
|
||||||
|
if r == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
r.durableMu.Lock()
|
||||||
|
defer r.durableMu.Unlock()
|
||||||
|
return r.durableStop
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) reapExpiredJobs(ctx context.Context) {
|
||||||
|
if r == nil || r.queue == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil {
|
||||||
|
logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) runClaimed(j *Job) {
|
||||||
|
log := logging.With("component", "jobs.durable")
|
||||||
|
var leaseHealthy atomic.Bool
|
||||||
|
leaseHealthy.Store(true)
|
||||||
|
defer func() {
|
||||||
|
r.mu.Lock()
|
||||||
|
delete(r.byID, j.ID)
|
||||||
|
r.mu.Unlock()
|
||||||
|
<-r.workerSem
|
||||||
|
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
|
||||||
|
r.durableWG.Done()
|
||||||
|
}()
|
||||||
|
stop := make(chan struct{})
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
t := time.NewTicker(durableLease / 4)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease)
|
||||||
|
if err == nil {
|
||||||
|
err = r.queue.CheckpointJob(ctx, recordFromJob(j))
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
leaseHealthy.Store(false)
|
||||||
|
j.RequestCancel()
|
||||||
|
log.Warn("heartbeat failed", "job_id", j.ID, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if requested {
|
||||||
|
j.RequestCancel()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
r.workerStart(j)
|
||||||
|
close(stop)
|
||||||
|
<-done
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
r.durableMu.Lock()
|
||||||
|
stopped := r.durableStop
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
c := recordFromJob(j)
|
||||||
|
if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed {
|
||||||
|
if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
|
||||||
|
log.Warn("release failed", "err", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !leaseHealthy.Load() {
|
||||||
|
// Leave an uncertain lease for expiry/reclaim. This consumes an attempt,
|
||||||
|
// whereas a deliberate graceful shutdown releases without using the budget.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled {
|
||||||
|
c.Status = StatusFailed
|
||||||
|
msg := "worker returned without terminal status"
|
||||||
|
c.Error = &msg
|
||||||
|
}
|
||||||
|
if err := r.queue.FinishJob(ctx, c); err != nil {
|
||||||
|
log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type blockingClaimQueue struct {
|
||||||
|
store.JobQueue
|
||||||
|
entered chan int
|
||||||
|
resume chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
|
||||||
|
q.entered <- limit
|
||||||
|
<-q.resume
|
||||||
|
return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
|
||||||
|
t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
|
||||||
|
m := store.NewMemory()
|
||||||
|
queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
|
||||||
|
var runs atomic.Int32
|
||||||
|
r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
|
||||||
|
r.UseDurableQueue(q)
|
||||||
|
StartDurableQueueLoop(context.Background(), r)
|
||||||
|
select {
|
||||||
|
case limit := <-q.entered:
|
||||||
|
if limit != cap(r.workerSem) {
|
||||||
|
t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("no claim")
|
||||||
|
}
|
||||||
|
r.RequestCancelAll()
|
||||||
|
drained := make(chan error, 1)
|
||||||
|
go func() { drained <- r.Drain(context.Background()) }()
|
||||||
|
select {
|
||||||
|
case <-drained:
|
||||||
|
t.Fatal("drained before in-flight claim returned")
|
||||||
|
case <-time.After(20 * time.Millisecond):
|
||||||
|
}
|
||||||
|
close(q.resume)
|
||||||
|
select {
|
||||||
|
case err := <-drained:
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
case <-time.After(3 * time.Second):
|
||||||
|
t.Fatal("shutdown did not drain")
|
||||||
|
}
|
||||||
|
if runs.Load() != 0 {
|
||||||
|
t.Fatal("started worker after stop")
|
||||||
|
}
|
||||||
|
got, err := m.GetJob(context.Background(), "tenant", queued.ID)
|
||||||
|
if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
|
||||||
|
t.Fatalf("not released: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
m := store.NewMemory()
|
||||||
|
key := "request"
|
||||||
|
a := NewRegistry(nil)
|
||||||
|
a.UseDurableQueue(m)
|
||||||
|
j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||||
|
if err != nil || len(claimed) != 1 {
|
||||||
|
t.Fatalf("claim %+v %v", claimed, err)
|
||||||
|
}
|
||||||
|
claimed[0].Status = StatusSucceeded
|
||||||
|
claimed[0].Meta = map[string]any{"result": "ok"}
|
||||||
|
if err = m.FinishJob(ctx, claimed[0]); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b := NewRegistry(nil)
|
||||||
|
b.UseDurableQueue(m)
|
||||||
|
got, err := b.GetContext(ctx, "tenant", j.ID)
|
||||||
|
if err != nil || got.Status != StatusSucceeded {
|
||||||
|
t.Fatalf("shared get %+v %v", got, err)
|
||||||
|
}
|
||||||
|
list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
|
||||||
|
if err != nil || len(list) != 1 {
|
||||||
|
t.Fatalf("shared list %+v %v", list, err)
|
||||||
|
}
|
||||||
|
replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
|
||||||
|
if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
|
||||||
|
t.Fatalf("terminal replay %+v %v %v", replay, created, err)
|
||||||
|
}
|
||||||
|
if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
|
||||||
|
t.Fatalf("changed payload accepted: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
m := store.NewMemory()
|
||||||
|
r := NewRegistry(nil)
|
||||||
|
r.UseDurableQueue(m)
|
||||||
|
parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key := "deploy_from:" + parent.ID
|
||||||
|
child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
worker := &Worker{Store: m, Registry: r}
|
||||||
|
worker.Process(parent)
|
||||||
|
if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
|
||||||
|
t.Fatalf("did not recover child: %+v", parent.Snapshot())
|
||||||
|
}
|
||||||
|
items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
|
||||||
|
if err != nil || len(items) != 1 {
|
||||||
|
t.Fatalf("duplicate child: %+v %v", items, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
+100
-3
@@ -11,6 +11,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"evobgp/internal/observability"
|
"evobgp/internal/observability"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
)
|
)
|
||||||
@@ -38,6 +39,11 @@ type Job struct {
|
|||||||
Error *string
|
Error *string
|
||||||
ProgressPct *int16
|
ProgressPct *int16
|
||||||
Meta map[string]any
|
Meta map[string]any
|
||||||
|
OwnerID string
|
||||||
|
FencingToken int64
|
||||||
|
RequestSequence int64
|
||||||
|
Attempt int
|
||||||
|
requestHash string
|
||||||
cancelRequested bool
|
cancelRequested bool
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
}
|
}
|
||||||
@@ -225,6 +231,30 @@ type Registry struct {
|
|||||||
// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
|
// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
|
||||||
// deploy coalescing under tenantRefreshMu (instead of polling job statuses).
|
// deploy coalescing under tenantRefreshMu (instead of polling job statuses).
|
||||||
inflightRefresh map[string]int
|
inflightRefresh map[string]int
|
||||||
|
queue store.JobQueue
|
||||||
|
ownerID string
|
||||||
|
durableMu sync.Mutex
|
||||||
|
durableWG sync.WaitGroup
|
||||||
|
durableLoopWG sync.WaitGroup
|
||||||
|
durableStop bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// UseDurableQueue makes the shared store authoritative for new work and job reads.
|
||||||
|
func (r *Registry) UseDurableQueue(q store.JobQueue) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.queue = q
|
||||||
|
r.ownerID = uuid.NewString()
|
||||||
|
}
|
||||||
|
|
||||||
|
func jobFromRecord(c store.JobRecord) *Job {
|
||||||
|
return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested}
|
||||||
|
}
|
||||||
|
|
||||||
|
func recordFromJob(j *Job) store.JobRecord {
|
||||||
|
j.mu.Lock()
|
||||||
|
defer j.mu.Unlock()
|
||||||
|
return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested}
|
||||||
}
|
}
|
||||||
|
|
||||||
type idempoKey struct {
|
type idempoKey struct {
|
||||||
@@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int {
|
|||||||
return 8
|
return 8
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func jobRegistryPendingLimit() int {
|
||||||
|
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
return 10000
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Registry) ActiveCountUnsafe() int {
|
||||||
|
n := 0
|
||||||
|
for _, j := range r.byID {
|
||||||
|
st := j.statusLocked()
|
||||||
|
if st == StatusQueued || st == StatusRunning {
|
||||||
|
n++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
// pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
|
// pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
|
||||||
func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
||||||
if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
|
if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
|
||||||
@@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
|||||||
|
|
||||||
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
||||||
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||||
|
return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnqueueContext commits a durable job before returning it to an HTTP caller.
|
||||||
|
func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||||
|
return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// EnqueueRequestContext preserves idempotency of dynamically resolved selectors.
|
||||||
|
func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) {
|
||||||
|
if r.queue != nil {
|
||||||
|
r.durableMu.Lock()
|
||||||
|
stopped := r.durableStop
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
if stopped {
|
||||||
|
return nil, false, fmt.Errorf("jobs: shutting down")
|
||||||
|
}
|
||||||
|
c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
return jobFromRecord(c), created, nil
|
||||||
|
}
|
||||||
|
hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
maxJobs := registryMaxJobsFromEnv()
|
maxJobs := registryMaxJobsFromEnv()
|
||||||
r.pruneTerminalIfOver(maxJobs)
|
r.pruneTerminalIfOver(maxJobs)
|
||||||
@@ -359,13 +434,16 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
|||||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||||
k := idempoKey{tenant: tenantID, key: *idempotencyKey}
|
k := idempoKey{tenant: tenantID, key: *idempotencyKey}
|
||||||
if existing, ok := r.byIdempo[k]; ok {
|
if existing, ok := r.byIdempo[k]; ok {
|
||||||
st := existing.statusLocked()
|
|
||||||
if st == StatusQueued || st == StatusRunning {
|
|
||||||
r.mu.Unlock()
|
r.mu.Unlock()
|
||||||
|
if existing.requestHash != hash {
|
||||||
|
return nil, false, store.ErrJobConflict
|
||||||
|
}
|
||||||
return existing, false, nil
|
return existing, false, nil
|
||||||
}
|
}
|
||||||
delete(r.byIdempo, k)
|
|
||||||
}
|
}
|
||||||
|
if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() {
|
||||||
|
r.mu.Unlock()
|
||||||
|
return nil, false, store.ErrJobQueueFull
|
||||||
}
|
}
|
||||||
|
|
||||||
j := &Job{
|
j := &Job{
|
||||||
@@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
|||||||
ModuleID: moduleID,
|
ModuleID: moduleID,
|
||||||
CreatedAt: time.Now().UTC(),
|
CreatedAt: time.Now().UTC(),
|
||||||
Meta: cloneMeta(meta),
|
Meta: cloneMeta(meta),
|
||||||
|
requestHash: hash,
|
||||||
}
|
}
|
||||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||||
r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
|
r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
|
||||||
@@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int {
|
|||||||
if r == nil {
|
if r == nil {
|
||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
if r.queue != nil {
|
||||||
|
r.durableMu.Lock()
|
||||||
|
r.durableStop = true
|
||||||
|
r.durableMu.Unlock()
|
||||||
|
}
|
||||||
r.mu.RLock()
|
r.mu.RLock()
|
||||||
defer r.mu.RUnlock()
|
defer r.mu.RUnlock()
|
||||||
n := 0
|
n := 0
|
||||||
@@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int {
|
|||||||
j.RequestCancel()
|
j.RequestCancel()
|
||||||
n++
|
n++
|
||||||
}
|
}
|
||||||
|
if r.queue != nil && len(r.workerSem) > n {
|
||||||
|
n = len(r.workerSem)
|
||||||
|
}
|
||||||
return n
|
return n
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error {
|
|||||||
if r == nil {
|
if r == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if r.queue != nil {
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
return nil
|
||||||
|
case <-ctx.Done():
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
ticker := time.NewTicker(50 * time.Millisecond)
|
ticker := time.NewTicker(50 * time.Millisecond)
|
||||||
defer ticker.Stop()
|
defer ticker.Stop()
|
||||||
for {
|
for {
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
package jobs
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FindIdempotentContext looks up a retained logical operation without inserting.
|
||||||
|
func (r *Registry) FindIdempotentContext(ctx context.Context, tenant, key string) (*Job, error) {
|
||||||
|
if r.queue != nil {
|
||||||
|
c, err := r.queue.FindJobByIdempotencyKey(ctx, tenant, key)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
j := jobFromRecord(c)
|
||||||
|
j.requestHash = c.RequestHash
|
||||||
|
return j, nil
|
||||||
|
}
|
||||||
|
r.mu.RLock()
|
||||||
|
defer r.mu.RUnlock()
|
||||||
|
j := r.byIdempo[idempoKey{tenant: tenant, key: key}]
|
||||||
|
if j == nil {
|
||||||
|
return nil, store.ErrJobNotFound
|
||||||
|
}
|
||||||
|
return j, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ReplayContext validates an immutable request before expensive live resolution.
|
||||||
|
func (r *Registry) ReplayContext(ctx context.Context, in store.JobEnqueue) (*Job, bool, error) {
|
||||||
|
if in.IdempotencyKey == nil || *in.IdempotencyKey == "" {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
j, err := r.FindIdempotentContext(ctx, in.TenantID, *in.IdempotencyKey)
|
||||||
|
if errors.Is(err, store.ErrJobNotFound) {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
h, err := store.JobRequestHash(in)
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, err
|
||||||
|
}
|
||||||
|
if j.requestHash != h {
|
||||||
|
return nil, false, store.ErrJobConflict
|
||||||
|
}
|
||||||
|
return j, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetContext reads a job from the shared store when durability is enabled.
|
||||||
|
func (r *Registry) GetContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||||
|
if r.queue == nil {
|
||||||
|
return r.Get(tenant, id)
|
||||||
|
}
|
||||||
|
c, err := r.queue.GetJob(ctx, tenant, id)
|
||||||
|
if errors.Is(err, store.ErrJobNotFound) {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return jobFromRecord(c), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListContext returns a bounded tenant page from the authoritative queue.
|
||||||
|
func (r *Registry) ListContext(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]*Job, string, bool, error) {
|
||||||
|
if r.queue == nil {
|
||||||
|
j, n, m := r.List(tenant, status, kind, cursor, limit)
|
||||||
|
return j, n, m, nil
|
||||||
|
}
|
||||||
|
c, next, more, err := r.queue.ListJobs(ctx, tenant, status, kind, cursor, limit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, "", false, err
|
||||||
|
}
|
||||||
|
out := make([]*Job, 0, len(c))
|
||||||
|
for _, v := range c {
|
||||||
|
out = append(out, jobFromRecord(v))
|
||||||
|
}
|
||||||
|
return out, next, more, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RequestCancelContext persists a cancel request visible to any worker replica.
|
||||||
|
func (r *Registry) RequestCancelContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||||
|
if r.queue == nil {
|
||||||
|
return r.RequestCancel(tenant, id)
|
||||||
|
}
|
||||||
|
c, err := r.queue.CancelJob(ctx, tenant, id)
|
||||||
|
if errors.Is(err, store.ErrJobNotFound) {
|
||||||
|
return nil, ErrNotFound
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r.mu.RLock()
|
||||||
|
local := r.byID[id]
|
||||||
|
r.mu.RUnlock()
|
||||||
|
if local != nil {
|
||||||
|
local.RequestCancel()
|
||||||
|
}
|
||||||
|
return jobFromRecord(c), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// RetryContext schedules an explicit retry of a failed shared operation.
|
||||||
|
func (r *Registry) RetryContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||||
|
if r.queue == nil {
|
||||||
|
return nil, store.ErrJobConflict
|
||||||
|
}
|
||||||
|
c, err := r.queue.RetryJob(ctx, tenant, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return jobFromRecord(c), nil
|
||||||
|
}
|
||||||
+570
-122
@@ -2,6 +2,7 @@ package jobs
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
@@ -69,10 +70,15 @@ type Worker struct {
|
|||||||
Store store.Backend
|
Store store.Backend
|
||||||
PgPool *pgxpool.Pool
|
PgPool *pgxpool.Pool
|
||||||
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
|
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
|
||||||
|
NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
|
||||||
// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
|
// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
|
||||||
Registry *Registry
|
Registry *Registry
|
||||||
// refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess).
|
// refreshGate decides which module refresh renders for a tenant.
|
||||||
refreshGate sync.Map // map[string]*sync.Mutex
|
// The mutex covers only that decision; the render itself runs outside the lock.
|
||||||
|
refreshGate sync.Map // map[string]*tenantRefreshGate
|
||||||
|
// dispatchGate serializes publish and deployment state for one speaker in this process.
|
||||||
|
// Agent wake runs outside the mutex.
|
||||||
|
dispatchGate sync.Map // map[string]*sync.Mutex
|
||||||
}
|
}
|
||||||
|
|
||||||
type revisionLogEntry struct {
|
type revisionLogEntry struct {
|
||||||
@@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) {
|
|||||||
j.MarkCancelled()
|
j.MarkCancelled()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// A committed child is the durable checkpoint for a parent that crashed
|
||||||
|
// before its own terminal write. Reuse it before rendering another revision.
|
||||||
|
if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
|
||||||
|
child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
|
||||||
|
if err == nil {
|
||||||
|
j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
|
||||||
|
if isRefreshKind(j.Kind) {
|
||||||
|
w.Registry.finalizeRefreshCoalesce(j.TenantID)
|
||||||
|
}
|
||||||
|
j.Succeed()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !errors.Is(err, store.ErrJobNotFound) {
|
||||||
|
j.Fail(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
switch j.Kind {
|
switch j.Kind {
|
||||||
case KindModuleRefresh:
|
case KindModuleRefresh:
|
||||||
@@ -185,17 +208,17 @@ func (w *Worker) runPeerReconcile(j *Job) {
|
|||||||
const peerJobTitle = "Обновление BGP пиров"
|
const peerJobTitle = "Обновление BGP пиров"
|
||||||
j.mergeMeta(map[string]any{"job_title": peerJobTitle})
|
j.mergeMeta(map[string]any{"job_title": peerJobTitle})
|
||||||
|
|
||||||
var revID string
|
|
||||||
latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
|
latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
|
||||||
triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
|
triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
j.Fail(err.Error())
|
j.Fail(err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if len(latest) == 0 {
|
// Peer edits bump render generation. Render from module snapshots so the
|
||||||
|
// new revision includes current prefixes without paging the previous one.
|
||||||
ctx, cancel := j.workContext()
|
ctx, cancel := j.workContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
j.MarkCancelled()
|
j.MarkCancelled()
|
||||||
@@ -204,32 +227,6 @@ func (w *Worker) runPeerReconcile(j *Job) {
|
|||||||
j.Fail(err.Error())
|
j.Fail(err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
revID = rid
|
|
||||||
} else {
|
|
||||||
baseRevID := latest[0].ID
|
|
||||||
rows := make([]store.PrefixRow, 0, 1024)
|
|
||||||
cursor := ""
|
|
||||||
for {
|
|
||||||
page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000)
|
|
||||||
rows = append(rows, page...)
|
|
||||||
if !more || strings.TrimSpace(next) == "" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
cursor = next
|
|
||||||
}
|
|
||||||
ctx, cancel := j.workContext()
|
|
||||||
defer cancel()
|
|
||||||
rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
|
|
||||||
if err != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
j.MarkCancelled()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
j.Fail(err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
revID = rid
|
|
||||||
}
|
|
||||||
j.mergeMeta(map[string]any{"revision_id": revID})
|
j.mergeMeta(map[string]any{"revision_id": revID})
|
||||||
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
|
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
|
||||||
j.mergeMeta(map[string]any{
|
j.mergeMeta(map[string]any{
|
||||||
@@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) {
|
|||||||
} else {
|
} else {
|
||||||
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
||||||
}
|
}
|
||||||
w.enqueueDeployAllSpeakers(j, j.TenantID, revID)
|
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
|
||||||
|
j.Fail(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
j.Succeed()
|
j.Succeed()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision)
|
|||||||
return "", fmt.Errorf("missing module_id for peer reconcile")
|
return "", fmt.Errorf("missing module_id for peer reconcile")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type tenantRefreshGate struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
rendering bool
|
||||||
|
dirty bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
|
||||||
|
v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
|
||||||
|
return v.(*tenantRefreshGate)
|
||||||
|
}
|
||||||
|
|
||||||
func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
|
func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
|
||||||
v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{})
|
return &w.tenantRefreshGate(tenantID).mu
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
|
||||||
|
v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
|
||||||
return v.(*sync.Mutex)
|
return v.(*sync.Mutex)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) {
|
|||||||
|
|
||||||
moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
|
moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
|
||||||
if len(moduleIDs) == 0 {
|
if len(moduleIDs) == 0 {
|
||||||
j.Fail("missing module_ids in job meta")
|
j.mergeMeta(map[string]any{"modules_refreshed": 0})
|
||||||
|
j.Succeed()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ctx, cancel := j.workContext()
|
ctx, cancel := j.workContext()
|
||||||
@@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
|||||||
j.Succeed()
|
j.Succeed()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
mu := w.tenantRefreshMu(j.TenantID)
|
gate := w.tenantRefreshGate(j.TenantID)
|
||||||
mu.Lock()
|
gate.mu.Lock()
|
||||||
defer mu.Unlock()
|
// The counter is incremented in Enqueue and decremented here, so the "last one"
|
||||||
ctx, cancel := j.workContext()
|
// decision does not depend on job status polling. Render runs after the unlock.
|
||||||
defer cancel()
|
|
||||||
// Determine whether this is the last outstanding refresh for the tenant. The counter is
|
|
||||||
// incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is
|
|
||||||
// deterministic regardless of ingest timing — unlike the previous status-polling approach
|
|
||||||
// (CountOtherActiveRefresh) which could race under -race.
|
|
||||||
isLastRefresh := true
|
isLastRefresh := true
|
||||||
if w.Registry != nil {
|
if w.Registry != nil {
|
||||||
isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
|
isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
|
||||||
}
|
}
|
||||||
if !isLastRefresh {
|
if !isLastRefresh || gate.rendering {
|
||||||
|
if gate.rendering {
|
||||||
|
gate.dirty = true
|
||||||
|
}
|
||||||
|
gate.mu.Unlock()
|
||||||
j.mergeMeta(map[string]any{
|
j.mergeMeta(map[string]any{
|
||||||
"deploy_apply_deferred": true,
|
"deploy_apply_deferred": true,
|
||||||
"deploy_apply_defer_reason": "parallel_module_refresh",
|
"deploy_apply_defer_reason": "parallel_module_refresh",
|
||||||
@@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
|||||||
j.Succeed()
|
j.Succeed()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
gate.rendering = true
|
||||||
|
gate.mu.Unlock()
|
||||||
|
defer func() {
|
||||||
|
gate.mu.Lock()
|
||||||
|
gate.rendering = false
|
||||||
|
gate.mu.Unlock()
|
||||||
|
}()
|
||||||
|
|
||||||
rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
ctx, cancel := j.workContext()
|
||||||
|
defer cancel()
|
||||||
|
rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if ctx.Err() != nil {
|
if errors.Is(err, errRenderSuperseded) {
|
||||||
|
j.Fail(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil || j.IsCancelRequested() {
|
||||||
j.MarkCancelled()
|
j.MarkCancelled()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
j.Fail(err.Error())
|
j.Fail(err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil || j.IsCancelRequested() {
|
||||||
|
j.MarkCancelled()
|
||||||
|
return
|
||||||
|
}
|
||||||
j.mergeMeta(map[string]any{"revision_id": rev})
|
j.mergeMeta(map[string]any{"revision_id": rev})
|
||||||
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
|
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
|
||||||
j.mergeMeta(map[string]any{
|
j.mergeMeta(map[string]any{
|
||||||
@@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
|||||||
} else {
|
} else {
|
||||||
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
||||||
}
|
}
|
||||||
w.enqueueDeployAllSpeakers(j, j.TenantID, rev)
|
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
|
||||||
|
j.Fail(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
j.Succeed()
|
j.Succeed()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
// renderQuietPassLimit stops a render loop that never observes a quiet input set.
|
||||||
|
// Hitting the cap fails the job; the last revision must not be published.
|
||||||
|
renderQuietPassLimit = 32
|
||||||
|
// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
|
||||||
|
speakerWakeLimit = 4
|
||||||
|
)
|
||||||
|
|
||||||
|
// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
|
||||||
|
// landed, the job was cancelled while inputs were still changing, or the quiet
|
||||||
|
// pass cap was exhausted.
|
||||||
|
var errRenderSuperseded = errors.New("newer snapshot still pending")
|
||||||
|
|
||||||
|
// renderTenantUntilQuiet builds a revision outside the tenant gate.
|
||||||
|
// It keeps rendering while another ingest marks the gate dirty or the snapshot
|
||||||
|
// generation moves, and returns a revision only after a fresh check shows the
|
||||||
|
// inputs are still the ones just rendered. Cancellation or the safety cap while
|
||||||
|
// inputs are still changing fails the render instead of publishing a stale revision.
|
||||||
|
func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
|
||||||
|
gate := w.tenantRefreshGate(j.TenantID)
|
||||||
|
var rev string
|
||||||
|
for pass := 0; pass < renderQuietPassLimit; pass++ {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
if takeRefreshDirty(gate) {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
return "", abort
|
||||||
|
}
|
||||||
|
before, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||||
|
if err != nil {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||||
|
if err != nil {
|
||||||
|
dirty := takeRefreshDirty(gate)
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil && dirty {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
return "", abort
|
||||||
|
}
|
||||||
|
if dirty {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
after, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||||
|
if err != nil {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
dirty := takeRefreshDirty(gate)
|
||||||
|
if dirty || renderInputsChanged(before, after) {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
confirm, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||||
|
if err != nil {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
return "", abort
|
||||||
|
}
|
||||||
|
return rev, nil
|
||||||
|
}
|
||||||
|
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||||
|
return "", errRenderWhileDirty(abort)
|
||||||
|
}
|
||||||
|
return "", errRenderWhileDirty(nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
func errRenderWhileDirty(cause error) error {
|
||||||
|
if cause == nil {
|
||||||
|
return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderJobAborted(ctx context.Context, j *Job) error {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if j != nil && j.IsCancelRequested() {
|
||||||
|
return context.Canceled
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func takeRefreshDirty(gate *tenantRefreshGate) bool {
|
||||||
|
if gate == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
gate.mu.Lock()
|
||||||
|
defer gate.mu.Unlock()
|
||||||
|
dirty := gate.dirty
|
||||||
|
gate.dirty = false
|
||||||
|
return dirty
|
||||||
|
}
|
||||||
|
|
||||||
|
type renderInputView struct {
|
||||||
|
generation int64
|
||||||
|
snapshots []store.ModuleSnapshotUse
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
|
||||||
|
if w == nil || w.Store == nil {
|
||||||
|
return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
|
||||||
|
}
|
||||||
|
gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
|
||||||
|
if err != nil {
|
||||||
|
if !errors.Is(err, store.ErrNotFound) {
|
||||||
|
return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
|
||||||
|
}
|
||||||
|
gen = 0
|
||||||
|
}
|
||||||
|
headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
|
||||||
|
if err != nil {
|
||||||
|
return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
|
||||||
|
}
|
||||||
|
return renderInputView{generation: gen, snapshots: headers}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func renderInputsChanged(before, after renderInputView) bool {
|
||||||
|
if before.generation != after.generation {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return !sameSnapshotHeaders(before.snapshots, after.snapshots)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
aa := append([]store.ModuleSnapshotUse(nil), a...)
|
||||||
|
bb := append([]store.ModuleSnapshotUse(nil), b...)
|
||||||
|
sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
|
||||||
|
sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
|
||||||
|
for i := range aa {
|
||||||
|
if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
// enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
|
// enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
|
||||||
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) {
|
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
|
||||||
if w == nil || w.Registry == nil {
|
if w == nil || w.Registry == nil {
|
||||||
return
|
return fmt.Errorf("jobs: deploy queue unavailable")
|
||||||
}
|
}
|
||||||
revID = strings.TrimSpace(revID)
|
revID = strings.TrimSpace(revID)
|
||||||
if revID == "" {
|
if revID == "" {
|
||||||
return
|
return fmt.Errorf("jobs: missing revision for deploy")
|
||||||
}
|
}
|
||||||
meta := map[string]any{"revision_id": revID}
|
meta := map[string]any{"revision_id": revID}
|
||||||
if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
|
if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
|
||||||
meta["trigger"] = tr
|
meta["trigger"] = tr
|
||||||
}
|
}
|
||||||
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta)
|
key := "deploy_from:" + j.ID
|
||||||
|
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
|
j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
|
||||||
return
|
return fmt.Errorf("jobs: enqueue deploy: %w", err)
|
||||||
}
|
}
|
||||||
if applyJob != nil {
|
if applyJob != nil {
|
||||||
j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
|
j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
|
||||||
}
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) runDeployApply(j *Job) {
|
func (w *Worker) runDeployApply(j *Job) {
|
||||||
revID := strings.TrimSpace(j.metaString("revision_id"))
|
revID := strings.TrimSpace(j.metaString("revision_id"))
|
||||||
spk := strings.TrimSpace(j.metaString("speaker_id"))
|
spk := strings.TrimSpace(j.metaString("speaker_id"))
|
||||||
hasSpeaker := spk != ""
|
|
||||||
if revID == "" {
|
if revID == "" {
|
||||||
j.Fail("missing revision_id in job meta")
|
j.Fail("missing revision_id in job meta")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if j.OwnerID == "" {
|
||||||
|
if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
|
||||||
|
j.Fail("deployment requires a current durable job lease")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
ctx, cancel := j.workContext()
|
ctx, cancel := j.workContext()
|
||||||
defer cancel()
|
defer cancel()
|
||||||
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
|
cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
|
||||||
if activeDir != "" {
|
var speakers []*store.Speaker
|
||||||
revObj, err := w.Store.GetRevision(j.TenantID, revID)
|
if spk != "" {
|
||||||
|
sp, err := w.Store.GetSpeaker(j.TenantID, spk)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
j.Fail(err.Error())
|
j.Fail(err.Error())
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
speakers = []*store.Speaker{sp}
|
||||||
|
} else {
|
||||||
|
speakers = w.Store.ListSpeakersForTenant(j.TenantID)
|
||||||
|
}
|
||||||
|
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
|
||||||
|
localAttempted := false
|
||||||
|
var localErr error
|
||||||
|
applyLocal := func() error {
|
||||||
|
if localAttempted {
|
||||||
|
return localErr
|
||||||
|
}
|
||||||
|
localAttempted = true
|
||||||
|
revObj, err := w.Store.GetRevision(j.TenantID, revID)
|
||||||
|
if err != nil {
|
||||||
|
localErr = err
|
||||||
|
return err
|
||||||
|
}
|
||||||
staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
|
staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
|
||||||
if staging == "" {
|
if staging == "" {
|
||||||
staging = os.TempDir() + "/evobgp-bird-staging"
|
staging = os.TempDir() + "/evobgp-bird-staging"
|
||||||
@@ -480,39 +702,292 @@ func (w *Worker) runDeployApply(j *Job) {
|
|||||||
}
|
}
|
||||||
ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
||||||
if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
|
if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
|
||||||
if ctx.Err() != nil {
|
localErr = err
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
tally := &deployTally{applied: make([]string, 0, len(speakers))}
|
||||||
|
slots := make(chan struct{}, speakerWakeLimit)
|
||||||
|
var wakeGroup sync.WaitGroup
|
||||||
|
acquireWake := func() bool {
|
||||||
|
select {
|
||||||
|
case slots <- struct{}{}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if cancelled() {
|
||||||
|
<-slots
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, sp := range speakers {
|
||||||
|
if cancelled() {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if sp == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
held := false
|
||||||
|
if speakerRemoteWake(sp) {
|
||||||
|
if !acquireWake() {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
|
||||||
|
if staged.wake && !staged.stop {
|
||||||
|
wakeGroup.Add(1)
|
||||||
|
go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
|
||||||
|
defer wakeGroup.Done()
|
||||||
|
defer func() { <-slots }()
|
||||||
|
wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
|
||||||
|
res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
|
||||||
|
cancelWake()
|
||||||
|
out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
|
||||||
|
tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
|
||||||
|
}(sp, staged.deployment)
|
||||||
|
} else if held {
|
||||||
|
<-slots
|
||||||
|
}
|
||||||
|
tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
|
||||||
|
if staged.stop {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wakeGroup.Wait()
|
||||||
|
results, applied, published, failed, unconfirmed := tally.snapshot()
|
||||||
|
j.mergeMeta(map[string]any{
|
||||||
|
"apply_summary": map[string]any{
|
||||||
|
"revision_id": revID,
|
||||||
|
"speakers_count": len(speakers),
|
||||||
|
"published_count": published,
|
||||||
|
"applied_count": len(applied),
|
||||||
|
"unconfirmed_count": unconfirmed,
|
||||||
|
"failed_count": failed,
|
||||||
|
"speaker_ids": applied,
|
||||||
|
"message": fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
|
||||||
|
},
|
||||||
|
"node_dispatch": map[string]any{"revision_id": revID, "results": results},
|
||||||
|
})
|
||||||
|
if cancelled() {
|
||||||
j.MarkCancelled()
|
j.MarkCancelled()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
j.Fail(err.Error())
|
mergeBirdPostApplyMeta(j)
|
||||||
|
if cancelled() {
|
||||||
|
j.MarkCancelled()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if failed > 0 {
|
||||||
|
j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
|
||||||
|
return
|
||||||
}
|
}
|
||||||
applied := make([]string, 0, 8)
|
j.Succeed()
|
||||||
var dispatchResults []nodedispatch.Result
|
}
|
||||||
applyOne := func(speakerID string) error {
|
|
||||||
if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil {
|
type speakerStage struct {
|
||||||
return err
|
stop bool
|
||||||
|
wake bool
|
||||||
|
deployment store.DeploymentSpeaker
|
||||||
|
result nodedispatch.Result
|
||||||
|
hasResult bool
|
||||||
|
published int
|
||||||
|
failed int
|
||||||
|
unconfirmed int
|
||||||
|
appliedID string
|
||||||
|
}
|
||||||
|
|
||||||
|
type deployTally struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
results []nodedispatch.Result
|
||||||
|
applied []string
|
||||||
|
published int
|
||||||
|
failed int
|
||||||
|
unconfirmed int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
if include {
|
||||||
|
t.results = append(t.results, res)
|
||||||
}
|
}
|
||||||
// Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy.
|
t.published += published
|
||||||
if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil {
|
t.failed += failed
|
||||||
return err
|
t.unconfirmed += unconfirmed
|
||||||
|
if appliedID != "" {
|
||||||
|
t.applied = append(t.applied, appliedID)
|
||||||
}
|
}
|
||||||
applied = append(applied, speakerID)
|
}
|
||||||
|
|
||||||
|
func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
|
||||||
|
t.mu.Lock()
|
||||||
|
defer t.mu.Unlock()
|
||||||
|
results = append([]nodedispatch.Result(nil), t.results...)
|
||||||
|
applied = append([]string(nil), t.applied...)
|
||||||
|
if applied == nil {
|
||||||
|
applied = []string{}
|
||||||
|
}
|
||||||
|
sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
|
||||||
|
sort.Strings(applied)
|
||||||
|
return results, applied, t.published, t.failed, t.unconfirmed
|
||||||
|
}
|
||||||
|
|
||||||
|
func speakerRemoteWake(sp *store.Speaker) bool {
|
||||||
|
if sp == nil || !nodedispatch.Enabled() {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
|
||||||
|
fence := deploymentFenceForJob(j)
|
||||||
|
record := func(state, confirmation, message string) error {
|
||||||
|
if j.OwnerID == "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
dispatchSpeaker := func(sp *store.Speaker) {
|
return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
|
||||||
if !nodedispatch.Enabled() || sp == nil {
|
}
|
||||||
return
|
markApplied := func(confirmation string) error {
|
||||||
|
if j.OwnerID == "" {
|
||||||
|
return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
|
||||||
|
}
|
||||||
|
return record("applied", confirmation, "")
|
||||||
|
}
|
||||||
|
return record, markApplied
|
||||||
|
}
|
||||||
|
|
||||||
|
// stageSpeakerDeploy publishes one speaker and records terminal states.
|
||||||
|
// Remote wakes are left to the caller so the speaker lock is not held across the network.
|
||||||
|
func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
|
||||||
|
var staged speakerStage
|
||||||
|
mu := w.speakerDispatchMu(sp.ID)
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if cancelled() {
|
||||||
|
staged.stop = true
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
var deployment store.DeploymentSpeaker
|
||||||
|
var publishErr error
|
||||||
|
if j.OwnerID != "" {
|
||||||
|
deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
|
||||||
|
} else {
|
||||||
|
publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
|
||||||
|
}
|
||||||
|
staged.deployment = deployment
|
||||||
|
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
|
||||||
|
if publishErr != nil {
|
||||||
|
staged.failed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
staged.published = 1
|
||||||
|
if deployment.State == "applied" {
|
||||||
|
staged.appliedID = sp.ID
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if cancelled() {
|
||||||
|
staged.stop = true
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if !speakerRemoteWake(sp) {
|
||||||
|
if activeDir != "" && strings.EqualFold(sp.Role, "master") {
|
||||||
|
err := applyLocal()
|
||||||
|
if cancelled() {
|
||||||
|
staged.stop = true
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
_ = record("failed", "unconfirmed", err.Error())
|
||||||
|
staged.failed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if err := markApplied("local_applied"); err != nil {
|
||||||
|
staged.failed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
staged.appliedID = sp.ID
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
|
||||||
|
staged.failed = 1
|
||||||
|
}
|
||||||
|
staged.unconfirmed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
|
||||||
|
staged.failed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
if deployment.DispatchAttempts >= 3 {
|
||||||
|
_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
|
||||||
|
staged.failed = 1
|
||||||
|
staged.hasResult = true
|
||||||
|
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
staged.wake = true
|
||||||
|
return staged
|
||||||
|
}
|
||||||
|
|
||||||
|
// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
|
||||||
|
// state update and is not held while WakeSpeaker is in flight.
|
||||||
|
func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
|
||||||
|
mu := w.speakerDispatchMu(sp.ID)
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if cancelled != nil && cancelled() {
|
||||||
|
return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
|
||||||
|
}
|
||||||
|
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
|
||||||
|
appliedID := ""
|
||||||
|
if res.Status == "ok" && res.AppliedRevisionID != "" {
|
||||||
|
if res.AppliedRevisionID != revID {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Confirmation = "unconfirmed"
|
||||||
|
res.Error = "agent reported a different applied revision"
|
||||||
|
} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Confirmation = "unconfirmed"
|
||||||
|
res.Error = err.Error()
|
||||||
|
} else if latest != revID {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Confirmation = "unconfirmed"
|
||||||
|
res.Error = "stale agent acknowledgement: desired revision changed"
|
||||||
|
} else if err := markApplied("agent_reported"); err != nil {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Confirmation = "unconfirmed"
|
||||||
|
res.Error = err.Error()
|
||||||
|
} else {
|
||||||
|
appliedID = sp.ID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if res.Status != "ok" || res.Confirmation != "agent_reported" {
|
||||||
|
state := "unconfirmed"
|
||||||
|
if res.Status != "ok" {
|
||||||
|
state = "unknown"
|
||||||
|
}
|
||||||
|
if err := record(state, "unconfirmed", res.Error); err != nil {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Error = err.Error()
|
||||||
}
|
}
|
||||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
|
||||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
ctx2, cancel := context.WithTimeout(ctx, 35*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID})
|
|
||||||
dispatchResults = append(dispatchResults, res)
|
|
||||||
patch := store.SpeakerMeta{
|
patch := store.SpeakerMeta{
|
||||||
LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano),
|
LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||||
LastDispatchStatus: res.Status,
|
LastDispatchStatus: res.Status,
|
||||||
@@ -520,57 +995,27 @@ func (w *Worker) runDeployApply(j *Job) {
|
|||||||
if res.Error != "" {
|
if res.Error != "" {
|
||||||
patch.LastDispatchError = res.Error
|
patch.LastDispatchError = res.Error
|
||||||
patch.SyncStatus = "error"
|
patch.SyncStatus = "error"
|
||||||
} else if res.Status == "ok" {
|
} else if res.Status == "ok" && res.Confirmation == "agent_reported" {
|
||||||
patch.LastDispatchError = ""
|
patch.LastDispatchError = ""
|
||||||
patch.SyncStatus = "synced"
|
patch.SyncStatus = "synced"
|
||||||
|
} else if res.Status == "ok" {
|
||||||
|
patch.SyncStatus = "unconfirmed"
|
||||||
}
|
}
|
||||||
merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
|
merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
|
||||||
_, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged})
|
if j.OwnerID == "" {
|
||||||
}
|
if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
|
||||||
if hasSpeaker && spk != "" {
|
res.Status = "error"
|
||||||
if err := applyOne(spk); err != nil {
|
res.Error = "dispatch metadata update: " + err.Error()
|
||||||
j.Fail(err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil {
|
|
||||||
dispatchSpeaker(sp)
|
|
||||||
}
|
|
||||||
if len(dispatchResults) > 0 {
|
|
||||||
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
|
|
||||||
"revision_id": revID,
|
|
||||||
"results": dispatchResults,
|
|
||||||
}})
|
|
||||||
}
|
|
||||||
mergeBirdPostApplyMeta(j)
|
|
||||||
j.Succeed()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
speakers := w.Store.ListSpeakersForTenant(j.TenantID)
|
|
||||||
for _, sp := range speakers {
|
|
||||||
if err := applyOne(sp.ID); err != nil {
|
|
||||||
j.Fail(err.Error())
|
|
||||||
return
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, sp := range speakers {
|
failed := 0
|
||||||
dispatchSpeaker(sp)
|
unconfirmed := 0
|
||||||
|
if res.Status == "ok" && res.Confirmation == "unconfirmed" {
|
||||||
|
unconfirmed = 1
|
||||||
|
} else if res.Status != "ok" {
|
||||||
|
failed = 1
|
||||||
}
|
}
|
||||||
j.mergeMeta(map[string]any{
|
return res, true, failed, unconfirmed, appliedID
|
||||||
"apply_summary": map[string]any{
|
|
||||||
"revision_id": revID,
|
|
||||||
"speakers_count": len(applied),
|
|
||||||
"speaker_ids": applied,
|
|
||||||
"message": fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)),
|
|
||||||
},
|
|
||||||
})
|
|
||||||
if len(dispatchResults) > 0 {
|
|
||||||
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
|
|
||||||
"revision_id": revID,
|
|
||||||
"results": dispatchResults,
|
|
||||||
}})
|
|
||||||
}
|
|
||||||
mergeBirdPostApplyMeta(j)
|
|
||||||
j.Succeed()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *Worker) runRollback(j *Job) {
|
func (w *Worker) runRollback(j *Job) {
|
||||||
@@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) {
|
|||||||
"message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
|
"message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
w.enqueueDeployAllSpeakers(j, j.TenantID, newID)
|
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
|
||||||
|
j.Fail(err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
j.Succeed()
|
j.Succeed()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,9 @@
|
|||||||
package jobs
|
package jobs
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) {
|
|||||||
t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
|
t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type renderHookStore struct {
|
||||||
|
*store.Memory
|
||||||
|
onCreate func(prefixes []store.PrefixRow)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error {
|
||||||
|
if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if s.onCreate != nil {
|
||||||
|
copied := append([]store.PrefixRow(nil), prefixes...)
|
||||||
|
s.onCreate(copied)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) {
|
||||||
|
t.Helper()
|
||||||
|
t.Setenv("EVOBGP_ASN_RESOLVE", "0")
|
||||||
|
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||||
|
mem = store.NewMemory()
|
||||||
|
mem.SeedDemo()
|
||||||
|
var modIP string
|
||||||
|
tenant, _, modIP, _, _ = mem.DemoIDs()
|
||||||
|
module = modIP
|
||||||
|
for _, mod := range mem.ListModules(tenant) {
|
||||||
|
if mod == nil || mod.ID == module {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
disabled := false
|
||||||
|
if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return tenant, module, mem
|
||||||
|
}
|
||||||
|
|
||||||
|
func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) {
|
||||||
|
t.Helper()
|
||||||
|
err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{
|
||||||
|
Prefix: prefix,
|
||||||
|
Source: "ip_range",
|
||||||
|
}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("snapshot %s: %v", prefix, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool {
|
||||||
|
t.Helper()
|
||||||
|
rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100)
|
||||||
|
for _, row := range rows {
|
||||||
|
if row.Prefix == prefix {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) {
|
||||||
|
t.Run("latest snapshot", func(t *testing.T) {
|
||||||
|
tenant, module, mem := newIPRenderFixture(t)
|
||||||
|
prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"}
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, prefixes[0])
|
||||||
|
hook := &renderHookStore{Memory: mem}
|
||||||
|
var wk *Worker
|
||||||
|
step := 0
|
||||||
|
hook.onCreate = func([]store.PrefixRow) {
|
||||||
|
step++
|
||||||
|
if step >= len(prefixes) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, prefixes[step])
|
||||||
|
gate := wk.tenantRefreshGate(tenant)
|
||||||
|
gate.mu.Lock()
|
||||||
|
gate.dirty = true
|
||||||
|
gate.mu.Unlock()
|
||||||
|
}
|
||||||
|
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||||
|
j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||||
|
wk.finishModuleRefreshSuccess(j, module)
|
||||||
|
if j.Status != StatusSucceeded {
|
||||||
|
errText := ""
|
||||||
|
if j.Error != nil {
|
||||||
|
errText = *j.Error
|
||||||
|
}
|
||||||
|
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||||
|
}
|
||||||
|
if step != len(prefixes) {
|
||||||
|
t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes))
|
||||||
|
}
|
||||||
|
rev := j.metaString("revision_id")
|
||||||
|
if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) {
|
||||||
|
t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1])
|
||||||
|
}
|
||||||
|
if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) {
|
||||||
|
t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2])
|
||||||
|
}
|
||||||
|
deployID := j.metaString("deploy_apply_job_id")
|
||||||
|
deploy, err := wk.Registry.Get(tenant, deployID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if deploy.metaString("revision_id") != rev {
|
||||||
|
t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("cancel does not deploy", func(t *testing.T) {
|
||||||
|
tenant, module, mem := newIPRenderFixture(t)
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24")
|
||||||
|
hook := &renderHookStore{Memory: mem}
|
||||||
|
var wk *Worker
|
||||||
|
var j *Job
|
||||||
|
commits := 0
|
||||||
|
hook.onCreate = func([]store.PrefixRow) {
|
||||||
|
commits++
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24")
|
||||||
|
gate := wk.tenantRefreshGate(tenant)
|
||||||
|
gate.mu.Lock()
|
||||||
|
gate.dirty = true
|
||||||
|
gate.mu.Unlock()
|
||||||
|
j.RequestCancel()
|
||||||
|
}
|
||||||
|
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||||
|
j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||||
|
wk.finishModuleRefreshSuccess(j, module)
|
||||||
|
if j.Status != StatusFailed {
|
||||||
|
t.Fatalf("status=%s, want failed", j.Status)
|
||||||
|
}
|
||||||
|
if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") {
|
||||||
|
t.Fatalf("error=%v, want superseded render", j.Error)
|
||||||
|
}
|
||||||
|
if commits != 1 {
|
||||||
|
t.Fatalf("render commits=%d, want 1", commits)
|
||||||
|
}
|
||||||
|
if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" {
|
||||||
|
t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy())
|
||||||
|
}
|
||||||
|
deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10)
|
||||||
|
if len(deploys) != 0 {
|
||||||
|
t.Fatalf("deploy jobs=%d, want 0", len(deploys))
|
||||||
|
}
|
||||||
|
revs, _, _ := mem.ListRevisions(tenant, "", "", 50)
|
||||||
|
for _, rev := range revs {
|
||||||
|
if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") {
|
||||||
|
t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("cap does not publish", func(t *testing.T) {
|
||||||
|
tenant, module, mem := newIPRenderFixture(t)
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24")
|
||||||
|
hook := &renderHookStore{Memory: mem}
|
||||||
|
var wk *Worker
|
||||||
|
commits := 0
|
||||||
|
hook.onCreate = func([]store.PrefixRow) {
|
||||||
|
commits++
|
||||||
|
installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits))
|
||||||
|
gate := wk.tenantRefreshGate(tenant)
|
||||||
|
gate.mu.Lock()
|
||||||
|
gate.dirty = true
|
||||||
|
gate.mu.Unlock()
|
||||||
|
}
|
||||||
|
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||||
|
j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||||
|
wk.finishModuleRefreshSuccess(j, module)
|
||||||
|
if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") {
|
||||||
|
errText := ""
|
||||||
|
if j.Error != nil {
|
||||||
|
errText = *j.Error
|
||||||
|
}
|
||||||
|
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||||
|
}
|
||||||
|
if commits != renderQuietPassLimit {
|
||||||
|
t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit)
|
||||||
|
}
|
||||||
|
if j.metaString("deploy_apply_job_id") != "" {
|
||||||
|
t.Fatal("safety cap published a deploy")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -107,14 +107,14 @@ func LookupWithResolver(
|
|||||||
if addr, err := netip.ParseAddr(raw); err == nil {
|
if addr, err := netip.ParseAddr(raw); err == nil {
|
||||||
out.QueryKind = KindIP
|
out.QueryKind = KindIP
|
||||||
out.Normalized = addr.String()
|
out.Normalized = addr.String()
|
||||||
if err := lookupIP(st, tenantID, addr, out, commByID, ""); err != nil {
|
if err := lookupIP(ctx, st, tenantID, addr, out, commByID, ""); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
} else if pfx, err := netip.ParsePrefix(raw); err == nil {
|
} else if pfx, err := netip.ParsePrefix(raw); err == nil {
|
||||||
masked := pfx.Masked()
|
masked := pfx.Masked()
|
||||||
out.QueryKind = KindCIDR
|
out.QueryKind = KindCIDR
|
||||||
out.Normalized = masked.String()
|
out.Normalized = masked.String()
|
||||||
if err := lookupCIDR(st, tenantID, masked, out, commByID); err != nil {
|
if err := lookupCIDR(ctx, st, tenantID, masked, out, commByID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -124,7 +124,7 @@ func LookupWithResolver(
|
|||||||
}
|
}
|
||||||
out.QueryKind = KindDomain
|
out.QueryKind = KindDomain
|
||||||
out.Normalized = fqdn
|
out.Normalized = fqdn
|
||||||
if err := lookupDomain(st, tenantID, fqdn, out, commByID); err != nil {
|
if err := lookupDomain(ctx, st, tenantID, fqdn, out, commByID); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if resolve == nil {
|
if resolve == nil {
|
||||||
@@ -178,7 +178,7 @@ func lookupResolvedIPs(
|
|||||||
out.ResolvedIPs = make([]string, 0, len(ips))
|
out.ResolvedIPs = make([]string, 0, len(ips))
|
||||||
for _, ip := range ips {
|
for _, ip := range ips {
|
||||||
out.ResolvedIPs = append(out.ResolvedIPs, ip.String())
|
out.ResolvedIPs = append(out.ResolvedIPs, ip.String())
|
||||||
if err := lookupIP(st, tenantID, ip, out, commByID, ip.String()); err != nil {
|
if err := lookupIP(ctx, st, tenantID, ip, out, commByID, ip.String()); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -186,6 +186,7 @@ func lookupResolvedIPs(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func lookupIP(
|
func lookupIP(
|
||||||
|
ctx context.Context,
|
||||||
st store.Backend,
|
st store.Backend,
|
||||||
tenantID string,
|
tenantID string,
|
||||||
addr netip.Addr,
|
addr netip.Addr,
|
||||||
@@ -227,7 +228,7 @@ func lookupIP(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -269,6 +270,7 @@ func prefixCoversQuery(listPfx, query netip.Prefix) bool {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func lookupCIDR(
|
func lookupCIDR(
|
||||||
|
ctx context.Context,
|
||||||
st store.Backend,
|
st store.Backend,
|
||||||
tenantID string,
|
tenantID string,
|
||||||
query netip.Prefix,
|
query netip.Prefix,
|
||||||
@@ -308,7 +310,7 @@ func lookupCIDR(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -338,7 +340,7 @@ func lookupCIDR(
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error {
|
func lookupDomain(ctx context.Context, st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error {
|
||||||
matchedModuleIDs := make(map[string]*store.Module)
|
matchedModuleIDs := make(map[string]*store.Module)
|
||||||
|
|
||||||
for _, mod := range st.ListModules(tenantID) {
|
for _, mod := range st.ListModules(tenantID) {
|
||||||
@@ -372,7 +374,7 @@ func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID
|
|||||||
}
|
}
|
||||||
|
|
||||||
for mid, mod := range matchedModuleIDs {
|
for mid, mod := range matchedModuleIDs {
|
||||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mid)
|
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mid)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,7 +35,7 @@ func TestLookupIPEntryAndSnapshot(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash1", []store.PrefixRow{
|
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash1", []store.PrefixRow{
|
||||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -129,7 +129,7 @@ func TestLookupDomainEntryAndSnapshot(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := m.SetModulePrefixSnapshot(tenant, mod.ID, "hash-d", []store.PrefixRow{
|
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash-d", []store.PrefixRow{
|
||||||
{Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"},
|
{Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"},
|
||||||
{Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"},
|
{Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
@@ -183,7 +183,7 @@ func TestLookupDomainResolvedIPAgainstRanges(t *testing.T) {
|
|||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-r", []store.PrefixRow{
|
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-r", []store.PrefixRow{
|
||||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -234,7 +234,7 @@ func TestLookupCIDREntryAndSnapshot(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-cidr", []store.PrefixRow{
|
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-cidr", []store.PrefixRow{
|
||||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
|
|||||||
@@ -61,13 +61,17 @@ func nodeHTTPClient() *http.Client {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetchLatestRevision(base, token, speaker string) (string, error) {
|
func fetchLatestRevision(base, token, speaker string) (string, error) {
|
||||||
|
return fetchLatestRevisionContext(context.Background(), base, token, speaker)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchLatestRevisionContext(parent context.Context, base, token, speaker string) (string, error) {
|
||||||
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
|
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
|
||||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
ctx, cancel := context.WithTimeout(parent, 60*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -75,13 +79,17 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
|
|||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
b, _ := io.ReadAll(resp.Body)
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||||
}
|
}
|
||||||
var out struct {
|
var out struct {
|
||||||
RevisionID string `json:"revision_id"`
|
RevisionID string `json:"revision_id"`
|
||||||
}
|
}
|
||||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
body, err := readHTTPBytes(resp, 64<<10)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(body, &out); err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
if out.RevisionID == "" {
|
if out.RevisionID == "" {
|
||||||
@@ -91,13 +99,21 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
||||||
|
return fetchBundleContext(context.Background(), base, token, speaker, revision, bundle.Limits{})
|
||||||
|
}
|
||||||
|
|
||||||
|
func fetchBundleContext(parent context.Context, base, token, speaker, revision string, limits bundle.Limits) ([]byte, error) {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision
|
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision
|
||||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
req.Header.Set("Authorization", "Bearer "+token)
|
req.Header.Set("Authorization", "Bearer "+token)
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
ctx, cancel := context.WithTimeout(parent, 120*time.Second)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -105,10 +121,37 @@ func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
|||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
b, _ := io.ReadAll(resp.Body)
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||||
}
|
}
|
||||||
return io.ReadAll(resp.Body)
|
return readHTTPBytes(resp, l.MaxCompressedBytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func readHTTPBytes(resp *http.Response, limit int64) ([]byte, error) {
|
||||||
|
if resp.ContentLength > limit {
|
||||||
|
return nil, bundle.ErrLimit
|
||||||
|
}
|
||||||
|
return bundle.ReadCompressed(resp.Body, bundle.Limits{MaxCompressedBytes: limit})
|
||||||
|
}
|
||||||
|
|
||||||
|
func readBundleFile(path string, limits bundle.Limits) ([]byte, error) {
|
||||||
|
l, err := limits.Normalized()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() { _ = f.Close() }()
|
||||||
|
info, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if info.Size() > l.MaxCompressedBytes {
|
||||||
|
return nil, bundle.ErrLimit
|
||||||
|
}
|
||||||
|
return bundle.ReadCompressed(f, l)
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) {
|
func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) {
|
||||||
@@ -146,7 +189,7 @@ func CmdVerifyBundle(args []string) int {
|
|||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
raw, err := os.ReadFile(*path)
|
raw, err := readBundleFile(*path, bundle.Limits{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
return 1
|
return 1
|
||||||
@@ -182,7 +225,7 @@ func CmdApplyBundle(args []string) int {
|
|||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
return 2
|
return 2
|
||||||
}
|
}
|
||||||
raw, err := os.ReadFile(*path)
|
raw, err := readBundleFile(*path, bundle.Limits{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err)
|
fmt.Fprintln(os.Stderr, err)
|
||||||
return 1
|
return 1
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
package nodecli
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
)
|
||||||
|
|
||||||
|
type countingBody struct {
|
||||||
|
io.Reader
|
||||||
|
read int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *countingBody) Read(p []byte) (int, error) {
|
||||||
|
n, e := b.Reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, e
|
||||||
|
}
|
||||||
|
func (b *countingBody) Close() error { return nil }
|
||||||
|
|
||||||
|
func TestHTTPBundleBoundKnownAndChunked(t *testing.T) {
|
||||||
|
for _, length := range []int64{-1, 2048} {
|
||||||
|
body := &countingBody{Reader: bytes.NewReader(bytes.Repeat([]byte{'x'}, 2048))}
|
||||||
|
_, err := readHTTPBytes(&http.Response{Body: body, ContentLength: length}, 1024)
|
||||||
|
if !errors.Is(err, bundle.ErrLimit) {
|
||||||
|
t.Fatalf("length=%d limit=%v", length, err)
|
||||||
|
}
|
||||||
|
if body.read > 1025 || (length > 1024 && body.read != 0) {
|
||||||
|
t.Fatalf("unbounded read %d for length %d", body.read, length)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLocalBundleSizeBound(t *testing.T) {
|
||||||
|
path := filepath.Join(t.TempDir(), "bundle.tar.gz")
|
||||||
|
if err := os.WriteFile(path, bytes.Repeat([]byte{'x'}, 2048), 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := readBundleFile(path, bundle.Limits{MaxCompressedBytes: 1024}); !errors.Is(err, bundle.ErrLimit) {
|
||||||
|
t.Fatalf("local input cap: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBundleHTTPMetadataAndDiagnosticsBound(t *testing.T) {
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch {
|
||||||
|
case strings.Contains(r.URL.Path, "/revisions/latest"):
|
||||||
|
_, _ = io.WriteString(w, strings.Repeat(" ", 65<<10))
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
if _, err := fetchLatestRevision(server.URL, "key", "speaker"); !errors.Is(err, bundle.ErrLimit) {
|
||||||
|
t.Fatalf("metadata unbounded: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := fetchBundle(server.URL, "key", "speaker", "revision"); err == nil || len(err.Error()) > 5<<10 {
|
||||||
|
t.Fatalf("diagnostic unbounded: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSyncFetchRespectsCallerCancellation(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := fetchBundleContext(ctx, "http://127.0.0.1:1", "key", "speaker", "revision", bundle.Limits{}); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("download ignored cancel: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := fetchLatestRevisionContext(ctx, "http://127.0.0.1:1", "key", "speaker"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("metadata ignored cancel: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,11 +12,11 @@ import (
|
|||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
"evobgp/internal/bundle"
|
"evobgp/internal/bundle"
|
||||||
"evobgp/internal/signing"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// SyncConfig drives pull → verify → apply on a replica node.
|
// SyncConfig drives pull → verify → apply on a replica node.
|
||||||
type SyncConfig struct {
|
type SyncConfig struct {
|
||||||
|
BundleLimits bundle.Limits // zero selects bounded large-config defaults
|
||||||
BaseURL string
|
BaseURL string
|
||||||
Token string
|
Token string
|
||||||
SpeakerID string
|
SpeakerID string
|
||||||
@@ -59,12 +59,12 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
|||||||
rev := strings.TrimSpace(cfg.RevisionID)
|
rev := strings.TrimSpace(cfg.RevisionID)
|
||||||
if rev == "" {
|
if rev == "" {
|
||||||
var err error
|
var err error
|
||||||
rev, err = fetchLatestRevision(cfg.BaseURL, cfg.Token, cfg.SpeakerID)
|
rev, err = fetchLatestRevisionContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
raw, err := fetchBundle(cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev)
|
raw, err := fetchBundleContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev, cfg.BundleLimits)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
@@ -75,7 +75,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
|||||||
if err := os.WriteFile(bundlePath, raw, 0o644); err != nil {
|
if err := os.WriteFile(bundlePath, raw, 0o644); err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
v, err := signing.VerifyGzippedTar(raw, pub)
|
v, err := bundle.VerifyGzippedTarWithLimits(raw, pub, cfg.BundleLimits)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
@@ -83,7 +83,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
|||||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
if err := bundle.WriteExtractedFiles(root, v); err != nil {
|
if err := bundle.WriteExtractedFilesWithLimits(root, v, cfg.BundleLimits); err != nil {
|
||||||
return SyncResult{}, err
|
return SyncResult{}, err
|
||||||
}
|
}
|
||||||
mainRel := v.FindMainBirdConf()
|
mainRel := v.FindMainBirdConf()
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"evobgp/internal/birdfmt"
|
"evobgp/internal/birdfmt"
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -52,11 +53,16 @@ func FetchBirdProtocols(ctx context.Context, sp *store.Speaker, opts Options) Bi
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
b, err := httpclient.ReadResponseBody(resp, 8<<20)
|
||||||
|
if err != nil {
|
||||||
|
res.Error = err.Error()
|
||||||
|
return res
|
||||||
|
}
|
||||||
var out struct {
|
var out struct {
|
||||||
Sessions []birdfmt.BGPSession `json:"sessions"`
|
Sessions []birdfmt.BGPSession `json:"sessions"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ type Result struct {
|
|||||||
SpeakerID string `json:"speaker_id"`
|
SpeakerID string `json:"speaker_id"`
|
||||||
Endpoint string `json:"endpoint,omitempty"`
|
Endpoint string `json:"endpoint,omitempty"`
|
||||||
Status string `json:"status"`
|
Status string `json:"status"`
|
||||||
|
Confirmation string `json:"confirmation,omitempty"`
|
||||||
AppliedRevisionID string `json:"applied_revision_id,omitempty"`
|
AppliedRevisionID string `json:"applied_revision_id,omitempty"`
|
||||||
Error string `json:"error,omitempty"`
|
Error string `json:"error,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -62,12 +63,13 @@ func Enabled() bool {
|
|||||||
|
|
||||||
// WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik).
|
// WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik).
|
||||||
func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
|
func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
|
||||||
res := Result{SpeakerID: sp.ID}
|
res := Result{}
|
||||||
if sp == nil {
|
if sp == nil {
|
||||||
res.Status = "error"
|
res.Status = "error"
|
||||||
res.Error = "nil speaker"
|
res.Error = "nil speaker"
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
res.SpeakerID = sp.ID
|
||||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||||
url := store.AgentSyncURL(meta)
|
url := store.AgentSyncURL(meta)
|
||||||
if url == "" {
|
if url == "" {
|
||||||
@@ -117,13 +119,31 @@ func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
|
|||||||
_ = resp.Body.Close()
|
_ = resp.Body.Close()
|
||||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||||
var out struct {
|
var out struct {
|
||||||
|
OK *bool `json:"ok"`
|
||||||
AppliedRevisionID string `json:"applied_revision_id"`
|
AppliedRevisionID string `json:"applied_revision_id"`
|
||||||
|
Error string `json:"error"`
|
||||||
|
}
|
||||||
|
if len(bytes.TrimSpace(b)) > 0 {
|
||||||
|
if err := json.Unmarshal(b, &out); err != nil {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Error = "invalid agent sync response"
|
||||||
|
return res
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (out.OK != nil && !*out.OK) || strings.TrimSpace(out.Error) != "" {
|
||||||
|
res.Status = "error"
|
||||||
|
res.Error = strings.TrimSpace(out.Error)
|
||||||
|
if res.Error == "" {
|
||||||
|
res.Error = "agent sync reported failure"
|
||||||
|
}
|
||||||
|
return res
|
||||||
}
|
}
|
||||||
_ = json.Unmarshal(b, &out)
|
|
||||||
res.Status = "ok"
|
res.Status = "ok"
|
||||||
res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID)
|
res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID)
|
||||||
if res.AppliedRevisionID == "" {
|
if res.AppliedRevisionID != "" {
|
||||||
res.AppliedRevisionID = strings.TrimSpace(opts.RevisionID)
|
res.Confirmation = "agent_reported"
|
||||||
|
} else {
|
||||||
|
res.Confirmation = "unconfirmed"
|
||||||
}
|
}
|
||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"evobgp/internal/httpclient"
|
||||||
"evobgp/internal/store"
|
"evobgp/internal/store"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -43,12 +44,16 @@ func FetchAgentHealth(ctx context.Context, sp *store.Speaker, opts Options) Agen
|
|||||||
return AgentHealthResult{Error: err.Error()}
|
return AgentHealthResult{Error: err.Error()}
|
||||||
}
|
}
|
||||||
defer func() { _ = resp.Body.Close() }()
|
defer func() { _ = resp.Body.Close() }()
|
||||||
b, _ := io.ReadAll(resp.Body)
|
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||||
return AgentHealthResult{
|
return AgentHealthResult{
|
||||||
Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))),
|
Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
b, err := httpclient.ReadResponseBody(resp, 64<<10)
|
||||||
|
if err != nil {
|
||||||
|
return AgentHealthResult{Error: err.Error()}
|
||||||
|
}
|
||||||
var out struct {
|
var out struct {
|
||||||
OK bool `json:"ok"`
|
OK bool `json:"ok"`
|
||||||
LastAppliedRevisionID string `json:"last_applied_revision_id"`
|
LastAppliedRevisionID string `json:"last_applied_revision_id"`
|
||||||
|
|||||||
@@ -0,0 +1,223 @@
|
|||||||
|
package nodedispatch_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"encoding/base64"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"evobgp/internal/bundle"
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMain(m *testing.M) {
|
||||||
|
if os.Getenv("EVOBGP_TEST_LEGACY_BIRD_HELPER") != "1" {
|
||||||
|
os.Exit(m.Run())
|
||||||
|
}
|
||||||
|
args := os.Args[1:]
|
||||||
|
log, err := os.OpenFile(os.Getenv("EVOBGP_TEST_BIRD_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||||
|
if err != nil {
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
_, _ = fmt.Fprintln(log, strings.Join(args, " "))
|
||||||
|
_ = log.Close()
|
||||||
|
switch {
|
||||||
|
case len(args) == 3 && args[0] == "-c" && args[2] == "-p":
|
||||||
|
if _, err := os.Stat(args[1]); err == nil {
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
case len(args) == 3 && args[0] == "-s" && args[2] == "configure":
|
||||||
|
os.Exit(0)
|
||||||
|
case len(args) == 5 && args[0] == "-s" && strings.Join(args[2:], " ") == "show protocols all":
|
||||||
|
fmt.Println("peer_test BGP master up 2026-01-01 Established")
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestLegacyAgentBinaryCurrentDispatch runs the historical agent as a separate
|
||||||
|
// process. All network traffic stays on loopback and BIRD executables are fakes.
|
||||||
|
func TestLegacyAgentBinaryCurrentDispatch(t *testing.T) {
|
||||||
|
binary := os.Getenv("EVOBGP_TEST_LEGACY_AGENT_BIN")
|
||||||
|
if binary == "" {
|
||||||
|
t.Skip("EVOBGP_TEST_LEGACY_AGENT_BIN not set")
|
||||||
|
}
|
||||||
|
binary, err := filepath.Abs(binary)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := os.Stat(binary); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const speakerID = "legacy-speaker"
|
||||||
|
const revisionID = "legacy-revision"
|
||||||
|
const nodeToken = "local-node-token"
|
||||||
|
const agentSecret = "local-agent-secret"
|
||||||
|
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
archive, err := bundle.BuildGzippedTar(revisionID, speakerID, map[string]string{
|
||||||
|
"bird.conf": "router id 192.0.2.1;\n",
|
||||||
|
}, priv)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var bundleGets atomic.Int32
|
||||||
|
controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if r.Header.Get("Authorization") != "Bearer "+nodeToken {
|
||||||
|
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/v1/speakers/" + speakerID + "/bundle/" + revisionID:
|
||||||
|
bundleGets.Add(1)
|
||||||
|
w.Header().Set("Content-Type", "application/gzip")
|
||||||
|
_, _ = w.Write(archive)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer controlPlane.Close()
|
||||||
|
|
||||||
|
root := t.TempDir()
|
||||||
|
logPath := filepath.Join(root, "bird-invocations")
|
||||||
|
helper, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
agentAddr := listener.Addr().String()
|
||||||
|
_ = listener.Close()
|
||||||
|
outputPath := filepath.Join(root, "agent.log")
|
||||||
|
output, err := os.Create(outputPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() { _ = output.Close() }()
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
cmd := exec.CommandContext(ctx, binary, "-listen", agentAddr, "serve")
|
||||||
|
cmd.Stdout, cmd.Stderr = output, output
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"EVOBGP_AGENT_SECRET="+agentSecret,
|
||||||
|
"EVOBGP_CONTROL_PLANE_URL="+controlPlane.URL,
|
||||||
|
"EVOBGP_NODE_TOKEN="+nodeToken,
|
||||||
|
"EVOBGP_SPEAKER_ID="+speakerID,
|
||||||
|
"EVOBGP_BUNDLE_PUBKEY_BASE64="+base64.StdEncoding.EncodeToString(pub),
|
||||||
|
"EVOBGP_BIRD_EXTRACT_DIR="+filepath.Join(root, "extract"),
|
||||||
|
"EVOBGP_BIRD_BIN="+helper,
|
||||||
|
"EVOBGP_BIRDC_BIN="+helper,
|
||||||
|
"EVOBGP_BIRDC_SOCKET="+filepath.Join(root, "fake.ctl"),
|
||||||
|
"EVOBGP_TEST_LEGACY_BIRD_HELPER=1",
|
||||||
|
"EVOBGP_TEST_BIRD_LOG="+logPath,
|
||||||
|
"TMPDIR="+root,
|
||||||
|
"TMP="+root,
|
||||||
|
"TEMP="+root,
|
||||||
|
)
|
||||||
|
if err := cmd.Start(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
cancel()
|
||||||
|
_ = cmd.Wait()
|
||||||
|
}()
|
||||||
|
|
||||||
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||||
|
// Under -race each helper process can spend a second flushing race output at
|
||||||
|
// exit. Sync executes parse-check and configure before returning its ACK.
|
||||||
|
client := &http.Client{Timeout: 10 * time.Second, Transport: legacyAgentTransport{addr: agentAddr, base: transport}}
|
||||||
|
defer transport.CloseIdleConnections()
|
||||||
|
sp := &store.Speaker{ID: speakerID, Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
AgentDomain: "legacy-agent.example.test", AgentSecret: agentSecret,
|
||||||
|
})}
|
||||||
|
deadline := time.Now().Add(10 * time.Second)
|
||||||
|
for {
|
||||||
|
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||||
|
if health.OK {
|
||||||
|
if health.LastAppliedRevisionID != "" {
|
||||||
|
t.Fatalf("initial health has applied revision: %+v", health)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
log, _ := os.ReadFile(outputPath)
|
||||||
|
t.Fatalf("legacy agent did not become healthy: %+v; output: %s", health, log)
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The historical agent must enforce the bearer token before reaching sync.
|
||||||
|
badReq, err := http.NewRequest(http.MethodPost, "https://legacy-agent.example.test/v1/agent/sync", strings.NewReader(`{"revision_id":"legacy-revision"}`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
badReq.Header.Set("Authorization", "Bearer wrong-secret")
|
||||||
|
badResp, err := client.Do(badReq)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
_ = badResp.Body.Close()
|
||||||
|
if badResp.StatusCode != http.StatusUnauthorized || bundleGets.Load() != 0 {
|
||||||
|
t.Fatalf("unauthorized sync: status=%d bundle_gets=%d", badResp.StatusCode, bundleGets.Load())
|
||||||
|
}
|
||||||
|
|
||||||
|
result := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{
|
||||||
|
HTTPClient: client, MaxRetries: 1, RevisionID: revisionID,
|
||||||
|
})
|
||||||
|
if result.Status != "ok" || result.Confirmation != "agent_reported" || result.AppliedRevisionID != revisionID {
|
||||||
|
log, _ := os.ReadFile(outputPath)
|
||||||
|
t.Fatalf("current dispatch to legacy agent: %+v; output: %s", result, log)
|
||||||
|
}
|
||||||
|
if bundleGets.Load() != 1 {
|
||||||
|
t.Fatalf("signed bundle gets = %d, want 1", bundleGets.Load())
|
||||||
|
}
|
||||||
|
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||||
|
if !health.OK || health.LastAppliedRevisionID != revisionID || health.LastSyncAt == "" {
|
||||||
|
t.Fatalf("post-sync legacy health: %+v", health)
|
||||||
|
}
|
||||||
|
protocols := nodedispatch.FetchBirdProtocols(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||||
|
if protocols.Error != "" || len(protocols.Sessions) != 1 || protocols.Sessions[0].State != "Established" {
|
||||||
|
t.Fatalf("legacy bird protocols: %+v", protocols)
|
||||||
|
}
|
||||||
|
mainConfig, err := os.ReadFile(filepath.Join(root, "extract", "bird.conf"))
|
||||||
|
if err != nil || string(mainConfig) != "router id 192.0.2.1;\n" {
|
||||||
|
t.Fatalf("extracted signed bundle: content=%q err=%v", mainConfig, err)
|
||||||
|
}
|
||||||
|
invocations, err := os.ReadFile(logPath)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
wantParse := fmt.Sprintf("-c %s -p", filepath.Join(root, "extract", "bird.conf"))
|
||||||
|
if !strings.Contains(string(invocations), wantParse) || !strings.Contains(string(invocations), "-s "+filepath.Join(root, "fake.ctl")+" configure") {
|
||||||
|
t.Fatalf("legacy agent did not parse-check and configure via stand-ins: %s", invocations)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type legacyAgentTransport struct {
|
||||||
|
addr string
|
||||||
|
base http.RoundTripper
|
||||||
|
}
|
||||||
|
|
||||||
|
func (tr legacyAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
copyReq := req.Clone(req.Context())
|
||||||
|
copyReq.URL.Scheme = "http"
|
||||||
|
copyReq.URL.Host = tr.addr
|
||||||
|
copyReq.Host = tr.addr
|
||||||
|
return tr.base.RoundTrip(copyReq)
|
||||||
|
}
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
package nodedispatch_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type legacyRoundTripFunc func(*http.Request) (*http.Response, error)
|
||||||
|
|
||||||
|
func (f legacyRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||||
|
|
||||||
|
func TestLegacyCompatibilityAgentSync(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, response, wantApplied string
|
||||||
|
status int
|
||||||
|
}{
|
||||||
|
{"legacy bare success", `{"ok":true}`, "", http.StatusOK},
|
||||||
|
{"legacy empty no-content", "", "", http.StatusNoContent},
|
||||||
|
{"legacy explicit revision", `{"ok":true,"applied_revision_id":"revision-1"}`, "revision-1", http.StatusOK},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var gotAuth, gotType string
|
||||||
|
var gotBody map[string]json.RawMessage
|
||||||
|
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" {
|
||||||
|
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
|
||||||
|
}
|
||||||
|
gotAuth, gotType = r.Header.Get("Authorization"), r.Header.Get("Content-Type")
|
||||||
|
body, _ := io.ReadAll(r.Body)
|
||||||
|
if err := json.Unmarshal(body, &gotBody); err != nil {
|
||||||
|
t.Errorf("sync JSON: %v", err)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.response)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
|
||||||
|
})}
|
||||||
|
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
|
||||||
|
if got.Status != "ok" || got.AppliedRevisionID != tc.wantApplied {
|
||||||
|
t.Fatalf("sync result: %+v", got)
|
||||||
|
}
|
||||||
|
wantConfirmation := "unconfirmed"
|
||||||
|
if tc.wantApplied != "" {
|
||||||
|
wantConfirmation = "agent_reported"
|
||||||
|
}
|
||||||
|
if got.Confirmation != wantConfirmation {
|
||||||
|
t.Fatalf("confirmation = %q, want %q", got.Confirmation, wantConfirmation)
|
||||||
|
}
|
||||||
|
if gotAuth != "Bearer legacy-secret" || gotType != "application/json" {
|
||||||
|
t.Fatalf("legacy headers: authorization=%q content-type=%q", gotAuth, gotType)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(gotBody, map[string]json.RawMessage{"revision_id": json.RawMessage(`"revision-1"`)}) {
|
||||||
|
t.Fatalf("unexpected fields sent to strict legacy decoder: %s", gotBody)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAgentSyncRejectsMalformedSuccess(t *testing.T) {
|
||||||
|
for _, response := range []string{`{"ok":false}`, `{"ok":true`, `{"ok":false,"applied_revision_id":"revision-1"}`} {
|
||||||
|
t.Run(response, func(t *testing.T) {
|
||||||
|
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
|
||||||
|
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
|
||||||
|
if got.Status != "error" || got.AppliedRevisionID != "" {
|
||||||
|
t.Fatalf("invalid 2xx response accepted: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyCompatibilityAgentHealthMissingNewFields(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, body, wantRevision, wantSync string
|
||||||
|
}{
|
||||||
|
{"bare legacy health", `{"ok":true}`, "", ""},
|
||||||
|
{"revision without time", `{"ok":true,"last_applied_revision_id":"revision-1"}`, "revision-1", ""},
|
||||||
|
{"revision with time", `{"ok":true,"last_applied_revision_id":"revision-1","last_sync_at":"2026-10-26T00:00:00Z"}`, "revision-1", "2026-10-26T00:00:00Z"},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||||
|
if r.Method != http.MethodGet || r.URL.Path != "/v1/agent/health" || r.Header.Get("Authorization") != "Bearer legacy-secret" {
|
||||||
|
t.Errorf("unexpected health request: %s %s", r.Method, r.URL)
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(tc.body)), Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
sp := &store.Speaker{ID: "speaker-1", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||||
|
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
|
||||||
|
})}
|
||||||
|
got := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||||
|
if !got.OK || got.Error != "" || got.LastAppliedRevisionID != tc.wantRevision || got.LastSyncAt != tc.wantSync {
|
||||||
|
t.Fatalf("legacy health: %+v", got)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package nodedispatch_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"evobgp/internal/nodedispatch"
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type telemetryTestBody struct {
|
||||||
|
io.Reader
|
||||||
|
read int
|
||||||
|
closed bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *telemetryTestBody) Read(p []byte) (int, error) {
|
||||||
|
n, err := b.Reader.Read(p)
|
||||||
|
b.read += n
|
||||||
|
return n, err
|
||||||
|
}
|
||||||
|
func (b *telemetryTestBody) Close() error { b.closed = true; return nil }
|
||||||
|
|
||||||
|
type telemetryErrorReader struct{}
|
||||||
|
|
||||||
|
func (telemetryErrorReader) Read([]byte) (int, error) { return 0, errors.New("injected body failure") }
|
||||||
|
|
||||||
|
func TestAgentTelemetryCompleteBodyLimits(t *testing.T) {
|
||||||
|
sp := &store.Speaker{ID: "legacy", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, prefix string
|
||||||
|
limit int
|
||||||
|
call func(nodedispatch.Options) string
|
||||||
|
}{
|
||||||
|
{"health", `{"ok":true}`, 64 << 10, func(opts nodedispatch.Options) string {
|
||||||
|
return nodedispatch.FetchAgentHealth(context.Background(), sp, opts).Error
|
||||||
|
}},
|
||||||
|
{"protocols", `{"sessions":[]}`, 8 << 20, func(opts nodedispatch.Options) string {
|
||||||
|
return nodedispatch.FetchBirdProtocols(context.Background(), sp, opts).Error
|
||||||
|
}},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
for _, known := range []bool{true, false} {
|
||||||
|
for _, extra := range []int{0, 1} {
|
||||||
|
data := tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra)
|
||||||
|
body := &telemetryTestBody{Reader: strings.NewReader(data)}
|
||||||
|
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
length := int64(-1)
|
||||||
|
if known {
|
||||||
|
length = int64(len(data))
|
||||||
|
}
|
||||||
|
return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
err := tc.call(nodedispatch.Options{HTTPClient: client})
|
||||||
|
if (extra == 0 && err != "") || (extra == 1 && err == "") || body.read > tc.limit+1 || !body.closed {
|
||||||
|
t.Fatalf("known=%v extra=%d read=%d closed=%v error=%s", known, extra, body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, status := range []int{200, 400} {
|
||||||
|
reader := io.MultiReader(strings.NewReader(tc.prefix), telemetryErrorReader{})
|
||||||
|
if status == 400 {
|
||||||
|
reader = io.MultiReader(strings.NewReader(strings.Repeat("x", 16000)))
|
||||||
|
}
|
||||||
|
body := &telemetryTestBody{Reader: reader}
|
||||||
|
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
return &http.Response{StatusCode: status, ContentLength: -1, Body: body, Header: make(http.Header)}, nil
|
||||||
|
})}
|
||||||
|
err := tc.call(nodedispatch.Options{HTTPClient: client})
|
||||||
|
if err == "" || !body.closed || (status == 400 && body.read > 4<<10) {
|
||||||
|
t.Fatalf("status=%d read=%d closed=%v error=%s", status, body.read, body.closed, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -20,10 +20,12 @@ import (
|
|||||||
const namespace = "evobgp"
|
const namespace = "evobgp"
|
||||||
|
|
||||||
var (
|
var (
|
||||||
metricsStore atomic.Value // store.Backend
|
metricsStore atomic.Value // storeBackendHolder
|
||||||
registerCollectorOnce sync.Once
|
registerCollectorOnce sync.Once
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type storeBackendHolder struct{ backend store.Backend }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
httpRequests = promauto.NewCounterVec(
|
httpRequests = promauto.NewCounterVec(
|
||||||
prometheus.CounterOpts{
|
prometheus.CounterOpts{
|
||||||
@@ -117,7 +119,7 @@ var (
|
|||||||
upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||||
Namespace: namespace,
|
Namespace: namespace,
|
||||||
Name: "upstream_breaker_open",
|
Name: "upstream_breaker_open",
|
||||||
Help: "1 if per-host HTTP circuit breaker is open.",
|
Help: "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.",
|
||||||
}, []string{"host"})
|
}, []string{"host"})
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) {
|
|||||||
|
|
||||||
// RecordJobAuditDepth updates durable job_audit depth gauges.
|
// RecordJobAuditDepth updates durable job_audit depth gauges.
|
||||||
func RecordJobAuditDepth(counts map[string]int64) {
|
func RecordJobAuditDepth(counts map[string]int64) {
|
||||||
|
for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} {
|
||||||
|
jobAuditDepth.WithLabelValues(status).Set(float64(counts[status]))
|
||||||
|
}
|
||||||
for status, n := range counts {
|
for status, n := range counts {
|
||||||
if status == "" {
|
if status == "" {
|
||||||
status = "unknown"
|
status = "unknown"
|
||||||
@@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) {
|
|||||||
upstreamBreakerOpen.WithLabelValues(host).Set(v)
|
upstreamBreakerOpen.WithLabelValues(host).Set(v)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RemoveUpstreamBreaker removes an expired host series together with its cache entry.
|
||||||
|
// Callers must retain active entries until their final metric update completes.
|
||||||
|
func RemoveUpstreamBreaker(host string) {
|
||||||
|
if host == "" {
|
||||||
|
host = "_"
|
||||||
|
}
|
||||||
|
upstreamBreakerOpen.DeleteLabelValues(host)
|
||||||
|
}
|
||||||
|
|
||||||
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
|
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
|
||||||
func SetBuildInfo(version, gitSHA string) {
|
func SetBuildInfo(version, gitSHA string) {
|
||||||
if version == "" {
|
if version == "" {
|
||||||
@@ -209,30 +223,45 @@ type memoryStoreCollector struct {
|
|||||||
prefixSumDesc *prometheus.Desc
|
prefixSumDesc *prometheus.Desc
|
||||||
peersDesc *prometheus.Desc
|
peersDesc *prometheus.Desc
|
||||||
peerStateDesc *prometheus.Desc
|
peerStateDesc *prometheus.Desc
|
||||||
|
storedMaxDesc *prometheus.Desc
|
||||||
|
storedSumDesc *prometheus.Desc
|
||||||
|
inventoryDesc *prometheus.Desc
|
||||||
|
storedStateDesc *prometheus.Desc
|
||||||
|
collectionSuccessDesc *prometheus.Desc
|
||||||
}
|
}
|
||||||
|
|
||||||
func newMemoryStoreCollector() *memoryStoreCollector {
|
func newMemoryStoreCollector() *memoryStoreCollector {
|
||||||
return &memoryStoreCollector{
|
return &memoryStoreCollector{
|
||||||
prefixMaxDesc: prometheus.NewDesc(
|
prefixMaxDesc: prometheus.NewDesc(
|
||||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
|
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
|
||||||
"Maximum materialized_prefix_count among all revisions in the store.",
|
"Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.",
|
||||||
nil, nil,
|
nil, nil,
|
||||||
),
|
),
|
||||||
prefixSumDesc: prometheus.NewDesc(
|
prefixSumDesc: prometheus.NewDesc(
|
||||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
|
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
|
||||||
"Sum of materialized_prefix_count over revisions (development aggregate).",
|
"Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.",
|
||||||
nil, nil,
|
nil, nil,
|
||||||
),
|
),
|
||||||
peersDesc: prometheus.NewDesc(
|
peersDesc: prometheus.NewDesc(
|
||||||
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
|
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
|
||||||
"BGP peers configured in the control-plane store.",
|
"Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.",
|
||||||
nil, nil,
|
nil, nil,
|
||||||
),
|
),
|
||||||
peerStateDesc: prometheus.NewDesc(
|
peerStateDesc: prometheus.NewDesc(
|
||||||
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
|
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
|
||||||
"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).",
|
"Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.",
|
||||||
[]string{"state"}, nil,
|
[]string{"state"}, nil,
|
||||||
),
|
),
|
||||||
|
storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"),
|
||||||
|
"Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil),
|
||||||
|
storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"),
|
||||||
|
"Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil),
|
||||||
|
inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"),
|
||||||
|
"Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil),
|
||||||
|
storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"),
|
||||||
|
"BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil),
|
||||||
|
collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"),
|
||||||
|
"1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
|
|||||||
ch <- c.prefixSumDesc
|
ch <- c.prefixSumDesc
|
||||||
ch <- c.peersDesc
|
ch <- c.peersDesc
|
||||||
ch <- c.peerStateDesc
|
ch <- c.peerStateDesc
|
||||||
|
ch <- c.storedMaxDesc
|
||||||
|
ch <- c.storedSumDesc
|
||||||
|
ch <- c.inventoryDesc
|
||||||
|
ch <- c.storedStateDesc
|
||||||
|
ch <- c.collectionSuccessDesc
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
||||||
@@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
|||||||
if v == nil {
|
if v == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
b, ok := v.(store.Backend)
|
holder, ok := v.(storeBackendHolder)
|
||||||
if !ok || b == nil {
|
if !ok || holder.backend == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
maxN, sumN := b.MaterializedPrefixStats()
|
b := holder.backend
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1)
|
||||||
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
|
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||||
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
|
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||||
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount()))
|
ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||||
for state, n := range b.PeerSessionCountsByState() {
|
ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers))
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers))
|
||||||
|
canonicalStates := make(map[string]int)
|
||||||
|
for state, n := range states {
|
||||||
if state == "" {
|
if state == "" {
|
||||||
state = "unknown"
|
state = "unknown"
|
||||||
}
|
}
|
||||||
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
|
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||||
|
canonicalStates[canonicalPeerState(state)] += n
|
||||||
|
}
|
||||||
|
for state, n := range canonicalStates {
|
||||||
|
ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func canonicalPeerState(state string) string {
|
||||||
|
switch strings.ToLower(strings.TrimSpace(state)) {
|
||||||
|
case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown":
|
||||||
|
return strings.ToLower(strings.TrimSpace(state))
|
||||||
|
default:
|
||||||
|
return "other"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) {
|
|||||||
if b == nil {
|
if b == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
metricsStore.Store(b)
|
metricsStore.Store(storeBackendHolder{backend: b})
|
||||||
registerCollectorOnce.Do(func() {
|
registerCollectorOnce.Do(func() {
|
||||||
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
|
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -0,0 +1,116 @@
|
|||||||
|
package observability
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
|
|
||||||
|
"evobgp/internal/store"
|
||||||
|
)
|
||||||
|
|
||||||
|
type failingMetricsBackend struct{ store.Backend }
|
||||||
|
|
||||||
|
func (failingMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||||
|
return 0, 0, 0, nil, errors.New("database unavailable")
|
||||||
|
}
|
||||||
|
|
||||||
|
type storedMetricsBackend struct{ store.Backend }
|
||||||
|
|
||||||
|
func (storedMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||||
|
return 12, 24, 1, map[string]int{"established": 1}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
type legacyStateBackend struct{ store.Backend }
|
||||||
|
|
||||||
|
func (legacyStateBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||||
|
return 0, 0, 2, map[string]int{" Established ": 1, "established": 1}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoreMetricsCollectionErrorOmitsAggregates(t *testing.T) {
|
||||||
|
metricsStore.Store(storeBackendHolder{backend: failingMetricsBackend{}})
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
families, err := registry.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(families) != 1 || families[0].GetName() != "evobgp_store_metrics_collection_success" {
|
||||||
|
t.Fatalf("expected only collection status on error, got %v", families)
|
||||||
|
}
|
||||||
|
if got := families[0].Metric[0].Gauge.GetValue(); got != 0 {
|
||||||
|
t.Fatalf("collection status = %v, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStoredRevisionMetricsDoNotClaimCurrentRoutes(t *testing.T) {
|
||||||
|
metricsStore.Store(storeBackendHolder{backend: storedMetricsBackend{}})
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
families, err := registry.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
values := make(map[string]float64)
|
||||||
|
for _, family := range families {
|
||||||
|
if len(family.Metric) > 0 && family.Metric[0].Gauge != nil {
|
||||||
|
values[family.GetName()] = family.Metric[0].Gauge.GetValue()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if values["evobgp_stored_revision_materialized_prefixes_sum"] != 24 || values["evobgp_materialized_prefixes_sum"] != 24 {
|
||||||
|
t.Fatalf("stored revision and legacy sums must match: %v", values)
|
||||||
|
}
|
||||||
|
if values["evobgp_store_metrics_collection_success"] != 1 {
|
||||||
|
t.Fatalf("successful collection not reported: %v", values)
|
||||||
|
}
|
||||||
|
if _, ok := values["evobgp_current_routes"]; ok {
|
||||||
|
t.Fatal("historical revision sum must not be published as current routes")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCanonicalPeerStateBoundsNewLabelValues(t *testing.T) {
|
||||||
|
if got := canonicalPeerState("Established"); got != "established" {
|
||||||
|
t.Fatalf("state = %q", got)
|
||||||
|
}
|
||||||
|
if got := canonicalPeerState("unexpected peer text"); got != "other" {
|
||||||
|
t.Fatalf("state = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLegacyPeerStateLabelPreservedWhileNewLabelIsCanonical(t *testing.T) {
|
||||||
|
metricsStore.Store(storeBackendHolder{backend: legacyStateBackend{}})
|
||||||
|
registry := prometheus.NewRegistry()
|
||||||
|
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
families, err := registry.Gather()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
legacyRaw, newCanonical := false, false
|
||||||
|
for _, family := range families {
|
||||||
|
for _, metric := range family.Metric {
|
||||||
|
if len(metric.Label) != 1 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
switch family.GetName() {
|
||||||
|
case "evobgp_bgp_peer_sessions":
|
||||||
|
if metric.Label[0].GetValue() == " Established " && metric.Gauge.GetValue() == 1 {
|
||||||
|
legacyRaw = true
|
||||||
|
}
|
||||||
|
case "evobgp_stored_bgp_peer_session_state":
|
||||||
|
if metric.Label[0].GetValue() == "established" && metric.Gauge.GetValue() == 2 {
|
||||||
|
newCanonical = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !legacyRaw || !newCanonical {
|
||||||
|
t.Fatalf("legacy raw state preserved=%v, new canonical state aggregated=%v", legacyRaw, newCanonical)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -86,20 +86,26 @@ func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *htt
|
|||||||
}
|
}
|
||||||
|
|
||||||
func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
|
func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
|
||||||
if rows, ok, err := moduleRowsFromSnapshot(st, tenantID, mod); err != nil {
|
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||||
return nil, err
|
|
||||||
} else if ok {
|
|
||||||
return rows, nil
|
|
||||||
}
|
|
||||||
var prior []store.PrefixRow
|
|
||||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
|
||||||
prior = snap.Prefixes
|
|
||||||
}
|
|
||||||
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
|
if ok && snap != nil {
|
||||||
|
if rows, matched, err := snapshotRowsIfCurrent(ctx, st, tenantID, mod, snap); err != nil {
|
||||||
|
return nil, err
|
||||||
|
} else if matched {
|
||||||
|
return rows, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var priorRows []store.PrefixRow
|
||||||
|
if ok && snap != nil {
|
||||||
|
priorRows = snap.Prefixes
|
||||||
|
}
|
||||||
|
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
return rows, nil
|
return rows, nil
|
||||||
|
|||||||
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user