Files
DenozordecandCursor f8e0520817 feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <[email protected]>
2026-11-02 17:23:59 +07:00

280 lines
8.0 KiB
Go

//go:build linux
package birddeploy
import (
"context"
"errors"
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
"syscall"
"testing"
"time"
"evobgp/internal/birdfmt"
"evobgp/internal/store"
)
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
t.Helper()
bin := t.TempDir()
parse := filepath.Join(bin, "bird")
configure := filepath.Join(bin, "birdc")
parseScript := `#!/bin/sh
if grep -q FAIL_PARSE "$2"; then exit 1; fi
exit 0
`
configureScript := fmt.Sprintf(`#!/bin/sh
root=%q
if [ -e "$root/.configure-wait" ]; then
rm "$root/.configure-wait"
touch "$root/.configure-started"
exec sleep 5
fi
if [ -e "$root/.configure-fail-once" ]; then
rm "$root/.configure-fail-once"
exit 1
fi
mkdir "$root/.configuring" || exit 1
sleep 0.05
rmdir "$root/.configuring"
`, active)
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
t.Fatal(err)
}
}
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
}
func revision(label string) *store.Revision {
return &store.Revision{PreviewFragments: map[string]string{
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
"bird.d/evobgp_new.conf": "# " + label + "\n",
}}
}
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
t.Helper()
active := t.TempDir()
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
t.Fatal(err)
}
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
t.Fatal(err)
}
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
t.Fatal(err)
}
}
return active, fakeCtl(t, active), old
}
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
active, ctl, _ := setupActive(t)
for _, label := range []string{"one", "two", "three"} {
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
t.Fatal(err)
}
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(main), "# "+label) {
t.Fatalf("wrong active generation: %s", main)
}
if strings.Contains(string(main), `include "bird.d/`) {
t.Fatalf("include not pinned: %s", main)
}
}
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
if err != nil {
t.Fatal(err)
}
if len(gens) != 2 {
t.Fatalf("want active and previous generations, got %d", len(gens))
}
for _, gen := range gens {
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("removed fragment retained in %s: %v", gen, err)
}
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
t.Fatalf("operator fragment missing: %v", err)
}
}
}
func TestApplyRevisionFaults(t *testing.T) {
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
t.Run(mode, func(t *testing.T) {
active, ctl, old := setupActive(t)
rev := revision(mode)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
cfg := Config{ActiveDir: active}
switch mode {
case "parse":
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
case "write":
rev.PreviewFragments["../escape.conf"] = "bad"
case "disk-full":
cfg.writeFile = func(path string, b []byte) error {
if filepath.Base(path) == "previous-bird.conf" {
return syscall.ENOSPC
}
return writeSynced(path, b)
}
case "configure":
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
t.Fatal(err)
}
case "cancel":
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
t.Fatal(err)
}
}
result := make(chan error, 1)
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
if mode == "cancel" {
deadline := time.After(3 * time.Second)
for {
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
break
}
select {
case <-deadline:
t.Fatal("configure did not start")
case <-time.After(10 * time.Millisecond):
}
}
cancel()
}
err := <-result
if err == nil {
t.Fatal("expected apply failure")
}
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
if readErr != nil {
t.Fatal(readErr)
}
if string(main) != string(old) {
t.Fatalf("old config not restored: %s; error: %v", main, err)
}
if mode == "configure" || mode == "cancel" {
if !strings.Contains(err.Error(), "recovery configure succeeded") {
t.Fatalf("unconfirmed recovery: %v", err)
}
}
})
}
}
func TestApplyRevisionSerializes(t *testing.T) {
active, ctl, _ := setupActive(t)
var wg sync.WaitGroup
errs := make(chan error, 2)
for _, label := range []string{"first", "second"} {
wg.Add(1)
go func(label string) {
defer wg.Done()
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
}(label)
}
wg.Wait()
close(errs)
for err := range errs {
if err != nil {
t.Fatal(err)
}
}
}
func TestLockHelperProcess(t *testing.T) {
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
if active == "" {
return
}
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
if err != nil {
t.Fatal(err)
}
defer func() { _ = lock.Close() }()
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
t.Fatal(err)
}
for {
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
return
}
time.Sleep(10 * time.Millisecond)
}
}
func TestApplyRevisionCrossProcessLock(t *testing.T) {
active, ctl, old := setupActive(t)
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
if err := cmd.Start(); err != nil {
t.Fatal(err)
}
defer func() {
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
if err := cmd.Wait(); err != nil {
t.Errorf("lock helper: %v", err)
}
}()
deadline := time.After(3 * time.Second)
for {
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
break
}
select {
case <-deadline:
t.Fatal("lock helper did not become ready")
case <-time.After(10 * time.Millisecond):
}
}
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
defer cancel()
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("want lock timeout, got %v", err)
}
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
if err != nil {
t.Fatal(err)
}
if string(main) != string(old) {
t.Fatalf("apply bypassed cross-process lock: %s", main)
}
}
func TestRealBirdParseGeneration(t *testing.T) {
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
if bin == "" {
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
}
active := t.TempDir()
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
t.Fatal(err)
}
ctl := fakeCtl(t, active)
ctl.Bird = bin
rev := &store.Revision{PreviewFragments: map[string]string{
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
}}
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
t.Fatal(err)
}
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
t.Fatalf("activated config fails real bird -p: %v", err)
}
}