Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
280 lines
8.0 KiB
Go
280 lines
8.0 KiB
Go
//go:build linux
|
|
|
|
package birddeploy
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"syscall"
|
|
"testing"
|
|
"time"
|
|
|
|
"evobgp/internal/birdfmt"
|
|
"evobgp/internal/store"
|
|
)
|
|
|
|
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
|
|
t.Helper()
|
|
bin := t.TempDir()
|
|
parse := filepath.Join(bin, "bird")
|
|
configure := filepath.Join(bin, "birdc")
|
|
parseScript := `#!/bin/sh
|
|
if grep -q FAIL_PARSE "$2"; then exit 1; fi
|
|
exit 0
|
|
`
|
|
configureScript := fmt.Sprintf(`#!/bin/sh
|
|
root=%q
|
|
if [ -e "$root/.configure-wait" ]; then
|
|
rm "$root/.configure-wait"
|
|
touch "$root/.configure-started"
|
|
exec sleep 5
|
|
fi
|
|
if [ -e "$root/.configure-fail-once" ]; then
|
|
rm "$root/.configure-fail-once"
|
|
exit 1
|
|
fi
|
|
mkdir "$root/.configuring" || exit 1
|
|
sleep 0.05
|
|
rmdir "$root/.configuring"
|
|
`, active)
|
|
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
|
|
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
|
|
}
|
|
|
|
func revision(label string) *store.Revision {
|
|
return &store.Revision{PreviewFragments: map[string]string{
|
|
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
|
|
"bird.d/evobgp_new.conf": "# " + label + "\n",
|
|
}}
|
|
}
|
|
|
|
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
|
|
t.Helper()
|
|
active := t.TempDir()
|
|
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
|
|
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
|
|
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return active, fakeCtl(t, active), old
|
|
}
|
|
|
|
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
|
|
active, ctl, _ := setupActive(t)
|
|
for _, label := range []string{"one", "two", "three"} {
|
|
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(main), "# "+label) {
|
|
t.Fatalf("wrong active generation: %s", main)
|
|
}
|
|
if strings.Contains(string(main), `include "bird.d/`) {
|
|
t.Fatalf("include not pinned: %s", main)
|
|
}
|
|
}
|
|
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(gens) != 2 {
|
|
t.Fatalf("want active and previous generations, got %d", len(gens))
|
|
}
|
|
for _, gen := range gens {
|
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
|
|
t.Fatalf("removed fragment retained in %s: %v", gen, err)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
|
|
t.Fatalf("operator fragment missing: %v", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestApplyRevisionFaults(t *testing.T) {
|
|
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
|
|
t.Run(mode, func(t *testing.T) {
|
|
active, ctl, old := setupActive(t)
|
|
rev := revision(mode)
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
cfg := Config{ActiveDir: active}
|
|
switch mode {
|
|
case "parse":
|
|
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
|
|
case "write":
|
|
rev.PreviewFragments["../escape.conf"] = "bad"
|
|
case "disk-full":
|
|
cfg.writeFile = func(path string, b []byte) error {
|
|
if filepath.Base(path) == "previous-bird.conf" {
|
|
return syscall.ENOSPC
|
|
}
|
|
return writeSynced(path, b)
|
|
}
|
|
case "configure":
|
|
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
case "cancel":
|
|
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
result := make(chan error, 1)
|
|
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
|
|
if mode == "cancel" {
|
|
deadline := time.After(3 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("configure did not start")
|
|
case <-time.After(10 * time.Millisecond):
|
|
}
|
|
}
|
|
cancel()
|
|
}
|
|
err := <-result
|
|
if err == nil {
|
|
t.Fatal("expected apply failure")
|
|
}
|
|
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
|
|
if readErr != nil {
|
|
t.Fatal(readErr)
|
|
}
|
|
if string(main) != string(old) {
|
|
t.Fatalf("old config not restored: %s; error: %v", main, err)
|
|
}
|
|
if mode == "configure" || mode == "cancel" {
|
|
if !strings.Contains(err.Error(), "recovery configure succeeded") {
|
|
t.Fatalf("unconfirmed recovery: %v", err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestApplyRevisionSerializes(t *testing.T) {
|
|
active, ctl, _ := setupActive(t)
|
|
var wg sync.WaitGroup
|
|
errs := make(chan error, 2)
|
|
for _, label := range []string{"first", "second"} {
|
|
wg.Add(1)
|
|
go func(label string) {
|
|
defer wg.Done()
|
|
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
|
|
}(label)
|
|
}
|
|
wg.Wait()
|
|
close(errs)
|
|
for err := range errs {
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestLockHelperProcess(t *testing.T) {
|
|
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
|
|
if active == "" {
|
|
return
|
|
}
|
|
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = lock.Close() }()
|
|
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for {
|
|
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
|
|
return
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func TestApplyRevisionCrossProcessLock(t *testing.T) {
|
|
active, ctl, old := setupActive(t)
|
|
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
|
|
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
|
|
if err := cmd.Start(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() {
|
|
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
|
|
if err := cmd.Wait(); err != nil {
|
|
t.Errorf("lock helper: %v", err)
|
|
}
|
|
}()
|
|
deadline := time.After(3 * time.Second)
|
|
for {
|
|
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
|
|
break
|
|
}
|
|
select {
|
|
case <-deadline:
|
|
t.Fatal("lock helper did not become ready")
|
|
case <-time.After(10 * time.Millisecond):
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
|
defer cancel()
|
|
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
|
|
t.Fatalf("want lock timeout, got %v", err)
|
|
}
|
|
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(main) != string(old) {
|
|
t.Fatalf("apply bypassed cross-process lock: %s", main)
|
|
}
|
|
}
|
|
|
|
func TestRealBirdParseGeneration(t *testing.T) {
|
|
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
|
|
if bin == "" {
|
|
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
|
|
}
|
|
active := t.TempDir()
|
|
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
|
|
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctl := fakeCtl(t, active)
|
|
ctl.Bird = bin
|
|
rev := &store.Revision{PreviewFragments: map[string]string{
|
|
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
|
|
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
|
|
}}
|
|
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
|
|
t.Fatalf("activated config fails real bird -p: %v", err)
|
|
}
|
|
}
|