feat(httpapi): harden control plane and speed up prefix refresh
Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
1 parent
924f111e48
commit
f8e0520817
197 files changed
+18655
-1416
No files matched your search
@@ -0,0 +1,312 @@
|
||||
package agentwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
maxConfigFiles = 512
|
||||
maxConfigBytes = 64 << 20
|
||||
maxFileBytes = 16 << 20
|
||||
maxIncludeDepth = 16
|
||||
maxDirectoryEntries = 4096
|
||||
)
|
||||
|
||||
var errMainNotReady = errors.New("main bird.conf is not ready")
|
||||
|
||||
// Only standalone literal include statements are hashable. Other valid BIRD
|
||||
// grammar falls back to the legacy configure-on-every-tick path.
|
||||
var includeLine = regexp.MustCompile(`(?m)^[\t ]*include[\t ]+"([^"\r\n]+)"[\t ]*;[\t ]*(?:#[^\r\n]*)?\r?$`)
|
||||
|
||||
type configScanner struct {
|
||||
ctx context.Context
|
||||
root string
|
||||
h hash.Hash
|
||||
seen map[string]bool
|
||||
files int
|
||||
bytes int64
|
||||
}
|
||||
|
||||
func hashConfigTree(ctx context.Context, activeDir string) ([sha256.Size]byte, error) {
|
||||
var zero [sha256.Size]byte
|
||||
root, err := filepath.Abs(activeDir)
|
||||
if err != nil {
|
||||
return zero, fmt.Errorf("agentwatch: active directory: %w", err)
|
||||
}
|
||||
root, err = filepath.EvalSymlinks(root)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return zero, fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||
}
|
||||
return zero, fmt.Errorf("agentwatch: resolve active directory: %w", err)
|
||||
}
|
||||
s := &configScanner{ctx: ctx, root: filepath.Clean(root), h: sha256.New(), seen: make(map[string]bool)}
|
||||
if err := s.visit(filepath.Join(s.root, "bird.conf"), 0); err != nil {
|
||||
return zero, err
|
||||
}
|
||||
var digest [sha256.Size]byte
|
||||
copy(digest[:], s.h.Sum(nil))
|
||||
return digest, nil
|
||||
}
|
||||
|
||||
func (s *configScanner) visit(path string, depth int) error {
|
||||
if err := s.ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if depth > maxIncludeDepth {
|
||||
return fmt.Errorf("agentwatch: include depth exceeds %d", maxIncludeDepth)
|
||||
}
|
||||
path = filepath.Clean(path)
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||
}
|
||||
if s.seen[path] {
|
||||
return nil
|
||||
}
|
||||
info, err := os.Lstat(path)
|
||||
if err != nil {
|
||||
if depth == 0 && errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("agentwatch: %w: %v", errMainNotReady, err)
|
||||
}
|
||||
return fmt.Errorf("agentwatch: inspect %q: %w", path, err)
|
||||
}
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("agentwatch: include %q is not a regular file", path)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: resolve %q: %w", path, err)
|
||||
}
|
||||
if _, err := s.withinRoot(resolved); err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Size() > maxFileBytes {
|
||||
return fmt.Errorf("agentwatch: include %q exceeds %d bytes", path, maxFileBytes)
|
||||
}
|
||||
if s.files >= maxConfigFiles || s.bytes+info.Size() > maxConfigBytes {
|
||||
return fmt.Errorf("agentwatch: configuration tree exceeds %d files or %d bytes", maxConfigFiles, maxConfigBytes)
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: open %q: %w", path, err)
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
limit := int64(maxFileBytes)
|
||||
if remaining := int64(maxConfigBytes) - s.bytes; remaining < limit {
|
||||
limit = remaining
|
||||
}
|
||||
content, err := readBounded(f, limit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: read %q: %w", path, err)
|
||||
}
|
||||
s.seen[path] = true
|
||||
s.files++
|
||||
s.bytes += int64(len(content))
|
||||
writeHash(s.h, filepath.ToSlash(rel), content)
|
||||
patterns, err := includePatterns(string(content))
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: %s: %w", rel, err)
|
||||
}
|
||||
for _, pattern := range patterns {
|
||||
if strings.ContainsAny(pattern, `[\`) {
|
||||
return fmt.Errorf("agentwatch: unsupported escaped include %q", pattern)
|
||||
}
|
||||
candidate := pattern
|
||||
if !filepath.IsAbs(candidate) {
|
||||
candidate = filepath.Join(filepath.Dir(path), filepath.FromSlash(candidate))
|
||||
}
|
||||
candidate = filepath.Clean(candidate)
|
||||
// The pattern itself changes the digest even when its glob currently matches nothing.
|
||||
writeHash(s.h, "include", []byte(filepath.ToSlash(candidate)))
|
||||
if _, err := s.withinRoot(candidate); err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.ContainsAny(candidate, "*?") {
|
||||
if strings.ContainsAny(filepath.Dir(candidate), "*?") {
|
||||
return fmt.Errorf("agentwatch: unsupported include directory glob %q", pattern)
|
||||
}
|
||||
resolvedDir, err := filepath.EvalSymlinks(filepath.Dir(candidate))
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: resolve include directory %q: %w", pattern, err)
|
||||
}
|
||||
if _, err := s.withinRoot(resolvedDir); err != nil {
|
||||
return err
|
||||
}
|
||||
matches, err := boundedGlob(candidate)
|
||||
if err != nil {
|
||||
return fmt.Errorf("agentwatch: include glob %q: %w", pattern, err)
|
||||
}
|
||||
for _, match := range matches {
|
||||
if err := s.visit(match, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err := s.visit(candidate, depth+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func readBounded(r io.Reader, limit int64) ([]byte, error) {
|
||||
content, err := io.ReadAll(io.LimitReader(r, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(content)) > limit {
|
||||
return nil, fmt.Errorf("configuration tree changed or exceeded size limit")
|
||||
}
|
||||
return content, nil
|
||||
}
|
||||
|
||||
// boundedGlob only handles * and ? in a filename. Broader BIRD glob grammar
|
||||
// falls back to every-tick configure instead of risking a false unchanged hash.
|
||||
func boundedGlob(pattern string) ([]string, error) {
|
||||
dir, base := filepath.Dir(pattern), filepath.Base(pattern)
|
||||
f, err := os.Open(dir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
names, err := f.Readdirnames(maxDirectoryEntries + 1)
|
||||
if err != nil && !errors.Is(err, io.EOF) {
|
||||
return nil, err
|
||||
}
|
||||
if len(names) > maxDirectoryEntries {
|
||||
return nil, fmt.Errorf("directory exceeds %d entries", maxDirectoryEntries)
|
||||
}
|
||||
matches := make([]string, 0)
|
||||
for _, name := range names {
|
||||
match, err := filepath.Match(base, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if match {
|
||||
matches = append(matches, filepath.Join(dir, name))
|
||||
if len(matches) > maxConfigFiles {
|
||||
return nil, fmt.Errorf("glob exceeds %d matches", maxConfigFiles)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Strings(matches)
|
||||
return matches, nil
|
||||
}
|
||||
|
||||
func (s *configScanner) withinRoot(path string) (string, error) {
|
||||
rel, err := filepath.Rel(s.root, path)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return "", fmt.Errorf("agentwatch: include %q escapes active directory", path)
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func writeHash(h hash.Hash, name string, content []byte) {
|
||||
var size [8]byte
|
||||
binary.LittleEndian.PutUint64(size[:], uint64(len(name)))
|
||||
_, _ = h.Write(size[:])
|
||||
_, _ = h.Write([]byte(name))
|
||||
binary.LittleEndian.PutUint64(size[:], uint64(len(content)))
|
||||
_, _ = h.Write(size[:])
|
||||
_, _ = h.Write(content)
|
||||
}
|
||||
|
||||
func includePatterns(content string) ([]string, error) {
|
||||
tokens, unsupportedQuotedText := includeTokenOffsets(content)
|
||||
if unsupportedQuotedText {
|
||||
return nil, fmt.Errorf("unsupported quoted text or escape")
|
||||
}
|
||||
tokenSet := make(map[int]bool, len(tokens))
|
||||
for _, token := range tokens {
|
||||
tokenSet[token] = true
|
||||
}
|
||||
var patterns []string
|
||||
for _, match := range includeLine.FindAllStringSubmatchIndex(content, -1) {
|
||||
keyword := match[0] + strings.Index(content[match[0]:match[1]], "include")
|
||||
if !tokenSet[keyword] {
|
||||
continue
|
||||
} // text inside a block comment
|
||||
delete(tokenSet, keyword)
|
||||
patterns = append(patterns, content[match[2]:match[3]])
|
||||
}
|
||||
if len(tokenSet) != 0 {
|
||||
return nil, fmt.Errorf("unsupported include syntax")
|
||||
}
|
||||
return patterns, nil
|
||||
}
|
||||
|
||||
// Recognize include keywords outside BIRD comments and quoted strings.
|
||||
func includeTokenOffsets(config string) ([]int, bool) {
|
||||
var offsets []int
|
||||
quote := byte(0)
|
||||
lineComment, blockComment := false, false
|
||||
for i := 0; i < len(config); {
|
||||
c := config[i]
|
||||
if lineComment {
|
||||
if c == '\n' {
|
||||
lineComment = false
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if blockComment {
|
||||
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
|
||||
blockComment = false
|
||||
i += 2
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if quote != 0 {
|
||||
if c == '\\' {
|
||||
return offsets, true
|
||||
}
|
||||
if c == quote {
|
||||
quote = 0
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '#' || (c == '/' && i+1 < len(config) && config[i+1] == '/') {
|
||||
lineComment = true
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
|
||||
blockComment = true
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if c == '"' || c == '\'' {
|
||||
quote = c
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
|
||||
start := i
|
||||
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
|
||||
i++
|
||||
}
|
||||
if config[start:i] == "include" {
|
||||
offsets = append(offsets, start)
|
||||
}
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
return offsets, quote != 0 || blockComment
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
// Package agentwatch reloads BIRD when its active configuration tree changes.
|
||||
package agentwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"path/filepath"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Control is the subset of BIRD operations required by the watch loop.
|
||||
type Control interface {
|
||||
ParseCheck(context.Context, string) error
|
||||
Configure(context.Context) error
|
||||
}
|
||||
|
||||
// Config sets bounded polling and reconciliation behavior.
|
||||
type Config struct {
|
||||
ActiveDir string
|
||||
Every time.Duration
|
||||
ReconcileEvery time.Duration
|
||||
CommandTimeout time.Duration
|
||||
Logf func(string, ...any)
|
||||
}
|
||||
|
||||
type watcher struct {
|
||||
ctl Control
|
||||
cfg Config
|
||||
hash [sha256.Size]byte
|
||||
haveHash bool
|
||||
lastConfigured time.Time
|
||||
}
|
||||
|
||||
// Run starts with one check, then polls until ctx is cancelled. A successful
|
||||
// birdc exit advances the remembered hash; it is not proof of BIRD ACK.
|
||||
func Run(ctx context.Context, ctl Control, cfg Config) error {
|
||||
if ctl == nil || cfg.Every <= 0 || cfg.ReconcileEvery <= 0 || cfg.CommandTimeout <= 0 {
|
||||
return fmt.Errorf("agentwatch: control and positive intervals/timeouts are required")
|
||||
}
|
||||
if cfg.ActiveDir == "" {
|
||||
cfg.ActiveDir = "/etc/bird"
|
||||
}
|
||||
if cfg.Logf == nil {
|
||||
cfg.Logf = log.Printf
|
||||
}
|
||||
w := &watcher{ctl: ctl, cfg: cfg}
|
||||
if err := w.step(ctx, time.Now()); err != nil && ctx.Err() == nil {
|
||||
cfg.Logf("evobgp-agent watch: %v", err)
|
||||
}
|
||||
ticker := time.NewTicker(cfg.Every)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case now := <-ticker.C:
|
||||
if err := w.step(ctx, now); err != nil && ctx.Err() == nil {
|
||||
cfg.Logf("evobgp-agent watch: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *watcher) step(ctx context.Context, now time.Time) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
digest, scanErr := hashConfigTree(ctx, w.cfg.ActiveDir)
|
||||
if scanErr != nil {
|
||||
if errors.Is(scanErr, errMainNotReady) {
|
||||
return fmt.Errorf("configuration not ready: %w", scanErr)
|
||||
}
|
||||
// Unsupported/unhashable trees retain the old every-tick configure cadence.
|
||||
w.cfg.Logf("evobgp-agent watch: cannot hash complete config tree (%v); using every-tick parse/configure", scanErr)
|
||||
} else if w.haveHash && digest == w.hash && now.Sub(w.lastConfigured) < w.cfg.ReconcileEvery {
|
||||
return nil
|
||||
}
|
||||
main := filepath.Join(w.cfg.ActiveDir, "bird.conf")
|
||||
if err := w.command(ctx, func(cctx context.Context) error { return w.ctl.ParseCheck(cctx, main) }); err != nil {
|
||||
return fmt.Errorf("parse check: %w", err)
|
||||
}
|
||||
if scanErr == nil {
|
||||
stable, err := hashConfigTree(ctx, w.cfg.ActiveDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("configuration changed during parse check: %w", err)
|
||||
}
|
||||
if stable != digest {
|
||||
return fmt.Errorf("configuration changed during parse check; retrying")
|
||||
}
|
||||
}
|
||||
if err := w.command(ctx, w.ctl.Configure); err != nil {
|
||||
return fmt.Errorf("configure: %w", err)
|
||||
}
|
||||
w.haveHash = scanErr == nil
|
||||
if w.haveHash {
|
||||
w.hash = digest
|
||||
}
|
||||
w.lastConfigured = now
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *watcher) command(parent context.Context, run func(context.Context) error) error {
|
||||
ctx, cancel := context.WithTimeout(parent, w.cfg.CommandTimeout)
|
||||
defer cancel()
|
||||
return run(ctx)
|
||||
}
|
||||
@@ -0,0 +1,311 @@
|
||||
package agentwatch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fakeControl struct {
|
||||
parsed int
|
||||
configured int
|
||||
failConfigure int
|
||||
onParse func()
|
||||
}
|
||||
|
||||
func (f *fakeControl) ParseCheck(ctx context.Context, _ string) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
f.parsed++
|
||||
if f.onParse != nil {
|
||||
f.onParse()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeControl) Configure(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
f.configured++
|
||||
if f.failConfigure > 0 {
|
||||
f.failConfigure--
|
||||
return errors.New("birdc unavailable")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func testConfig(t *testing.T) (string, string) {
|
||||
t.Helper()
|
||||
root := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(root, "bird.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main := filepath.Join(root, "bird.conf")
|
||||
if err := os.WriteFile(main, []byte("router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fragment := filepath.Join(root, "bird.d", "peer.conf")
|
||||
if err := os.WriteFile(fragment, []byte("# peer v1\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return root, fragment
|
||||
}
|
||||
|
||||
func testWatcher(root string, ctl Control) *watcher {
|
||||
return &watcher{ctl: ctl, cfg: Config{ActiveDir: root, Every: 30 * time.Second, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}}}
|
||||
}
|
||||
|
||||
func TestHashConfigTreeTracksMainIncludesAndGlobMembership(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
first, err := hashConfigTree(context.Background(), root)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(fragment, []byte("# peer v2\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := hashConfigTree(context.Background(), root)
|
||||
if err != nil || second == first {
|
||||
t.Fatalf("included edit not detected: digest=%x err=%v", second, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.d", "manual.conf"), []byte("# manual\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
third, err := hashConfigTree(context.Background(), root)
|
||||
if err != nil || third == second {
|
||||
t.Fatalf("new glob member not detected: digest=%x err=%v", third, err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fourth, err := hashConfigTree(context.Background(), root)
|
||||
if err != nil || fourth == third {
|
||||
t.Fatalf("main edit not detected: digest=%x err=%v", fourth, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchSkipsUnchangedRetriesErrorsAndForcesReconciliation(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
ctl := &fakeControl{}
|
||||
w := testWatcher(root, ctl)
|
||||
now := time.Now()
|
||||
if err := w.step(context.Background(), now); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.step(context.Background(), now.Add(30*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctl.configured != 1 || ctl.parsed != 1 {
|
||||
t.Fatalf("unchanged tree configured again: %+v", ctl)
|
||||
}
|
||||
if err := os.WriteFile(fragment, []byte("# changed\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl.failConfigure = 1
|
||||
if err := w.step(context.Background(), now.Add(time.Minute)); err == nil {
|
||||
t.Fatal("expected configure failure")
|
||||
}
|
||||
if !w.haveHash || ctl.configured != 2 {
|
||||
t.Fatalf("failure changed remembered state: %+v", w)
|
||||
}
|
||||
if err := w.step(context.Background(), now.Add(90*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctl.configured != 3 {
|
||||
t.Fatalf("failed change was not retried: %+v", ctl)
|
||||
}
|
||||
if err := w.step(context.Background(), now.Add(7*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctl.configured != 4 {
|
||||
t.Fatalf("forced reconciliation missing: %+v", ctl)
|
||||
}
|
||||
// A fresh watcher must reconcile once even when another process already loaded the same bytes.
|
||||
restarted := testWatcher(root, ctl)
|
||||
if err := restarted.step(context.Background(), now.Add(8*time.Minute)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctl.configured != 5 {
|
||||
t.Fatalf("restart did not reconcile: %+v", ctl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchRejectsChangingTreeBeforeConfigure(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
ctl := &fakeControl{onParse: func() { _ = os.WriteFile(fragment, []byte("# raced\n"), 0o644) }}
|
||||
w := testWatcher(root, ctl)
|
||||
if err := w.step(context.Background(), time.Now()); err == nil || !strings.Contains(err.Error(), "changed during parse check") {
|
||||
t.Fatalf("want race retry, got %v", err)
|
||||
}
|
||||
if ctl.configured != 0 {
|
||||
t.Fatal("configured a tree that changed after parse check")
|
||||
}
|
||||
ctl.onParse = nil
|
||||
if err := w.step(context.Background(), time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ctl.configured != 1 {
|
||||
t.Fatal("stable tree was not configured")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedIncludeUsesLegacyEveryTickCadence(t *testing.T) {
|
||||
root, _ := testConfig(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include variable;\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl := &fakeControl{}
|
||||
w := testWatcher(root, ctl)
|
||||
now := time.Now()
|
||||
for i := 0; i < 3; i++ {
|
||||
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if ctl.parsed != 3 || ctl.configured != 3 || w.haveHash {
|
||||
t.Fatalf("unsupported tree was skipped: %+v", ctl)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMissingIncludedFileFallsBackButMissingMainWaits(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/peer.conf\";\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Remove(fragment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl := &fakeControl{}
|
||||
w := testWatcher(root, ctl)
|
||||
now := time.Now()
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := w.step(context.Background(), now.Add(time.Duration(i)*30*time.Second)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if ctl.configured != 2 {
|
||||
t.Fatalf("missing include skipped legacy cadence: %+v", ctl)
|
||||
}
|
||||
if err := os.Remove(filepath.Join(root, "bird.conf")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := w.step(context.Background(), now.Add(time.Minute)); !errors.Is(err, errMainNotReady) {
|
||||
t.Fatalf("missing main should wait, got %v", err)
|
||||
}
|
||||
if ctl.configured != 2 {
|
||||
t.Fatal("configured without main bird.conf")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashConfigTreeBoundsAndEscapes(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
if err := os.WriteFile(fragment, make([]byte, maxFileBytes+1), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("want size limit, got %v", err)
|
||||
}
|
||||
if err := os.WriteFile(fragment, []byte("# small\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outside := filepath.Join(t.TempDir(), "outside.conf")
|
||||
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \""+filepath.ToSlash(outside)+"\";\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "escapes active directory") {
|
||||
t.Fatalf("want outside include rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHashConfigTreeRejectsSymlinkInclude(t *testing.T) {
|
||||
root, fragment := testConfig(t)
|
||||
if err := os.Remove(fragment); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
outside := filepath.Join(t.TempDir(), "outside.conf")
|
||||
if err := os.WriteFile(outside, []byte("# outside\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outside, fragment); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "not a regular file") {
|
||||
t.Fatalf("want symlink rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncludeCommentsAndUnsupportedForms(t *testing.T) {
|
||||
patterns, err := includePatterns("# include \"ignored.conf\";\n/*\ninclude \"also-ignored.conf\";\n*/\ninclude \"bird.d/*.conf\";\n")
|
||||
if err != nil || len(patterns) != 1 || patterns[0] != "bird.d/*.conf" {
|
||||
t.Fatalf("comment handling failed: patterns=%v err=%v", patterns, err)
|
||||
}
|
||||
if _, err := includePatterns("if true then include \"dynamic.conf\";\n"); err == nil {
|
||||
t.Fatal("inline include should fall back to legacy cadence")
|
||||
}
|
||||
if _, err := includePatterns("include \"bird.d/[!a].conf\";\n"); err != nil {
|
||||
// Bracket glob is rejected by the scanner after lexical parsing.
|
||||
t.Fatalf("literal include statement should parse before glob validation: %v", err)
|
||||
}
|
||||
root, _ := testConfig(t)
|
||||
if err := os.WriteFile(filepath.Join(root, "bird.conf"), []byte("include \"bird.d/[!a].conf\";\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := hashConfigTree(context.Background(), root); err == nil || !strings.Contains(err.Error(), "unsupported") {
|
||||
t.Fatalf("bracket glob should fall back, got %v", err)
|
||||
}
|
||||
if _, err := includePatterns("define note = \"single\\\";\ninclude \"bird.d/real.conf\";\n"); err == nil {
|
||||
t.Fatal("backslash in quoted text must force conservative fallback")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBoundedGlobLimitsMatches(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
for i := 0; i <= maxConfigFiles; i++ {
|
||||
name := filepath.Join(root, fmt.Sprintf("peer-%03d.conf", i))
|
||||
if err := os.WriteFile(name, nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := boundedGlob(filepath.Join(root, "*.conf")); err == nil || !strings.Contains(err.Error(), "exceeds") {
|
||||
t.Fatalf("want bounded glob rejection, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadBoundedStopsAfterLimit(t *testing.T) {
|
||||
r := strings.NewReader("abcdefghij")
|
||||
if _, err := readBounded(r, 3); err == nil {
|
||||
t.Fatal("growing file exceeded limit")
|
||||
}
|
||||
if got := r.Len(); got != 6 {
|
||||
t.Fatalf("reader consumed beyond limit+1: %d bytes remain", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunStopsOnCancellation(t *testing.T) {
|
||||
root, _ := testConfig(t)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
ctl := &fakeControl{}
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- Run(ctx, ctl, Config{ActiveDir: root, Every: time.Hour, ReconcileEvery: 5 * time.Minute, CommandTimeout: time.Second, Logf: func(string, ...any) {}})
|
||||
}()
|
||||
cancel()
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("watch did not stop after cancellation")
|
||||
}
|
||||
}
|
||||
@@ -23,6 +23,8 @@ const DefaultASOverviewURL = "https://stat.ripe.net/data/as-overview/data.json"
|
||||
|
||||
// AnnouncedPrefixes returns currently announced IPv4/IPv6 prefixes for the ASN (best-effort via RIPEstat).
|
||||
func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip.Prefix, error) {
|
||||
ctx, cancel := ripeRequestContext(ctx)
|
||||
defer cancel()
|
||||
if hc == nil {
|
||||
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||
}
|
||||
@@ -43,13 +45,14 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
||||
return nil, fmt.Errorf("ripestat fetch AS%d: %w", asn, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return nil, fmt.Errorf("ripestat AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
||||
}
|
||||
body, err := httpclient.ReadResponseBody(resp, 32<<20)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("ripestat AS%d response: %w", asn, err)
|
||||
}
|
||||
|
||||
var wrap struct {
|
||||
Status string `json:"status"`
|
||||
@@ -85,6 +88,8 @@ func AnnouncedPrefixes(ctx context.Context, hc *http.Client, asn int64) ([]netip
|
||||
|
||||
// ASHolderName returns the holder / organization label for the ASN from RIPEstat as-overview (best-effort).
|
||||
func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, error) {
|
||||
ctx, cancel := ripeRequestContext(ctx)
|
||||
defer cancel()
|
||||
if hc == nil {
|
||||
hc = httpclient.New(httpclient.DefaultTimeout)
|
||||
}
|
||||
@@ -105,13 +110,14 @@ func ASHolderName(ctx context.Context, hc *http.Client, asn int64) (string, erro
|
||||
return "", fmt.Errorf("ripestat as-overview AS%d: %w", asn, err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return "", fmt.Errorf("ripestat as-overview AS%d: HTTP %s: %s", asn, resp.Status, truncateForErr(body, 200))
|
||||
}
|
||||
body, err := httpclient.ReadResponseBody(resp, 4<<20)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("ripestat as-overview AS%d response: %w", asn, err)
|
||||
}
|
||||
|
||||
var wrap struct {
|
||||
Status string `json:"status"`
|
||||
@@ -135,3 +141,10 @@ func truncateForErr(b []byte, n int) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func ripeRequestContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
if _, ok := ctx.Deadline(); ok {
|
||||
return context.WithCancel(ctx)
|
||||
}
|
||||
return context.WithTimeout(ctx, httpclient.DefaultTimeout)
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package asnresolve
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/httpclient"
|
||||
)
|
||||
|
||||
type ripeRoundTrip func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f ripeRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type ripeCountingBody struct {
|
||||
io.Reader
|
||||
read int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *ripeCountingBody) Read(p []byte) (int, error) {
|
||||
n, e := b.Reader.Read(p)
|
||||
b.read += n
|
||||
return n, e
|
||||
}
|
||||
func (b *ripeCountingBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
func TestRIPECompleteResponseLimits(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
limit int
|
||||
prefix string
|
||||
call func(context.Context, *http.Client) error
|
||||
}{
|
||||
{"announced", 32 << 20, `{"status":"ok","data":{"prefixes":[]}}`, func(ctx context.Context, hc *http.Client) error { _, e := AnnouncedPrefixes(ctx, hc, 64496); return e }},
|
||||
{"holder", 4 << 20, `{"status":"ok","data":{"holder":"Example"}}`, func(ctx context.Context, hc *http.Client) error { _, e := ASHolderName(ctx, hc, 64496); return e }},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
for _, known := range []bool{true, false} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
body := &ripeCountingBody{Reader: strings.NewReader(tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra))}
|
||||
hc := &http.Client{Transport: ripeRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||
if _, ok := r.Context().Deadline(); !ok {
|
||||
t.Error("missing request deadline")
|
||||
}
|
||||
length := int64(-1)
|
||||
if known {
|
||||
length = int64(tc.limit + extra)
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Body: body, ContentLength: length, Header: make(http.Header)}, nil
|
||||
})}
|
||||
err := tc.call(context.Background(), hc)
|
||||
if extra == 0 && err != nil {
|
||||
t.Fatalf("exact limit rejected known=%v: %v", known, err)
|
||||
}
|
||||
if extra == 1 && !errors.Is(err, httpclient.ErrResponseTooLarge) {
|
||||
t.Fatalf("oversized valid prefix accepted known=%v: %v", known, err)
|
||||
}
|
||||
if body.read > tc.limit+1 || !body.closed {
|
||||
t.Fatalf("read=%d closed=%v", body.read, body.closed)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRIPEErrorBodySmallAndContextPreserved(t *testing.T) {
|
||||
body := &ripeCountingBody{Reader: strings.NewReader(strings.Repeat("x", 1<<20))}
|
||||
hc := &http.Client{Transport: ripeRoundTrip(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 400, Status: "400 Bad Request", Body: body, Header: make(http.Header)}, nil
|
||||
})}
|
||||
if _, err := AnnouncedPrefixes(context.Background(), hc, 1); err == nil || body.read > 4<<10 || !body.closed {
|
||||
t.Fatalf("non200 unbounded read=%d close=%v err=%v", body.read, body.closed, err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
|
||||
defer cancel()
|
||||
hc.Transport = ripeRoundTrip(func(r *http.Request) (*http.Response, error) { <-r.Context().Done(); return nil, r.Context().Err() })
|
||||
if _, err := ASHolderName(ctx, hc, 1); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("caller deadline lost: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRIPEGzipExpandedLimit(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
gz := gzip.NewWriter(w)
|
||||
_, _ = io.WriteString(gz, `{"status":"ok","data":{"holder":"Example"}}`+strings.Repeat(" ", 4<<20))
|
||||
_ = gz.Close()
|
||||
}))
|
||||
defer server.Close()
|
||||
t.Setenv("EVOBGP_RIPESTAT_AS_OVERVIEW_URL", server.URL)
|
||||
if _, err := ASHolderName(context.Background(), server.Client(), 1); !errors.Is(err, httpclient.ErrResponseTooLarge) {
|
||||
t.Fatalf("gzip bypassed decoded limit: %v", err)
|
||||
}
|
||||
}
|
||||
+382
-101
@@ -1,152 +1,433 @@
|
||||
// Package birddeploy applies revision preview fragments to a shared BIRD volume with parse-check and LKG (plan §13).
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// Config drives two-phase deploy from a revision in the store.
|
||||
const recoveryTimeout = 30 * time.Second
|
||||
|
||||
var includeLine = regexp.MustCompile(`(?m)^([ \t]*include[ \t]+")([^"]+)("[ \t]*;[ \t]*(?:#[^\n]*)?$)`)
|
||||
|
||||
// Config identifies the BIRD instance. StagingDir remains for caller compatibility;
|
||||
// generations live inside ActiveDir so referenced files survive process restart.
|
||||
type Config struct {
|
||||
ActiveDir string // e.g. /etc/bird (live config tree)
|
||||
StagingDir string // e.g. /tmp/evobgp-staging (same host/volume as ActiveDir)
|
||||
ActiveDir string
|
||||
StagingDir string
|
||||
BirdBin string
|
||||
BirdcBin string
|
||||
Socket string // birdc -s
|
||||
Socket string
|
||||
writeFile func(string, []byte) error // test-only fault injection; nil uses synced writes
|
||||
}
|
||||
|
||||
// ApplyRevision writes preview fragments to staging, runs bird -p, swaps into active, runs birdc configure; restores LKG on failure.
|
||||
// ApplyRevision installs a checked generation through one atomic bird.conf rename.
|
||||
func ApplyRevision(ctx context.Context, ctl *birdfmt.BirdCtl, rev *store.Revision, cfg Config) error {
|
||||
if rev == nil || len(rev.PreviewFragments) == 0 {
|
||||
return fmt.Errorf("birddeploy: no preview fragments")
|
||||
}
|
||||
active := strings.TrimSpace(cfg.ActiveDir)
|
||||
staging := strings.TrimSpace(cfg.StagingDir)
|
||||
if active == "" || staging == "" {
|
||||
return fmt.Errorf("birddeploy: ActiveDir and StagingDir required")
|
||||
if strings.TrimSpace(cfg.ActiveDir) == "" {
|
||||
return fmt.Errorf("birddeploy: ActiveDir required")
|
||||
}
|
||||
if err := os.MkdirAll(staging, 0o755); err != nil {
|
||||
return err
|
||||
active, err := filepath.Abs(cfg.ActiveDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: resolve active directory: %w", err)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||
return err
|
||||
if err := os.MkdirAll(active, 0o755); err != nil {
|
||||
return fmt.Errorf("birddeploy: create active directory: %w", err)
|
||||
}
|
||||
lkg := filepath.Join(active, ".evobgp_lkg")
|
||||
|
||||
// Phase 1: write staging tree
|
||||
for rel, content := range rev.PreviewFragments {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
if !isDeployableBirdFragment(rel) {
|
||||
continue
|
||||
lock, err := lockTarget(ctx, filepath.Join(active, ".evobgp-apply.lock"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: acquire apply lock: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := lock.Close(); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "birddeploy: release apply lock: %v\n", err)
|
||||
}
|
||||
dst := filepath.Join(staging, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dst, []byte(content), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
mainConf := filepath.Join(staging, "bird.conf")
|
||||
if _, err := os.Stat(mainConf); err != nil {
|
||||
return fmt.Errorf("birddeploy: staging missing bird.conf: %w", err)
|
||||
}()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fmt.Errorf("birddeploy: apply cancelled: %w", err)
|
||||
}
|
||||
bc := ctl
|
||||
if bc == nil {
|
||||
bc = &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
||||
}
|
||||
if err := bc.ParseCheck(ctx, mainConf); err != nil {
|
||||
return fmt.Errorf("birddeploy: bird -p failed: %w", err)
|
||||
writeFile := cfg.writeFile
|
||||
if writeFile == nil {
|
||||
writeFile = writeSynced
|
||||
}
|
||||
|
||||
// Snapshot LKG (best-effort copy of current active bird.conf + bird.d)
|
||||
_ = os.RemoveAll(lkg)
|
||||
_ = snapshotBirdTree(active, lkg)
|
||||
|
||||
// Phase 2: atomic swap into active
|
||||
for rel, content := range rev.PreviewFragments {
|
||||
rel = strings.TrimPrefix(rel, "/")
|
||||
if !isDeployableBirdFragment(rel) {
|
||||
continue
|
||||
mainPath := filepath.Join(active, "bird.conf")
|
||||
mainInfo, err := os.Lstat(mainPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: inspect current bird.conf: %w", err)
|
||||
}
|
||||
if !mainInfo.Mode().IsRegular() {
|
||||
return fmt.Errorf("birddeploy: current bird.conf must be a regular file; refusing to replace %s", mainInfo.Mode())
|
||||
}
|
||||
oldMain, err := os.ReadFile(mainPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: read current bird.conf: %w", err)
|
||||
}
|
||||
if err := bc.ParseCheck(ctx, mainPath); err != nil {
|
||||
return fmt.Errorf("birddeploy: current configuration is not a usable LKG: %w", err)
|
||||
}
|
||||
gen, err := os.MkdirTemp(active, ".evobgp-generation-")
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: create generation: %w", err)
|
||||
}
|
||||
activated := false
|
||||
defer func() {
|
||||
if !activated {
|
||||
if err := os.RemoveAll(gen); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "birddeploy: remove unactivated generation %s: %v\n", gen, err)
|
||||
}
|
||||
}
|
||||
dst := filepath.Join(active, rel)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
_ = restoreLKG(lkg, active)
|
||||
return err
|
||||
}
|
||||
tmp := dst + ".tmp"
|
||||
if err := os.WriteFile(tmp, []byte(content), 0o644); err != nil {
|
||||
_ = restoreLKG(lkg, active)
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
_ = restoreLKG(lkg, active)
|
||||
return err
|
||||
}()
|
||||
if err := writeGenerationWith(active, gen, rev.PreviewFragments, writeFile); err != nil {
|
||||
return err
|
||||
}
|
||||
newMain, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("birddeploy: read generated bird.conf: %w", err)
|
||||
}
|
||||
if err := bc.ParseCheck(ctx, filepath.Join(gen, "bird.conf")); err != nil {
|
||||
return fmt.Errorf("birddeploy: staged bird -p failed: %w", err)
|
||||
}
|
||||
if err := writeFile(filepath.Join(gen, "previous-bird.conf"), oldMain); err != nil {
|
||||
return fmt.Errorf("birddeploy: save LKG: %w", err)
|
||||
}
|
||||
if err := syncDir(gen); err != nil {
|
||||
return fmt.Errorf("birddeploy: sync LKG generation: %w", err)
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return fmt.Errorf("birddeploy: apply cancelled: %w", err)
|
||||
}
|
||||
swapped, err := replaceMain(active, newMain)
|
||||
activated = swapped
|
||||
if err != nil {
|
||||
if swapped {
|
||||
return recoverPrevious(bc, active, gen, oldMain, fmt.Errorf("activate generation: %w", err))
|
||||
}
|
||||
return fmt.Errorf("birddeploy: activate generation: %w", err)
|
||||
}
|
||||
if err := bc.Configure(ctx); err != nil {
|
||||
_ = restoreLKG(lkg, active)
|
||||
_ = bc.Configure(ctx)
|
||||
return fmt.Errorf("birddeploy: birdc configure failed, restored LKG: %w", err)
|
||||
return recoverPrevious(bc, active, gen, oldMain, err)
|
||||
}
|
||||
if err := pruneGenerations(active, gen, oldMain); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "birddeploy: configuration applied; generation cleanup failed: %v\n", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// isDeployableBirdFragment skips UI-only preview keys (e.g. flattened full config) not referenced from bird.conf.
|
||||
func isDeployableBirdFragment(rel string) bool {
|
||||
return rel != "" && !strings.HasPrefix(rel, "_")
|
||||
func recoverPrevious(bc *birdfmt.BirdCtl, active, gen string, oldMain []byte, cause error) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), recoveryTimeout)
|
||||
defer cancel()
|
||||
if _, err := replaceMain(active, oldMain); err != nil {
|
||||
return fmt.Errorf("birddeploy: configure failed: %w; restore failed: %v", cause, err)
|
||||
}
|
||||
if err := bc.Configure(ctx); err != nil {
|
||||
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored on disk but recovery configure failed: %v", cause, err)
|
||||
}
|
||||
if err := os.RemoveAll(gen); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "birddeploy: LKG restored; remove failed generation %s: %v\n", gen, err)
|
||||
}
|
||||
return fmt.Errorf("birddeploy: configure failed: %w; LKG restored and recovery configure succeeded", cause)
|
||||
}
|
||||
|
||||
func snapshotBirdTree(activeRoot, dstRoot string) error {
|
||||
if err := os.MkdirAll(dstRoot, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
mc := filepath.Join(activeRoot, "bird.conf")
|
||||
if b, err := os.ReadFile(mc); err == nil {
|
||||
_ = os.WriteFile(filepath.Join(dstRoot, "bird.conf"), b, 0o644)
|
||||
}
|
||||
bd := filepath.Join(activeRoot, "bird.d")
|
||||
if st, err := os.Stat(bd); err == nil && st.IsDir() {
|
||||
_ = filepath.WalkDir(bd, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
suffix, _ := filepath.Rel(activeRoot, path)
|
||||
out := filepath.Join(dstRoot, suffix)
|
||||
_ = os.MkdirAll(filepath.Dir(out), 0o755)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
return os.WriteFile(out, b, 0o644)
|
||||
})
|
||||
}
|
||||
return nil
|
||||
func writeGeneration(active, gen string, fragments map[string]string) error {
|
||||
return writeGenerationWith(active, gen, fragments, writeSynced)
|
||||
}
|
||||
|
||||
func restoreLKG(lkg, active string) error {
|
||||
if _, err := os.Stat(filepath.Join(lkg, "bird.conf")); err != nil {
|
||||
return err
|
||||
func writeGenerationWith(active, gen string, fragments map[string]string, writeFile func(string, []byte) error) error {
|
||||
main, ok := fragments["bird.conf"]
|
||||
if !ok {
|
||||
return fmt.Errorf("birddeploy: preview missing bird.conf")
|
||||
}
|
||||
_ = filepath.WalkDir(lkg, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil || d.IsDir() {
|
||||
return err
|
||||
}
|
||||
suffix, _ := filepath.Rel(lkg, path)
|
||||
out := filepath.Join(active, suffix)
|
||||
_ = os.MkdirAll(filepath.Dir(out), 0o755)
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
oldBirdD := filepath.Join(active, birdfmt.DirBirdD)
|
||||
newBirdD := filepath.Join(gen, birdfmt.DirBirdD)
|
||||
if err := os.MkdirAll(newBirdD, 0o755); err != nil {
|
||||
return fmt.Errorf("birddeploy: create generation bird.d: %w", err)
|
||||
}
|
||||
if err := filepath.WalkDir(oldBirdD, func(path string, d fs.DirEntry, walkErr error) error {
|
||||
if errors.Is(walkErr, fs.ErrNotExist) && path == oldBirdD {
|
||||
return nil
|
||||
}
|
||||
return os.WriteFile(out, b, 0o644)
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
rel, err := filepath.Rel(oldBirdD, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.Type()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("operator snippet %q is a symlink", rel)
|
||||
}
|
||||
if rel == "." {
|
||||
if !d.IsDir() {
|
||||
return fmt.Errorf("birddeploy: bird.d is not a directory")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(filepath.Join(newBirdD, rel), 0o755)
|
||||
}
|
||||
if !d.Type().IsRegular() {
|
||||
return fmt.Errorf("operator snippet %q is not a regular file", rel)
|
||||
}
|
||||
_, replaced := fragments[filepath.ToSlash(filepath.Join(birdfmt.DirBirdD, rel))]
|
||||
if strings.HasPrefix(filepath.Base(rel), "evobgp_") || replaced {
|
||||
return nil
|
||||
}
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if countIncludeTokens(string(b)) > 0 {
|
||||
return fmt.Errorf("operator snippet %q has a nested include; cannot pin the complete tree", rel)
|
||||
}
|
||||
return writeFile(filepath.Join(newBirdD, rel), b)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("birddeploy: copy operator snippets: %w", err)
|
||||
}
|
||||
for rel, content := range fragments {
|
||||
if rel == "bird.conf" || strings.HasPrefix(rel, "_") {
|
||||
continue
|
||||
}
|
||||
if countIncludeTokens(content) > 0 {
|
||||
return fmt.Errorf("birddeploy: nested include in preview fragment %q is unsupported", rel)
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(rel))
|
||||
if filepath.IsAbs(clean) || clean == "." || clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) || filepath.Ext(clean) != ".conf" {
|
||||
return fmt.Errorf("birddeploy: unsafe preview path %q", rel)
|
||||
}
|
||||
dst := filepath.Join(gen, clean)
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return fmt.Errorf("birddeploy: create fragment directory: %w", err)
|
||||
}
|
||||
if err := writeFile(dst, []byte(content)); err != nil {
|
||||
return fmt.Errorf("birddeploy: write %s: %w", rel, err)
|
||||
}
|
||||
}
|
||||
var rewriteErr error
|
||||
if !standaloneIncludesOnly(main) {
|
||||
return fmt.Errorf("birddeploy: unsupported include syntax in bird.conf")
|
||||
}
|
||||
rewritten := includeLine.ReplaceAllStringFunc(main, func(line string) string {
|
||||
m := includeLine.FindStringSubmatch(line)
|
||||
inc := m[2]
|
||||
if filepath.IsAbs(inc) {
|
||||
rel, err := filepath.Rel(active, inc)
|
||||
if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
rewriteErr = fmt.Errorf("birddeploy: external absolute include %q is unsupported", inc)
|
||||
return line
|
||||
}
|
||||
inc = rel
|
||||
}
|
||||
clean := filepath.Clean(filepath.FromSlash(inc))
|
||||
if clean == ".." || strings.HasPrefix(clean, ".."+string(os.PathSeparator)) || !strings.HasPrefix(clean, birdfmt.DirBirdD+string(os.PathSeparator)) {
|
||||
rewriteErr = fmt.Errorf("birddeploy: unsupported relative include %q", inc)
|
||||
return line
|
||||
}
|
||||
return m[1] + filepath.ToSlash(filepath.Join(gen, clean)) + m[3]
|
||||
})
|
||||
if rewriteErr != nil {
|
||||
return rewriteErr
|
||||
}
|
||||
if err := writeFile(filepath.Join(gen, ".evobgp-owned"), []byte("evobgp-generation-v1\n")); err != nil {
|
||||
return fmt.Errorf("birddeploy: mark generation: %w", err)
|
||||
}
|
||||
if err := writeFile(filepath.Join(gen, "bird.conf"), []byte(rewritten)); err != nil {
|
||||
return fmt.Errorf("birddeploy: write generated bird.conf: %w", err)
|
||||
}
|
||||
return syncTree(gen)
|
||||
}
|
||||
|
||||
func standaloneIncludesOnly(config string) bool {
|
||||
tokens := includeTokenOffsets(config)
|
||||
matches := includeLine.FindAllStringIndex(config, -1)
|
||||
if len(tokens) != len(matches) {
|
||||
return false
|
||||
}
|
||||
for i, match := range matches {
|
||||
word := strings.Index(config[match[0]:match[1]], "include")
|
||||
if word < 0 || tokens[i] != match[0]+word {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func countIncludeTokens(config string) int { return len(includeTokenOffsets(config)) }
|
||||
|
||||
// includeTokenOffsets skips BIRD comments and strings, retaining exact offsets
|
||||
// so a commented standalone include cannot mask a live inline include.
|
||||
func includeTokenOffsets(config string) []int {
|
||||
var offsets []int
|
||||
quote := byte(0)
|
||||
lineComment, blockComment := false, false
|
||||
for i := 0; i < len(config); {
|
||||
c := config[i]
|
||||
if lineComment {
|
||||
if c == '\n' {
|
||||
lineComment = false
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if blockComment {
|
||||
if c == '*' && i+1 < len(config) && config[i+1] == '/' {
|
||||
blockComment = false
|
||||
i += 2
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if quote != 0 {
|
||||
// BIRD quoted text does not use C-style backslash escapes.
|
||||
// Skipping the following quote could hide a live include.
|
||||
if c == quote {
|
||||
quote = 0
|
||||
}
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '#' {
|
||||
lineComment = true
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if c == '/' && i+1 < len(config) && config[i+1] == '*' {
|
||||
blockComment = true
|
||||
i += 2
|
||||
continue
|
||||
}
|
||||
if c == '"' || c == '\'' {
|
||||
quote = c
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' {
|
||||
start := i
|
||||
for i < len(config) && ((config[i] >= 'a' && config[i] <= 'z') || (config[i] >= 'A' && config[i] <= 'Z') || (config[i] >= '0' && config[i] <= '9') || config[i] == '_') {
|
||||
i++
|
||||
}
|
||||
if config[start:i] == "include" {
|
||||
offsets = append(offsets, start)
|
||||
}
|
||||
continue
|
||||
}
|
||||
i++
|
||||
}
|
||||
return offsets
|
||||
}
|
||||
|
||||
func syncTree(root string) error {
|
||||
var dirs []string
|
||||
if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if d.IsDir() {
|
||||
dirs = append(dirs, path)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
sort.Slice(dirs, func(i, j int) bool { return len(dirs[i]) > len(dirs[j]) })
|
||||
for _, dir := range dirs {
|
||||
if err := syncDir(dir); err != nil {
|
||||
return fmt.Errorf("sync %s: %w", dir, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pruneGenerations(active, current string, previous []byte) error {
|
||||
entries, err := os.ReadDir(active)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var stale []string
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || !strings.HasPrefix(entry.Name(), ".evobgp-generation-") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(active, entry.Name())
|
||||
if path == current || strings.Contains(string(previous), filepath.ToSlash(path)+"/") {
|
||||
continue
|
||||
}
|
||||
marker, err := os.ReadFile(filepath.Join(path, ".evobgp-owned"))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if string(marker) != "evobgp-generation-v1\n" {
|
||||
continue
|
||||
}
|
||||
stale = append(stale, path)
|
||||
}
|
||||
sort.Strings(stale)
|
||||
for _, path := range stale {
|
||||
if err := os.RemoveAll(path); err != nil {
|
||||
return fmt.Errorf("remove %s: %w", path, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replaceMain(active string, content []byte) (bool, error) {
|
||||
tmp, err := os.CreateTemp(active, ".evobgp-bird-conf-")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer func() { _ = os.Remove(tmp.Name()) }()
|
||||
if err := tmp.Chmod(0o644); err != nil {
|
||||
_ = tmp.Close()
|
||||
return false, err
|
||||
}
|
||||
if _, err := tmp.Write(content); err != nil {
|
||||
_ = tmp.Close()
|
||||
return false, err
|
||||
}
|
||||
if err := tmp.Sync(); err != nil {
|
||||
_ = tmp.Close()
|
||||
return false, err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.Rename(tmp.Name(), filepath.Join(active, "bird.conf")); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, syncDir(active)
|
||||
}
|
||||
|
||||
func writeSynced(path string, b []byte) error {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := f.Write(b); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
return f.Close()
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//go:build linux
|
||||
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func fakeCtl(t *testing.T, active string) *birdfmt.BirdCtl {
|
||||
t.Helper()
|
||||
bin := t.TempDir()
|
||||
parse := filepath.Join(bin, "bird")
|
||||
configure := filepath.Join(bin, "birdc")
|
||||
parseScript := `#!/bin/sh
|
||||
if grep -q FAIL_PARSE "$2"; then exit 1; fi
|
||||
exit 0
|
||||
`
|
||||
configureScript := fmt.Sprintf(`#!/bin/sh
|
||||
root=%q
|
||||
if [ -e "$root/.configure-wait" ]; then
|
||||
rm "$root/.configure-wait"
|
||||
touch "$root/.configure-started"
|
||||
exec sleep 5
|
||||
fi
|
||||
if [ -e "$root/.configure-fail-once" ]; then
|
||||
rm "$root/.configure-fail-once"
|
||||
exit 1
|
||||
fi
|
||||
mkdir "$root/.configuring" || exit 1
|
||||
sleep 0.05
|
||||
rmdir "$root/.configuring"
|
||||
`, active)
|
||||
for path, script := range map[string]string{parse: parseScript, configure: configureScript} {
|
||||
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return &birdfmt.BirdCtl{Bird: parse, Birdc: configure}
|
||||
}
|
||||
|
||||
func revision(label string) *store.Revision {
|
||||
return &store.Revision{PreviewFragments: map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/*.conf\";\n# " + label + "\n",
|
||||
"bird.d/evobgp_new.conf": "# " + label + "\n",
|
||||
}}
|
||||
}
|
||||
|
||||
func setupActive(t *testing.T) (string, *birdfmt.BirdCtl, []byte) {
|
||||
t.Helper()
|
||||
active := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
old := []byte("router id 192.0.2.2;\ninclude \"bird.d/*.conf\";\n# old\n")
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for name, content := range map[string]string{"evobgp_removed.conf": "# removed\n", "operator.conf": "# operator\n"} {
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.d", name), []byte(content), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return active, fakeCtl(t, active), old
|
||||
}
|
||||
|
||||
func TestApplyRevisionGenerationAndPrune(t *testing.T) {
|
||||
active, ctl, _ := setupActive(t)
|
||||
for _, label := range []string{"one", "two", "three"} {
|
||||
if err := ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(main), "# "+label) {
|
||||
t.Fatalf("wrong active generation: %s", main)
|
||||
}
|
||||
if strings.Contains(string(main), `include "bird.d/`) {
|
||||
t.Fatalf("include not pinned: %s", main)
|
||||
}
|
||||
}
|
||||
gens, err := filepath.Glob(filepath.Join(active, ".evobgp-generation-*"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(gens) != 2 {
|
||||
t.Fatalf("want active and previous generations, got %d", len(gens))
|
||||
}
|
||||
for _, gen := range gens {
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !errors.Is(err, os.ErrNotExist) {
|
||||
t.Fatalf("removed fragment retained in %s: %v", gen, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", "operator.conf")); err != nil {
|
||||
t.Fatalf("operator fragment missing: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionFaults(t *testing.T) {
|
||||
for _, mode := range []string{"parse", "write", "disk-full", "configure", "cancel"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
active, ctl, old := setupActive(t)
|
||||
rev := revision(mode)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
cfg := Config{ActiveDir: active}
|
||||
switch mode {
|
||||
case "parse":
|
||||
rev.PreviewFragments["bird.conf"] += "FAIL_PARSE\n"
|
||||
case "write":
|
||||
rev.PreviewFragments["../escape.conf"] = "bad"
|
||||
case "disk-full":
|
||||
cfg.writeFile = func(path string, b []byte) error {
|
||||
if filepath.Base(path) == "previous-bird.conf" {
|
||||
return syscall.ENOSPC
|
||||
}
|
||||
return writeSynced(path, b)
|
||||
}
|
||||
case "configure":
|
||||
if err := os.WriteFile(filepath.Join(active, ".configure-fail-once"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case "cancel":
|
||||
if err := os.WriteFile(filepath.Join(active, ".configure-wait"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
result := make(chan error, 1)
|
||||
go func() { result <- ApplyRevision(ctx, ctl, rev, cfg) }()
|
||||
if mode == "cancel" {
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".configure-started")); err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("configure did not start")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
err := <-result
|
||||
if err == nil {
|
||||
t.Fatal("expected apply failure")
|
||||
}
|
||||
main, readErr := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if string(main) != string(old) {
|
||||
t.Fatalf("old config not restored: %s; error: %v", main, err)
|
||||
}
|
||||
if mode == "configure" || mode == "cancel" {
|
||||
if !strings.Contains(err.Error(), "recovery configure succeeded") {
|
||||
t.Fatalf("unconfirmed recovery: %v", err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionSerializes(t *testing.T) {
|
||||
active, ctl, _ := setupActive(t)
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 2)
|
||||
for _, label := range []string{"first", "second"} {
|
||||
wg.Add(1)
|
||||
go func(label string) {
|
||||
defer wg.Done()
|
||||
errs <- ApplyRevision(context.Background(), ctl, revision(label), Config{ActiveDir: active})
|
||||
}(label)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLockHelperProcess(t *testing.T) {
|
||||
active := os.Getenv("EVOBGP_LOCK_HELPER_ACTIVE")
|
||||
if active == "" {
|
||||
return
|
||||
}
|
||||
lock, err := lockTarget(context.Background(), filepath.Join(active, ".evobgp-apply.lock"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = lock.Close() }()
|
||||
if err := os.WriteFile(filepath.Join(active, ".helper-ready"), nil, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".helper-done")); err == nil {
|
||||
return
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionCrossProcessLock(t *testing.T) {
|
||||
active, ctl, old := setupActive(t)
|
||||
cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelperProcess$")
|
||||
cmd.Env = append(os.Environ(), "EVOBGP_LOCK_HELPER_ACTIVE="+active)
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_ = os.WriteFile(filepath.Join(active, ".helper-done"), nil, 0o644)
|
||||
if err := cmd.Wait(); err != nil {
|
||||
t.Errorf("lock helper: %v", err)
|
||||
}
|
||||
}()
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
if _, err := os.Stat(filepath.Join(active, ".helper-ready")); err == nil {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("lock helper did not become ready")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := ApplyRevision(ctx, ctl, revision("blocked"), Config{ActiveDir: active}); !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("want lock timeout, got %v", err)
|
||||
}
|
||||
main, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(main) != string(old) {
|
||||
t.Fatalf("apply bypassed cross-process lock: %s", main)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRealBirdParseGeneration(t *testing.T) {
|
||||
bin := strings.TrimSpace(os.Getenv("EVOBGP_TEST_BIRD_BIN"))
|
||||
if bin == "" {
|
||||
t.Skip("EVOBGP_TEST_BIRD_BIN is not set")
|
||||
}
|
||||
active := t.TempDir()
|
||||
old := []byte("router id 192.0.2.2;\nprotocol device {}\n")
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctl := fakeCtl(t, active)
|
||||
ctl.Bird = bin
|
||||
rev := &store.Revision{PreviewFragments: map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\ninclude \"bird.d/evobgp_new.conf\";\nprotocol device {}\n",
|
||||
"bird.d/evobgp_new.conf": "# complete generated fragment\n",
|
||||
}}
|
||||
if err := ApplyRevision(context.Background(), ctl, rev, Config{ActiveDir: active}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ctl.ParseCheck(context.Background(), filepath.Join(active, "bird.conf")); err != nil {
|
||||
t.Fatalf("activated config fails real bird -p: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestWriteGenerationPinsCompleteIncludeTree(t *testing.T) {
|
||||
active := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte("# operator"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.d", "evobgp_removed.conf"), []byte("# removed"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fragments := map[string]string{"bird.conf": "include \"bird.d/*.conf\";\n", "bird.d/evobgp_new.conf": "# new"}
|
||||
if err := writeGeneration(active, gen, fragments); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
main, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(main), filepath.ToSlash(filepath.Join(gen, "bird.d", "*.conf"))) {
|
||||
t.Fatalf("include is not pinned to generation: %s", main)
|
||||
}
|
||||
for _, name := range []string{"operator.conf", "evobgp_new.conf"} {
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", name)); err != nil {
|
||||
t.Fatalf("%s missing: %v", name, err)
|
||||
}
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(gen, "bird.d", "evobgp_removed.conf")); !os.IsNotExist(err) {
|
||||
t.Fatalf("removed fragment retained: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteGenerationRejectsUnsafePaths(t *testing.T) {
|
||||
for _, bad := range []string{"../escape.conf", "bird.d/../../escape.conf", "other.conf"} {
|
||||
t.Run(bad, func(t *testing.T) {
|
||||
active := t.TempDir()
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", bad: "bad"}
|
||||
if err := writeGeneration(active, gen, fragments); err == nil {
|
||||
t.Fatal("unsafe path accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteGenerationRejectsExternalAndNestedIncludes(t *testing.T) {
|
||||
for _, main := range []string{
|
||||
`include "/outside/bird.d/peer.conf";` + "\n",
|
||||
`include "other/peer.conf";` + "\n",
|
||||
`include "bird.d/peer.conf"; include "other.conf";` + "\n",
|
||||
`router id 192.0.2.1; include "/outside/peer.conf";` + "\n",
|
||||
"/*\ninclude \"bird.d/decoy.conf\";\n*/\nrouter id 192.0.2.1; include \"/outside/live.conf\";\n",
|
||||
} {
|
||||
active := t.TempDir()
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main}); err == nil {
|
||||
t.Fatalf("include accepted: %s", main)
|
||||
}
|
||||
}
|
||||
active := t.TempDir()
|
||||
if err := os.Mkdir(filepath.Join(active, "bird.d"), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.d", "operator.conf"), []byte(`define X=1; include "/outside/foo.conf";`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
|
||||
t.Fatal("nested operator include accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountIncludeTokensIgnoresCommentsAndStrings(t *testing.T) {
|
||||
content := "# include \"ignored.conf\";\n" +
|
||||
"/* include \"ignored.conf\"; */\n" +
|
||||
"define note = \"include in string\";\n" +
|
||||
"include \"bird.d/real.conf\";\n"
|
||||
if got := countIncludeTokens(content); got != 1 {
|
||||
t.Fatalf("want one include, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIncludeAfterBackslashStringCannotHideNestedInclude(t *testing.T) {
|
||||
content := `define note = "\";` + "\n" + `include "/outside/hidden.conf";` + "\n"
|
||||
if got := countIncludeTokens(content); got != 1 {
|
||||
t.Fatalf("live include hidden after quoted backslash: %d", got)
|
||||
}
|
||||
active := t.TempDir()
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fragments := map[string]string{"bird.conf": "router id 192.0.2.1;\n", "bird.d/evobgp_test.conf": content}
|
||||
if err := writeGeneration(active, gen, fragments); err == nil {
|
||||
t.Fatal("nested external include accepted after quoted backslash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteGenerationRewritesAbsoluteActiveInclude(t *testing.T) {
|
||||
active := t.TempDir()
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inc := filepath.ToSlash(filepath.Join(active, "bird.d", "peer.conf"))
|
||||
main := "include \"" + inc + "\";\n"
|
||||
if err := writeGeneration(active, gen, map[string]string{"bird.conf": main, "bird.d/peer.conf": "# peer"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(gen, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(string(b), filepath.ToSlash(filepath.Join(gen, "bird.d", "peer.conf"))) {
|
||||
t.Fatalf("absolute include not rewritten: %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteGenerationRejectsSymlinkBirdDRoot(t *testing.T) {
|
||||
active := t.TempDir()
|
||||
outside := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(outside, "operator.conf"), []byte("# operator"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(outside, filepath.Join(active, "bird.d")); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
old := []byte("router id 192.0.2.2;\n")
|
||||
if err := os.WriteFile(filepath.Join(active, "bird.conf"), old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gen := filepath.Join(active, ".evobgp-generation-test")
|
||||
if err := os.Mkdir(gen, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := writeGeneration(active, gen, map[string]string{"bird.conf": "router id 192.0.2.1;\n"}); err == nil {
|
||||
t.Fatal("bird.d symlink accepted")
|
||||
}
|
||||
current, err := os.ReadFile(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(current) != string(old) {
|
||||
t.Fatalf("active config changed: %s", current)
|
||||
}
|
||||
if _, err := os.ReadFile(filepath.Join(outside, "operator.conf")); err != nil {
|
||||
t.Fatalf("operator snippet changed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyRevisionRejectsSymlinkMain(t *testing.T) {
|
||||
active := t.TempDir()
|
||||
target := filepath.Join(t.TempDir(), "operator-bird.conf")
|
||||
old := []byte("router id 192.0.2.2;\n")
|
||||
if err := os.WriteFile(target, old, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Symlink(target, filepath.Join(active, "bird.conf")); err != nil {
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
err := ApplyRevision(context.Background(), nil, &store.Revision{PreviewFragments: map[string]string{"bird.conf": "router id 192.0.2.1;\n"}}, Config{ActiveDir: active})
|
||||
if err == nil || !strings.Contains(err.Error(), "regular file") {
|
||||
t.Fatalf("want explicit symlink rejection, got %v", err)
|
||||
}
|
||||
info, err := os.Lstat(filepath.Join(active, "bird.conf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if info.Mode()&os.ModeSymlink == 0 {
|
||||
t.Fatal("active symlink replaced")
|
||||
}
|
||||
current, err := os.ReadFile(target)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(current) != string(old) {
|
||||
t.Fatalf("operator target changed: %s", current)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//go:build linux
|
||||
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func lockTarget(ctx context.Context, path string) (*os.File, error) {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for {
|
||||
err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
|
||||
if err == nil {
|
||||
return f, nil
|
||||
}
|
||||
if !errors.Is(err, unix.EWOULDBLOCK) && !errors.Is(err, unix.EAGAIN) {
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = f.Close()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
//go:build !linux && !windows
|
||||
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
)
|
||||
|
||||
func lockTarget(context.Context, string) (*os.File, error) {
|
||||
return nil, errors.New("birddeploy: cross-process apply lock is unsupported on this platform")
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
//go:build windows
|
||||
|
||||
package birddeploy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
func lockTarget(ctx context.Context, path string) (*os.File, error) {
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var overlapped windows.Overlapped
|
||||
for {
|
||||
err = windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped)
|
||||
if err == nil {
|
||||
return f, nil
|
||||
}
|
||||
if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = f.Close()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(25 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//go:build !windows
|
||||
|
||||
package birddeploy
|
||||
|
||||
import "os"
|
||||
|
||||
func syncDir(path string) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Sync()
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
//go:build windows
|
||||
|
||||
package birddeploy
|
||||
|
||||
// Windows does not support syncing a directory handle through os.File.Sync.
|
||||
func syncDir(string) error { return nil }
|
||||
+77
-11
@@ -3,34 +3,100 @@ package bundle
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
pathpkg "path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// WriteExtractedFiles writes payload files from a verified bundle under root (directories created as needed).
|
||||
func WriteExtractedFiles(root string, v *VerifiedContents) error {
|
||||
return WriteExtractedFilesWithLimits(root, v, Limits{})
|
||||
}
|
||||
|
||||
// WriteExtractedFilesWithLimits confines all filesystem operations to an open
|
||||
// directory handle. Replacing existing files uses unlink + exclusive creation,
|
||||
// so existing hard links and symlinks cannot redirect writes outside the root.
|
||||
func WriteExtractedFilesWithLimits(root string, v *VerifiedContents, limits Limits) error {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if v == nil {
|
||||
return fmt.Errorf("bundle: nil verified contents")
|
||||
}
|
||||
if len(v.Manifest.Files) > l.MaxEntries-2 {
|
||||
return ErrLimit
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var total int64
|
||||
for _, fe := range v.Manifest.Files {
|
||||
if err := validatePath(fe.Path, l.MaxPathBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if seen[strings.ToLower(fe.Path)] {
|
||||
return fmt.Errorf("bundle: duplicate path %q", fe.Path)
|
||||
}
|
||||
seen[strings.ToLower(fe.Path)] = true
|
||||
data, ok := v.Files[fe.Path]
|
||||
if !ok {
|
||||
return fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||
}
|
||||
if int64(len(data)) > l.MaxFileBytes || int64(len(data)) > l.MaxExpandedBytes-total {
|
||||
return ErrLimit
|
||||
}
|
||||
total += int64(len(data))
|
||||
}
|
||||
root = filepath.Clean(root)
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
confined, err := os.OpenRoot(root)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = confined.Close() }()
|
||||
for _, fe := range v.Manifest.Files {
|
||||
data, ok := v.Files[fe.Path]
|
||||
if !ok {
|
||||
return fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||
}
|
||||
rel := pathpkg.Clean(strings.TrimPrefix(fe.Path, "/"))
|
||||
if rel == "." || strings.HasPrefix(rel, "..") {
|
||||
return fmt.Errorf("bundle: unsafe path %q", fe.Path)
|
||||
parts := strings.Split(fe.Path, "/")
|
||||
for i := 1; i < len(parts); i++ {
|
||||
dir := filepath.FromSlash(strings.Join(parts[:i], "/"))
|
||||
if err := confined.Mkdir(dir, 0o755); err != nil && !os.IsExist(err) {
|
||||
return err
|
||||
}
|
||||
info, err := confined.Lstat(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() || info.Mode()&os.ModeSymlink != 0 {
|
||||
return fmt.Errorf("bundle: unsafe directory %q", dir)
|
||||
}
|
||||
}
|
||||
dest := filepath.Join(root, filepath.FromSlash(rel))
|
||||
relToRoot, err := filepath.Rel(root, dest)
|
||||
if err != nil || strings.HasPrefix(relToRoot, "..") {
|
||||
return fmt.Errorf("bundle: path escapes root: %q", fe.Path)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
|
||||
dest := filepath.FromSlash(fe.Path)
|
||||
info, err := confined.Lstat(dest)
|
||||
if err == nil {
|
||||
if !info.Mode().IsRegular() {
|
||||
return fmt.Errorf("bundle: unsafe destination %q", dest)
|
||||
}
|
||||
if err = confined.Remove(dest); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dest, data, 0o644); err != nil {
|
||||
f, err := confined.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, writeErr := f.Write(data)
|
||||
closeErr := f.Close()
|
||||
if writeErr != nil {
|
||||
return writeErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/ed25519"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"sort"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// legacyManifestV1 mirrors the decoder fields present in the locally available
|
||||
// v1.0.0 source. It deliberately has no future capability or deployment fields.
|
||||
type legacyManifestV1 struct {
|
||||
RevisionID string `json:"revision_id"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Files []struct {
|
||||
Path string `json:"path"`
|
||||
SHA256 string `json:"sha256"`
|
||||
} `json:"files"`
|
||||
Algorithm string `json:"signature_algorithm"`
|
||||
PublicKeyBase64 string `json:"public_key_base64"`
|
||||
}
|
||||
|
||||
func TestLegacyCompatibilityFrozenSignedCurrentBaseline(t *testing.T) {
|
||||
raw, err := os.ReadFile("testdata/current_baseline_signed_bundle.tar.gz")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{0x42}, ed25519.SeedSize)).Public().(ed25519.PublicKey)
|
||||
verified, err := VerifyGzippedTar(raw, pub)
|
||||
if err != nil {
|
||||
t.Fatalf("frozen signed baseline rejected: %v", err)
|
||||
}
|
||||
if verified.Manifest.RevisionID != "revision-legacy" || verified.Manifest.SpeakerID != "speaker-legacy" || len(verified.Files) != 2 {
|
||||
t.Fatalf("frozen baseline contents: %+v", verified.Manifest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCompatibilityBundleWire(t *testing.T) {
|
||||
seed := bytes.Repeat([]byte{0x42}, ed25519.SeedSize)
|
||||
priv := ed25519.NewKeyFromSeed(seed)
|
||||
pub := priv.Public().(ed25519.PublicKey)
|
||||
fragments := map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\n",
|
||||
"bird.d/evobgp_prefixes_v4.conf": "# revision\n",
|
||||
"_preview": "must not ship",
|
||||
}
|
||||
raw, err := BuildGzippedTar("revision-legacy", "speaker-legacy", fragments, priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = gr.Close() }()
|
||||
tr := tar.NewReader(gr)
|
||||
entries := map[string][]byte{}
|
||||
var order []string
|
||||
for {
|
||||
h, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body, err := io.ReadAll(tr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
order = append(order, h.Name)
|
||||
entries[h.Name] = body
|
||||
}
|
||||
wantOrder := []string{"manifest.json", "bundle.sig", "bird.conf", "bird.d/evobgp_prefixes_v4.conf"}
|
||||
if !reflect.DeepEqual(order, wantOrder) {
|
||||
t.Fatalf("legacy tar layout: got %v, want %v", order, wantOrder)
|
||||
}
|
||||
var top map[string]json.RawMessage
|
||||
if err := json.Unmarshal(entries["manifest.json"], &top); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantFields := []string{"created_at", "files", "public_key_base64", "revision_id", "signature_algorithm", "speaker_id"}
|
||||
fields := make([]string, 0, len(top))
|
||||
for k := range top {
|
||||
fields = append(fields, k)
|
||||
}
|
||||
sort.Strings(fields)
|
||||
if !reflect.DeepEqual(fields, wantFields) {
|
||||
t.Fatalf("manifest fields: got %v, want %v", fields, wantFields)
|
||||
}
|
||||
var m legacyManifestV1
|
||||
if err := json.Unmarshal(entries["manifest.json"], &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m.RevisionID != "revision-legacy" || m.SpeakerID != "speaker-legacy" || m.CreatedAt == "" || m.Algorithm != "ed25519" || m.PublicKeyBase64 != base64.StdEncoding.EncodeToString(pub) {
|
||||
t.Fatalf("legacy manifest fields: %+v", m)
|
||||
}
|
||||
if !ed25519.Verify(pub, entries["manifest.json"], entries["bundle.sig"]) {
|
||||
t.Fatal("legacy Ed25519 verifier rejected current signed manifest")
|
||||
}
|
||||
if len(m.Files) != 2 {
|
||||
t.Fatalf("legacy manifest file count: %d", len(m.Files))
|
||||
}
|
||||
for _, f := range m.Files {
|
||||
sum := sha256.Sum256(entries[f.Path])
|
||||
if hex.EncodeToString(sum[:]) != f.SHA256 {
|
||||
t.Fatalf("legacy SHA-256 mismatch for %q", f.Path)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ErrLimit reports that a transport, archive or extraction budget was exceeded.
|
||||
var ErrLimit = errors.New("bundle: resource limit exceeded")
|
||||
|
||||
// Limits bounds both archive transport and decoded tar data, including metadata.
|
||||
// Zero fields select defaults. These ceilings permit large (1M-prefix) configs;
|
||||
// they are safety ceilings, not a promise of suitability for a small-RAM node.
|
||||
type Limits struct {
|
||||
MaxCompressedBytes int64
|
||||
MaxExpandedBytes int64
|
||||
MaxFileBytes int64
|
||||
MaxManifestBytes int64
|
||||
MaxEntries int
|
||||
MaxPathBytes int
|
||||
}
|
||||
|
||||
// DefaultLimits permits large production configs while bounding untrusted input.
|
||||
func DefaultLimits() Limits {
|
||||
return Limits{MaxCompressedBytes: 128 << 20, MaxExpandedBytes: 512 << 20, MaxFileBytes: 256 << 20, MaxManifestBytes: 2 << 20, MaxEntries: 4096, MaxPathBytes: 1024}
|
||||
}
|
||||
|
||||
// Normalized fills zero fields with defaults and rejects invalid budgets.
|
||||
func (l Limits) Normalized() (Limits, error) {
|
||||
d := DefaultLimits()
|
||||
values := []struct {
|
||||
v *int64
|
||||
fallback int64
|
||||
}{{&l.MaxCompressedBytes, d.MaxCompressedBytes}, {&l.MaxExpandedBytes, d.MaxExpandedBytes}, {&l.MaxFileBytes, d.MaxFileBytes}, {&l.MaxManifestBytes, d.MaxManifestBytes}}
|
||||
for _, x := range values {
|
||||
if *x.v < 0 || *x.v == int64(^uint64(0)>>1) {
|
||||
return l, fmt.Errorf("bundle: invalid byte limit")
|
||||
}
|
||||
if *x.v == 0 {
|
||||
*x.v = x.fallback
|
||||
}
|
||||
}
|
||||
if l.MaxEntries < 0 || l.MaxPathBytes < 0 {
|
||||
return l, fmt.Errorf("bundle: invalid entry/path limit")
|
||||
}
|
||||
if l.MaxEntries == 0 {
|
||||
l.MaxEntries = d.MaxEntries
|
||||
}
|
||||
if l.MaxPathBytes == 0 {
|
||||
l.MaxPathBytes = d.MaxPathBytes
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
func validatePath(name string, limit int) error {
|
||||
if name == "" || len(name) > limit || path.IsAbs(name) || path.Clean(name) != name || name == "." || strings.ContainsAny(name, "\\:\x00") {
|
||||
return fmt.Errorf("bundle: unsafe path %q", name)
|
||||
}
|
||||
for _, part := range strings.Split(name, "/") {
|
||||
if part == ".." || strings.TrimRight(part, " .") != part {
|
||||
return fmt.Errorf("bundle: unsafe path %q", name)
|
||||
}
|
||||
base := strings.ToUpper(strings.SplitN(part, ".", 2)[0])
|
||||
if base == "CON" || base == "PRN" || base == "AUX" || base == "NUL" || (len(base) == 4 && (strings.HasPrefix(base, "COM") || strings.HasPrefix(base, "LPT")) && base[3] >= '1' && base[3] <= '9') {
|
||||
return fmt.Errorf("bundle: reserved path %q", name)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type limitedWriter struct {
|
||||
w io.Writer
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func (w *limitedWriter) Write(p []byte) (int, error) {
|
||||
if int64(len(p)) > w.remaining {
|
||||
return 0, ErrLimit
|
||||
}
|
||||
n, err := w.w.Write(p)
|
||||
w.remaining -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadCompressed rejects oversized input while reading, including chunked HTTP
|
||||
// bodies and local files whose size changes after stat.
|
||||
func ReadCompressed(r io.Reader, limits Limits) ([]byte, error) {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := io.ReadAll(io.LimitReader(r, l.MaxCompressedBytes+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(b)) > l.MaxCompressedBytes {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
@@ -0,0 +1,286 @@
|
||||
package bundle
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/ed25519"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testEntry struct {
|
||||
header tar.Header
|
||||
body []byte
|
||||
}
|
||||
|
||||
func archiveEntries(t *testing.T, entries []testEntry) []byte {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
gz := gzip.NewWriter(&b)
|
||||
tw := tar.NewWriter(gz)
|
||||
for _, entry := range entries {
|
||||
h := entry.header
|
||||
if h.Typeflag == 0 {
|
||||
h.Typeflag = tar.TypeReg
|
||||
}
|
||||
h.Size = int64(len(entry.body))
|
||||
if err := tw.WriteHeader(&h); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(entry.body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
func testKey() ed25519.PrivateKey {
|
||||
return ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize))
|
||||
}
|
||||
|
||||
func TestBundleResourceLimits(t *testing.T) {
|
||||
pub := testKey().Public().(ed25519.PublicKey)
|
||||
cases := []struct {
|
||||
name string
|
||||
entries []testEntry
|
||||
limits Limits
|
||||
}{
|
||||
{"file", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxFileBytes: 1024}},
|
||||
{"expanded", []testEntry{{header: tar.Header{Name: "bird.conf"}, body: bytes.Repeat([]byte{'a'}, 8192)}}, Limits{MaxExpandedBytes: 4096}},
|
||||
{"manifest", []testEntry{{header: tar.Header{Name: manifestName}, body: bytes.Repeat([]byte{'a'}, 1025)}}, Limits{MaxManifestBytes: 1024}},
|
||||
{"entries", []testEntry{{header: tar.Header{Name: "a"}}, {header: tar.Header{Name: "b"}}, {header: tar.Header{Name: "c"}}}, Limits{MaxEntries: 2}},
|
||||
{"pax-metadata", []testEntry{{header: tar.Header{Name: "bird.conf", Format: tar.FormatPAX, PAXRecords: map[string]string{"comment": strings.Repeat("a", 8192)}}}}, Limits{MaxExpandedBytes: 4096}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
raw := archiveEntries(t, tc.entries)
|
||||
if _, err := VerifyGzippedTarWithLimits(raw, pub, tc.limits); !errors.Is(err, ErrLimit) {
|
||||
t.Fatalf("want limit, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
if _, err := VerifyGzippedTarWithLimits(make([]byte, 65), pub, Limits{MaxCompressedBytes: 64}); !errors.Is(err, ErrLimit) {
|
||||
t.Fatalf("compressed cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleRejectsUnsafeEntries(t *testing.T) {
|
||||
pub := testKey().Public().(ed25519.PublicKey)
|
||||
for _, name := range []string{"../escape", "/etc/bird.conf", `bird.d\escape`, "C:/escape", "bird.d/../escape", "a//b", "a/./b", "NUL", "bird.conf:stream", "a. /b"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: name}, body: []byte("x")}})
|
||||
if _, err := VerifyGzippedTar(raw, pub); err == nil {
|
||||
t.Fatal("unsafe path accepted")
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, kind := range []byte{tar.TypeSymlink, tar.TypeLink, tar.TypeDir, tar.TypeFifo} {
|
||||
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf", Typeflag: kind, Linkname: "outside"}}})
|
||||
if _, err := VerifyGzippedTar(raw, pub); err == nil {
|
||||
t.Fatalf("type %d accepted", kind)
|
||||
}
|
||||
}
|
||||
raw := archiveEntries(t, []testEntry{{header: tar.Header{Name: "bird.conf"}}, {header: tar.Header{Name: "bird.conf"}}})
|
||||
if _, err := VerifyGzippedTar(raw, pub); err == nil || !strings.Contains(err.Error(), "duplicate") {
|
||||
t.Fatalf("duplicate accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleChecksTrailerAndTrailingExpansion(t *testing.T) {
|
||||
priv := testKey()
|
||||
pub := priv.Public().(ed25519.PublicKey)
|
||||
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "router id 192.0.2.1;"}, priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
corrupt := append([]byte{}, raw...)
|
||||
corrupt[len(corrupt)-8] ^= 0xff
|
||||
if _, err = VerifyGzippedTar(corrupt, pub); err == nil {
|
||||
t.Fatal("gzip checksum ignored after tar EOF")
|
||||
}
|
||||
var extra bytes.Buffer
|
||||
gz := gzip.NewWriter(&extra)
|
||||
if _, err = gz.Write(bytes.Repeat([]byte{'x'}, 32<<10)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
joined := append(append([]byte{}, raw...), extra.Bytes()...)
|
||||
if _, err = VerifyGzippedTarWithLimits(joined, pub, Limits{MaxExpandedBytes: 8 << 10}); !errors.Is(err, ErrLimit) {
|
||||
t.Fatalf("trailing gzip member bypassed budget: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleRejectsPAXSparseRegularFile(t *testing.T) {
|
||||
// archive/tar expands PAX sparse holes even though the returned type is a
|
||||
// regular file. Build a physical one-byte entry with a 4096-byte logical size.
|
||||
record := func(key, value string) string {
|
||||
body := key + "=" + value + "\n"
|
||||
n := len(body) + 2
|
||||
for {
|
||||
next := len(fmt.Sprint(n)) + 1 + len(body)
|
||||
if next == n {
|
||||
return fmt.Sprintf("%d %s", n, body)
|
||||
}
|
||||
n = next
|
||||
}
|
||||
}
|
||||
pax := record("GNU.sparse.size", "4096") + record("GNU.sparse.numblocks", "1") + record("GNU.sparse.map", "0,1")
|
||||
var physical bytes.Buffer
|
||||
tw := tar.NewWriter(&physical)
|
||||
for _, entry := range []testEntry{{header: tar.Header{Name: "PaxHeader", Size: int64(len(pax))}, body: []byte(pax)}, {header: tar.Header{Name: "bird.conf", Size: 1}, body: []byte("x")}} {
|
||||
if err := tw.WriteHeader(&entry.header); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(entry.body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data := physical.Bytes()
|
||||
data[156] = tar.TypeXHeader
|
||||
for i := 148; i < 156; i++ {
|
||||
data[i] = ' '
|
||||
}
|
||||
sum := 0
|
||||
for _, b := range data[:512] {
|
||||
sum += int(b)
|
||||
}
|
||||
copy(data[148:156], fmt.Sprintf("%06o\x00 ", sum))
|
||||
h, err := tar.NewReader(bytes.NewReader(data)).Next()
|
||||
if err != nil || h.Typeflag != tar.TypeReg || h.Size != 4096 {
|
||||
t.Fatalf("invalid sparse regression fixture %+v %v", h, err)
|
||||
}
|
||||
var compressed bytes.Buffer
|
||||
gz := gzip.NewWriter(&compressed)
|
||||
if _, err = gz.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = gz.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = VerifyGzippedTar(compressed.Bytes(), testKey().Public().(ed25519.PublicKey)); err == nil || !strings.Contains(err.Error(), "sparse") {
|
||||
t.Fatalf("PAX sparse accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleRejectsUnmanifestedPayload(t *testing.T) {
|
||||
priv := testKey()
|
||||
raw, err := BuildGzippedTar("r", "s", map[string]string{"bird.conf": "x"}, priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = gr.Close() }()
|
||||
tr := tar.NewReader(gr)
|
||||
var entries []testEntry
|
||||
for {
|
||||
h, e := tr.Next()
|
||||
if e == io.EOF {
|
||||
break
|
||||
}
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
b, e := io.ReadAll(tr)
|
||||
if e != nil {
|
||||
t.Fatal(e)
|
||||
}
|
||||
entries = append(entries, testEntry{header: *h, body: b})
|
||||
}
|
||||
entries = append(entries, testEntry{header: tar.Header{Name: "unsigned.conf"}, body: []byte("unsigned")})
|
||||
if _, err = VerifyGzippedTar(archiveEntries(t, entries), priv.Public().(ed25519.PublicKey)); err == nil {
|
||||
t.Fatal("unmanifested payload accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractionDoesNotFollowLinks(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
outside := filepath.Join(t.TempDir(), "outside")
|
||||
if err := os.WriteFile(outside, []byte("preserve"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dest := filepath.Join(root, "bird.conf")
|
||||
if err := os.Link(outside, dest); err != nil {
|
||||
if runtime.GOOS == "linux" {
|
||||
t.Fatalf("Linux link regression requires hard links: %v", err)
|
||||
}
|
||||
t.Skipf("hard links unavailable: %v", err)
|
||||
}
|
||||
v := &VerifiedContents{Manifest: Manifest{Files: []FileEntry{{Path: "bird.conf"}}}, Files: map[string][]byte{"bird.conf": []byte("replacement")}}
|
||||
if err := WriteExtractedFiles(root, v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := os.ReadFile(outside)
|
||||
if err != nil || string(got) != "preserve" {
|
||||
t.Fatalf("hard link modified outside: %s %v", got, err)
|
||||
}
|
||||
if err = os.Remove(dest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = os.Symlink(outside, dest); err != nil {
|
||||
if runtime.GOOS == "linux" {
|
||||
t.Fatalf("Linux link regression requires symlinks: %v", err)
|
||||
}
|
||||
t.Skipf("symlinks unavailable: %v", err)
|
||||
}
|
||||
if err = WriteExtractedFiles(root, v); err == nil {
|
||||
t.Fatal("symlink destination accepted")
|
||||
}
|
||||
got, err = os.ReadFile(outside)
|
||||
if err != nil || string(got) != "preserve" {
|
||||
t.Fatal("symlink modified outside")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleBuilderBoundsAndPortablePaths(t *testing.T) {
|
||||
for _, name := range []string{"/bird.conf", "a/../bird.conf", manifestName, sigFileName, `bird.d\bad`} {
|
||||
if _, err := BuildGzippedTar("r", "s", map[string]string{name: "x"}, testKey()); err == nil {
|
||||
t.Fatalf("invalid builder path accepted: %q", name)
|
||||
}
|
||||
}
|
||||
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": strings.Repeat("x", 1025)}, testKey(), Limits{MaxFileBytes: 1024}); !errors.Is(err, ErrLimit) {
|
||||
t.Fatalf("builder file cap %v", err)
|
||||
}
|
||||
if _, err := BuildGzippedTarWithLimits("r", "s", map[string]string{"bird.conf": "x"}, testKey(), Limits{MaxExpandedBytes: 1024}); !errors.Is(err, ErrLimit) {
|
||||
t.Fatalf("builder tar overhead cap %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// This is a decoded-size regression using repeated representative route lines,
|
||||
// not a unique-prefix cardinality test or a BIRD validity/performance benchmark.
|
||||
func TestDefaultLimitsAllowMillionRepresentativeIPv6RouteLines(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("large representative config")
|
||||
}
|
||||
config := strings.Repeat("route 2001:db8:1234:5678::/64 blackhole;\n", 1_000_000)
|
||||
raw, err := BuildGzippedTar("large", "speaker", map[string]string{"bird.conf": config}, testKey())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v, err := VerifyGzippedTar(raw, testKey().Public().(ed25519.PublicKey))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(v.Files["bird.conf"]) != len(config) {
|
||||
t.Fatal("large config truncated")
|
||||
}
|
||||
}
|
||||
+35
-8
@@ -9,7 +9,6 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -36,22 +35,47 @@ const manifestName = "manifest.json"
|
||||
|
||||
// BuildGzippedTar builds a .tar.gz with manifest.json, bundle.sig (Ed25519 over manifest JSON), and payload files.
|
||||
func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||
return BuildGzippedTarWithLimits(revisionID, speakerID, fragments, priv, Limits{})
|
||||
}
|
||||
|
||||
// BuildGzippedTarWithLimits preserves the manifest/signature wire format while
|
||||
// refusing payloads that exceed the receiving node's configured budgets.
|
||||
func BuildGzippedTarWithLimits(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey, limits Limits) ([]byte, error) {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(priv) != ed25519.PrivateKeySize {
|
||||
return nil, fmt.Errorf("bundle: invalid ed25519 private key size")
|
||||
}
|
||||
pub := priv.Public().(ed25519.PublicKey)
|
||||
|
||||
norm := make(map[string]string, len(fragments))
|
||||
norm := make(map[string]string)
|
||||
seen := map[string]bool{}
|
||||
var total int64
|
||||
for p, content := range fragments {
|
||||
p = path.Clean(strings.TrimPrefix(p, "/"))
|
||||
if p == "." || strings.HasPrefix(p, "..") {
|
||||
return nil, fmt.Errorf("bundle: invalid path %q", p)
|
||||
}
|
||||
if strings.HasPrefix(p, "_") {
|
||||
continue // UI-only preview aggregate, not shipped to nodes
|
||||
}
|
||||
if len(norm) >= l.MaxEntries-2 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
if err := validatePath(p, l.MaxPathBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == manifestName || p == sigFileName || seen[strings.ToLower(p)] {
|
||||
return nil, fmt.Errorf("bundle: duplicate or reserved path %q", p)
|
||||
}
|
||||
seen[strings.ToLower(p)] = true
|
||||
if int64(len(content)) > l.MaxFileBytes || int64(len(content)) > l.MaxExpandedBytes-total {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
total += int64(len(content))
|
||||
norm[p] = content
|
||||
}
|
||||
if len(norm) > l.MaxEntries-2 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
|
||||
var files []FileEntry
|
||||
var payload [][]byte
|
||||
@@ -75,11 +99,14 @@ func BuildGzippedTar(revisionID, speakerID string, fragments map[string]string,
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(manifestJSON)) > l.MaxManifestBytes {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
sig := ed25519.Sign(priv, manifestJSON)
|
||||
|
||||
var buf bytes.Buffer
|
||||
gw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(gw)
|
||||
gw := gzip.NewWriter(&limitedWriter{w: &buf, remaining: l.MaxCompressedBytes})
|
||||
tw := tar.NewWriter(&limitedWriter{w: gw, remaining: l.MaxExpandedBytes})
|
||||
|
||||
add := func(name string, body []byte) error {
|
||||
hdr := &tar.Header{
|
||||
|
||||
Binary file not shown.
+92
-18
@@ -11,44 +11,95 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// VerifiedContents is the result of verifying a downloaded bundle on evobgp-node.
|
||||
// VerifiedContents contains only payloads covered by a verified signed manifest.
|
||||
type VerifiedContents struct {
|
||||
Manifest Manifest
|
||||
Files map[string][]byte
|
||||
}
|
||||
|
||||
// VerifyGzippedTar checks Ed25519 signature over manifest.json and SHA-256 of each file.
|
||||
func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents, error) {
|
||||
gr, err := gzip.NewReader(bytes.NewReader(bundle))
|
||||
// VerifyGzippedTar verifies the unchanged signed manifest format with default limits.
|
||||
func VerifyGzippedTar(raw []byte, pub ed25519.PublicKey) (*VerifiedContents, error) {
|
||||
return VerifyGzippedTarWithLimits(raw, pub, Limits{})
|
||||
}
|
||||
|
||||
// VerifyGzippedTarWithLimits verifies the signature and hashes under explicit
|
||||
// compressed, expanded, per-entry and metadata budgets.
|
||||
func VerifyGzippedTarWithLimits(raw []byte, pub ed25519.PublicKey, limits Limits) (*VerifiedContents, error) {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(raw)) > l.MaxCompressedBytes {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
if len(pub) != ed25519.PublicKeySize {
|
||||
return nil, fmt.Errorf("bundle: invalid public key size")
|
||||
}
|
||||
gr, err := gzip.NewReader(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = gr.Close() }()
|
||||
|
||||
var manifestRaw []byte
|
||||
var sig []byte
|
||||
files := make(map[string][]byte)
|
||||
tr := tar.NewReader(gr)
|
||||
decoded := &io.LimitedReader{R: gr, N: l.MaxExpandedBytes + 1}
|
||||
tr := tar.NewReader(decoded)
|
||||
var manifestRaw, sig []byte
|
||||
files := map[string][]byte{}
|
||||
seen := map[string]bool{}
|
||||
count := 0
|
||||
var logicalBytes int64
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if decoded.N == 0 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := path.Clean(hdr.Name)
|
||||
if strings.HasPrefix(name, "..") {
|
||||
return nil, fmt.Errorf("bundle: illegal tar entry %q", hdr.Name)
|
||||
count++
|
||||
if count > l.MaxEntries {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
data, err := io.ReadAll(tr)
|
||||
if err = validatePath(hdr.Name, l.MaxPathBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
name := hdr.Name
|
||||
folded := strings.ToLower(name)
|
||||
if seen[folded] {
|
||||
return nil, fmt.Errorf("bundle: duplicate tar entry %q", name)
|
||||
}
|
||||
seen[folded] = true
|
||||
if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
|
||||
return nil, fmt.Errorf("bundle: nonregular tar entry %q", name)
|
||||
}
|
||||
for key := range hdr.PAXRecords {
|
||||
if strings.HasPrefix(key, "GNU.sparse.") {
|
||||
return nil, fmt.Errorf("bundle: sparse entry %q is unsupported", name)
|
||||
}
|
||||
}
|
||||
max := l.MaxFileBytes
|
||||
if name == manifestName {
|
||||
max = l.MaxManifestBytes
|
||||
}
|
||||
if name == sigFileName {
|
||||
max = ed25519.SignatureSize
|
||||
}
|
||||
if hdr.Size < 0 || hdr.Size > max || hdr.Size >= decoded.N || hdr.Size > l.MaxExpandedBytes-logicalBytes {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
logicalBytes += hdr.Size
|
||||
data, err := io.ReadAll(io.LimitReader(tr, max+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > max || decoded.N == 0 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
switch name {
|
||||
case manifestName:
|
||||
manifestRaw = data
|
||||
@@ -58,17 +109,36 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
|
||||
files[name] = data
|
||||
}
|
||||
}
|
||||
if len(manifestRaw) == 0 || len(sig) == 0 {
|
||||
// tar EOF precedes the gzip trailer. Drain through the same byte budget so
|
||||
// checksums, extra gzip members and trailing decompression are also bounded.
|
||||
if _, err = io.Copy(io.Discard, decoded); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if decoded.N == 0 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
if len(manifestRaw) == 0 || len(sig) != ed25519.SignatureSize {
|
||||
return nil, fmt.Errorf("bundle: missing manifest or signature")
|
||||
}
|
||||
if !ed25519.Verify(pub, manifestRaw, sig) {
|
||||
return nil, fmt.Errorf("bundle: ed25519 signature mismatch")
|
||||
}
|
||||
var m Manifest
|
||||
if err := json.Unmarshal(manifestRaw, &m); err != nil {
|
||||
if err = json.Unmarshal(manifestRaw, &m); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(m.Files) > l.MaxEntries-2 {
|
||||
return nil, ErrLimit
|
||||
}
|
||||
listed := map[string]bool{}
|
||||
for _, fe := range m.Files {
|
||||
if err = validatePath(fe.Path, l.MaxPathBytes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fe.Path == manifestName || fe.Path == sigFileName || listed[strings.ToLower(fe.Path)] {
|
||||
return nil, fmt.Errorf("bundle: duplicate or reserved manifest path %q", fe.Path)
|
||||
}
|
||||
listed[strings.ToLower(fe.Path)] = true
|
||||
body, ok := files[fe.Path]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("bundle: missing file %q", fe.Path)
|
||||
@@ -78,10 +148,13 @@ func VerifyGzippedTar(bundle []byte, pub ed25519.PublicKey) (*VerifiedContents,
|
||||
return nil, fmt.Errorf("bundle: checksum mismatch for %q", fe.Path)
|
||||
}
|
||||
}
|
||||
if len(listed) != len(files) {
|
||||
return nil, fmt.Errorf("bundle: payload is not covered by manifest")
|
||||
}
|
||||
return &VerifiedContents{Manifest: m, Files: files}, nil
|
||||
}
|
||||
|
||||
// ParsePublicKeyHex decodes a 64-char hex Ed25519 public key (32 bytes).
|
||||
// ParsePublicKeyHex decodes a 32-byte Ed25519 public key from hexadecimal.
|
||||
func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
|
||||
s = strings.TrimSpace(strings.TrimPrefix(strings.TrimPrefix(s, "0x"), "0X"))
|
||||
raw, err := hex.DecodeString(s)
|
||||
@@ -94,7 +167,8 @@ func ParsePublicKeyHex(s string) (ed25519.PublicKey, error) {
|
||||
return ed25519.PublicKey(raw), nil
|
||||
}
|
||||
|
||||
// PublicKeyFromManifestBase64 uses the key embedded in the signed manifest (cross-check with expected pubkey optional).
|
||||
// PublicKeyFromManifestBase64 decodes the manifest key. Callers must still trust
|
||||
// an independently provisioned key when authenticating the bundle.
|
||||
func PublicKeyFromManifestBase64(m *Manifest) (ed25519.PublicKey, error) {
|
||||
if m.PublicKeyB64 == "" {
|
||||
return nil, fmt.Errorf("bundle: manifest missing public_key_base64")
|
||||
|
||||
+267
-23
@@ -8,48 +8,292 @@ import (
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree (idempotent via schema_migrations).
|
||||
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) error {
|
||||
if _, err := pool.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil {
|
||||
return fmt.Errorf("db: schema_migrations: %w", err)
|
||||
}
|
||||
const (
|
||||
postgresMigrationLockKey int64 = 0x45766f4247504d47 // "EvoBGPMG", scoped to the database.
|
||||
migrationLockTimeout = 30 * time.Second
|
||||
migrationCleanupTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
// PostgreSQL migrations are transaction-only. Non-transactional statements
|
||||
// need an explicit, separately reviewed execution path.
|
||||
|
||||
// ApplyPostgresMigrations runs ordered *.up.sql from an embedded subtree.
|
||||
// Each migration and its version row commit atomically under a session advisory lock.
|
||||
func ApplyPostgresMigrations(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS, subdir string) (retErr error) {
|
||||
entries, err := fs.ReadDir(fsys, subdir)
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("db: read migrations: %w", err)
|
||||
}
|
||||
var ups []string
|
||||
for _, e := range entries {
|
||||
if e.IsDir() || !strings.HasSuffix(e.Name(), ".up.sql") {
|
||||
continue
|
||||
if !e.IsDir() && strings.HasSuffix(e.Name(), ".up.sql") {
|
||||
ups = append(ups, e.Name())
|
||||
}
|
||||
ups = append(ups, e.Name())
|
||||
}
|
||||
sort.Strings(ups)
|
||||
// Include pool acquisition in the lock timeout. The lock precedes even
|
||||
// schema_migrations creation, because another process may start from an empty DB.
|
||||
lockCtx, cancelLock := context.WithTimeout(ctx, migrationLockTimeout)
|
||||
conn, err := pool.Acquire(lockCtx)
|
||||
if err != nil {
|
||||
cancelLock()
|
||||
return fmt.Errorf("db: acquire migration connection: %w", err)
|
||||
}
|
||||
defer conn.Release()
|
||||
if _, err := conn.Exec(lockCtx, `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
|
||||
cancelLock()
|
||||
// Cancellation may race with a successful server-side lock acquisition.
|
||||
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||
defer closeCancel()
|
||||
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
|
||||
return errors.Join(fmt.Errorf("db: acquire migration lock: %w", err), fmt.Errorf("db: close migration connection: %w", closeErr))
|
||||
}
|
||||
return fmt.Errorf("db: acquire migration lock: %w", err)
|
||||
}
|
||||
cancelLock()
|
||||
defer func() {
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||
defer cancel()
|
||||
var unlocked bool
|
||||
if err := conn.QueryRow(cleanupCtx, `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
|
||||
if err == nil {
|
||||
err = errors.New("lock was not held")
|
||||
}
|
||||
retErr = errors.Join(retErr, fmt.Errorf("db: release migration lock: %w", err))
|
||||
// Never return a possibly locked session to the pool.
|
||||
closeCtx, closeCancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||
defer closeCancel()
|
||||
if closeErr := conn.Conn().Close(closeCtx); closeErr != nil {
|
||||
retErr = errors.Join(retErr, fmt.Errorf("db: close migration connection: %w", closeErr))
|
||||
}
|
||||
}
|
||||
}()
|
||||
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (version TEXT NOT NULL PRIMARY KEY)`); err != nil {
|
||||
return fmt.Errorf("db: schema_migrations: %w", err)
|
||||
}
|
||||
for _, name := range ups {
|
||||
ver := strings.TrimSuffix(name, ".up.sql")
|
||||
var dummy int
|
||||
err := pool.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, ver).Scan(&dummy)
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
body, err := fs.ReadFile(fsys, path.Join(subdir, name))
|
||||
if err != nil {
|
||||
return err
|
||||
return fmt.Errorf("db: read migration %s: %w", name, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, string(body)); err != nil {
|
||||
if bad := unsupportedMigrationStatement(string(body)); bad != "" {
|
||||
return fmt.Errorf("db: migration %s uses unsupported non-transactional SQL %q", name, bad)
|
||||
}
|
||||
if err := applyPostgresMigration(ctx, conn, strings.TrimSuffix(name, ".up.sql"), body); err != nil {
|
||||
return fmt.Errorf("db: migrate %s: %w", name, err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, ver); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// unsupportedMigrationStatement inspects statement starts after masking SQL
|
||||
// comments and quoted strings. In particular, a comment after a semicolon
|
||||
// must not conceal COMMIT or another command that would escape our transaction.
|
||||
func unsupportedMigrationStatement(sql string) string {
|
||||
const (
|
||||
normal = iota
|
||||
singleQuote
|
||||
doubleQuote
|
||||
lineComment
|
||||
blockComment
|
||||
dollarQuote
|
||||
)
|
||||
state := normal
|
||||
blockDepth := 0
|
||||
dollarTag := ""
|
||||
escapeString := false
|
||||
var clean strings.Builder
|
||||
for i := 0; i < len(sql); {
|
||||
switch state {
|
||||
case normal:
|
||||
switch {
|
||||
case sql[i] == '\'':
|
||||
escapeString = i > 0 && (sql[i-1] == 'E' || sql[i-1] == 'e') && (i == 1 || !isSQLIdentifierByte(sql[i-2]))
|
||||
state = singleQuote
|
||||
clean.WriteByte(' ')
|
||||
i++
|
||||
case sql[i] == '"':
|
||||
state = doubleQuote
|
||||
clean.WriteByte(' ')
|
||||
i++
|
||||
case strings.HasPrefix(sql[i:], "--"):
|
||||
state = lineComment
|
||||
clean.WriteByte(' ')
|
||||
i += 2
|
||||
case strings.HasPrefix(sql[i:], "/*"):
|
||||
state = blockComment
|
||||
blockDepth = 1
|
||||
clean.WriteByte(' ')
|
||||
i += 2
|
||||
case sql[i] == '$' && (i == 0 || !isSQLIdentifierByte(sql[i-1])):
|
||||
end := i + 1
|
||||
if end < len(sql) && isDollarTagStartByte(sql[end]) {
|
||||
end++
|
||||
for end < len(sql) && isDollarTagByte(sql[end]) {
|
||||
end++
|
||||
}
|
||||
}
|
||||
if end < len(sql) && sql[end] == '$' {
|
||||
dollarTag = sql[i : end+1]
|
||||
state = dollarQuote
|
||||
clean.WriteByte(' ')
|
||||
i = end + 1
|
||||
} else {
|
||||
clean.WriteByte(sql[i])
|
||||
i++
|
||||
}
|
||||
default:
|
||||
clean.WriteByte(sql[i])
|
||||
i++
|
||||
}
|
||||
case singleQuote:
|
||||
switch {
|
||||
case escapeString && sql[i] == '\\' && i+1 < len(sql):
|
||||
i += 2
|
||||
case sql[i] == '\'' && i+1 < len(sql) && sql[i+1] == '\'':
|
||||
i += 2
|
||||
case sql[i] == '\'':
|
||||
state = normal
|
||||
i++
|
||||
default:
|
||||
i++
|
||||
}
|
||||
case doubleQuote:
|
||||
if sql[i] == '"' {
|
||||
if i+1 < len(sql) && sql[i+1] == '"' {
|
||||
i += 2
|
||||
} else {
|
||||
state = normal
|
||||
i++
|
||||
}
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
case lineComment:
|
||||
if sql[i] == '\n' || sql[i] == '\r' {
|
||||
clean.WriteByte('\n')
|
||||
state = normal
|
||||
}
|
||||
i++
|
||||
case blockComment:
|
||||
switch {
|
||||
case strings.HasPrefix(sql[i:], "/*"):
|
||||
blockDepth++
|
||||
i += 2
|
||||
case strings.HasPrefix(sql[i:], "*/"):
|
||||
blockDepth--
|
||||
i += 2
|
||||
if blockDepth == 0 {
|
||||
state = normal
|
||||
}
|
||||
default:
|
||||
i++
|
||||
}
|
||||
case dollarQuote:
|
||||
if strings.HasPrefix(sql[i:], dollarTag) {
|
||||
i += len(dollarTag)
|
||||
state = normal
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, statement := range strings.Split(clean.String(), ";") {
|
||||
words := strings.Fields(strings.ToUpper(statement))
|
||||
if isUnsupportedMigrationCommand(words) {
|
||||
return strings.Join(words[:min(len(words), 4)], " ")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func isSQLIdentifierByte(b byte) bool {
|
||||
return isDollarTagByte(b) || b == '$'
|
||||
}
|
||||
|
||||
func isDollarTagStartByte(b byte) bool {
|
||||
return b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b == '_' || b >= 0x80
|
||||
}
|
||||
|
||||
func isDollarTagByte(b byte) bool {
|
||||
return isDollarTagStartByte(b) || b >= '0' && b <= '9'
|
||||
}
|
||||
|
||||
func isUnsupportedMigrationCommand(words []string) bool {
|
||||
if len(words) == 0 {
|
||||
return false
|
||||
}
|
||||
second := ""
|
||||
if len(words) > 1 {
|
||||
second = words[1]
|
||||
}
|
||||
switch words[0] {
|
||||
case "BEGIN", "COMMIT", "END", "ROLLBACK", "ABORT", "SAVEPOINT", "REINDEX", "VACUUM", "CLUSTER":
|
||||
return true
|
||||
case "START", "PREPARE":
|
||||
return second == "TRANSACTION"
|
||||
case "RELEASE":
|
||||
return second == "SAVEPOINT" || second != ""
|
||||
case "ALTER":
|
||||
return second == "SYSTEM"
|
||||
case "CREATE":
|
||||
if second == "DATABASE" || second == "TABLESPACE" {
|
||||
return true
|
||||
}
|
||||
idx := 2
|
||||
if second == "UNIQUE" {
|
||||
idx = 3
|
||||
}
|
||||
return len(words) > idx && words[idx-1] == "INDEX" && words[idx] == "CONCURRENTLY"
|
||||
case "DROP":
|
||||
return second == "DATABASE" || second == "TABLESPACE" || second == "INDEX" && len(words) > 2 && words[2] == "CONCURRENTLY"
|
||||
case "REFRESH":
|
||||
return len(words) > 3 && second == "MATERIALIZED" && words[2] == "VIEW" && words[3] == "CONCURRENTLY"
|
||||
case "DISCARD":
|
||||
return second == "ALL"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func applyPostgresMigration(ctx context.Context, conn *pgxpool.Conn, version string, body []byte) (retErr error) {
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if committed {
|
||||
return
|
||||
}
|
||||
cleanupCtx, cancel := context.WithTimeout(context.Background(), migrationCleanupTimeout)
|
||||
defer cancel()
|
||||
if err := tx.Rollback(cleanupCtx); err != nil && !errors.Is(err, pgx.ErrTxClosed) {
|
||||
retErr = errors.Join(retErr, fmt.Errorf("db: rollback migration %s: %w", version, err))
|
||||
}
|
||||
}()
|
||||
var present int
|
||||
err = tx.QueryRow(ctx, `SELECT 1 FROM schema_migrations WHERE version = $1`, version).Scan(&present)
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if !errors.Is(err, pgx.ErrNoRows) {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, string(body)); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, version); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
committed = true
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/migrations"
|
||||
)
|
||||
|
||||
func TestPostgresEmbeddedMigrationsTransactional(t *testing.T) {
|
||||
entries, err := fs.ReadDir(migrations.Postgres, "postgres")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if !strings.HasSuffix(entry.Name(), ".up.sql") {
|
||||
continue
|
||||
}
|
||||
body, err := fs.ReadFile(migrations.Postgres, path.Join("postgres", entry.Name()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bad := unsupportedMigrationStatement(string(body)); bad != "" {
|
||||
t.Errorf("%s contains unsupported transaction control or DDL: %q", entry.Name(), bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationRejectsTransactionControl(t *testing.T) {
|
||||
for _, statement := range []string{
|
||||
"COMMIT;", "END;", "ROLLBACK;", "ABORT;", "BEGIN;",
|
||||
"CREATE INDEX CONCURRENTLY idx ON t (id);", "VACUUM t;",
|
||||
"CREATE TABLE t(id int); /* boundary */ COMMIT; SELECT 1/0;",
|
||||
"/* comment */COMMIT;",
|
||||
"-- comment\nCOMMIT;",
|
||||
"/* nested /* comment */ still comment */ END;",
|
||||
"CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$",
|
||||
"-- comment\rCOMMIT;",
|
||||
`foo$E'abc\'; COMMIT;`,
|
||||
`éE'abc\'; COMMIT;`,
|
||||
} {
|
||||
if unsupportedMigrationStatement(statement) == "" {
|
||||
t.Errorf("did not reject %q", statement)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationIgnoresQuotedControlWords(t *testing.T) {
|
||||
for _, statement := range []string{
|
||||
"SELECT 'COMMIT;'::text;",
|
||||
"SELECT $$COMMIT;$$;",
|
||||
"SELECT $tag$END;$tag$;",
|
||||
"SELECT E'ABORT;\\'';",
|
||||
"SELECT \"ROLLBACK\" FROM t;",
|
||||
"-- COMMIT;\nSELECT 1;",
|
||||
} {
|
||||
if bad := unsupportedMigrationStatement(statement); bad != "" {
|
||||
t.Errorf("rejected quoted or commented SQL %q: %s", statement, bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
+24
-11
@@ -25,17 +25,7 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if max := os.Getenv("EVOBGP_DB_MAX_CONNS"); max != "" {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(max)); err == nil && n > 0 {
|
||||
cfg.MaxConns = int32(n)
|
||||
}
|
||||
}
|
||||
if min := os.Getenv("EVOBGP_DB_MIN_CONNS"); min != "" {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(min)); err == nil && n >= 0 {
|
||||
cfg.MinConns = int32(n)
|
||||
}
|
||||
}
|
||||
cfg.MaxConnLifetime = 30 * time.Minute
|
||||
applyPostgresPoolConfig(cfg)
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -47,6 +37,29 @@ func OpenPostgresPool(ctx context.Context, dsn string) (*pgxpool.Pool, error) {
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
// applyPostgresPoolConfig sets pool limits. An explicit EVOBGP_DB_MAX_CONNS is kept.
|
||||
// When it is unset, the pool defaults to 8 connections so it can cover the default
|
||||
// worker count instead of pgx's default of 4.
|
||||
func applyPostgresPoolConfig(cfg *pgxpool.Config) {
|
||||
if cfg == nil {
|
||||
return
|
||||
}
|
||||
if max := strings.TrimSpace(os.Getenv("EVOBGP_DB_MAX_CONNS")); max != "" {
|
||||
if n, err := strconv.Atoi(max); err == nil && n > 0 {
|
||||
cfg.MaxConns = int32(n)
|
||||
}
|
||||
} else {
|
||||
cfg.MaxConns = 8
|
||||
}
|
||||
if min := strings.TrimSpace(os.Getenv("EVOBGP_DB_MIN_CONNS")); min != "" {
|
||||
if n, err := strconv.Atoi(min); err == nil && n >= 0 {
|
||||
cfg.MinConns = int32(n)
|
||||
}
|
||||
}
|
||||
cfg.MaxConnLifetime = 30 * time.Minute
|
||||
cfg.MaxConnIdleTime = 5 * time.Minute
|
||||
}
|
||||
|
||||
// OpenSQLite applies sqlite/*.up.sql then opens the DB (foreign keys on).
|
||||
func OpenSQLite(ctx context.Context, filePath string) (*sql.DB, error) {
|
||||
dsn := fmt.Sprintf("file:%s?_pragma=foreign_keys(1)", strings.TrimPrefix(filePath, "file:"))
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestApplyPostgresPoolConfigDefaultAndExplicit(t *testing.T) {
|
||||
t.Setenv("EVOBGP_DB_MAX_CONNS", "")
|
||||
t.Setenv("EVOBGP_DB_MIN_CONNS", "")
|
||||
cfg, err := pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPostgresPoolConfig(cfg)
|
||||
if cfg.MaxConns != 8 {
|
||||
t.Fatalf("default max conns: %d", cfg.MaxConns)
|
||||
}
|
||||
if cfg.MaxConnIdleTime != 5*time.Minute {
|
||||
t.Fatalf("idle time: %s", cfg.MaxConnIdleTime)
|
||||
}
|
||||
|
||||
t.Setenv("EVOBGP_DB_MAX_CONNS", "25")
|
||||
cfg, err = pgxpool.ParseConfig("postgres://localhost/evobgp?sslmode=disable")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
applyPostgresPoolConfig(cfg)
|
||||
if cfg.MaxConns != 25 {
|
||||
t.Fatalf("explicit max conns overridden: %d", cfg.MaxConns)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io/fs"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"evobgp/migrations"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestPostgresMigrationsIntegration(t *testing.T) {
|
||||
pool := isolatedMigrationPool(t)
|
||||
ctx := context.Background()
|
||||
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||
t.Fatalf("first migration pass: %v", err)
|
||||
}
|
||||
if err := ApplyPostgresMigrations(ctx, pool, migrations.Postgres, "postgres"); err != nil {
|
||||
t.Fatalf("second migration pass: %v", err)
|
||||
}
|
||||
files, err := fs.ReadDir(migrations.Postgres, "postgres")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := 0
|
||||
for _, file := range files {
|
||||
if strings.HasSuffix(file.Name(), ".up.sql") {
|
||||
want++
|
||||
}
|
||||
}
|
||||
var got int
|
||||
if err := pool.QueryRow(ctx, `SELECT count(*)::int FROM schema_migrations`).Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("applied migrations = %d, want %d", got, want)
|
||||
}
|
||||
var extension string
|
||||
if err := pool.QueryRow(ctx, `SELECT extname FROM pg_extension WHERE extname = 'pg_stat_statements'`).Scan(&extension); err != nil {
|
||||
t.Fatalf("pg_stat_statements extension missing: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationConcurrentApplyIntegration(t *testing.T) {
|
||||
pool := isolatedMigrationPool(t)
|
||||
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE migration_probe (id integer PRIMARY KEY); SELECT pg_sleep(0.1);`)}}
|
||||
var wg sync.WaitGroup
|
||||
errors := make(chan error, 2)
|
||||
for i := 0; i < 2; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
errors <- ApplyPostgresMigrations(context.Background(), pool, files, "sql")
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errors)
|
||||
for err := range errors {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assertMigrationVersionCount(t, pool, 1)
|
||||
}
|
||||
|
||||
func TestPostgresMigrationRollbackIntegration(t *testing.T) {
|
||||
pool := isolatedMigrationPool(t)
|
||||
files := fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); SELECT 1 / 0;`)}}
|
||||
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil {
|
||||
t.Fatal("expected migration error")
|
||||
}
|
||||
assertMigrationVersionCount(t, pool, 0)
|
||||
var exists bool
|
||||
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("failed migration left a table behind")
|
||||
}
|
||||
// A transaction terminator hidden behind a comment must be rejected before
|
||||
// the first DDL statement can run.
|
||||
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE migration_partial (id integer); /* boundary */ COMMIT; SELECT 1 / 0;`)}}
|
||||
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
|
||||
t.Fatalf("comment-hidden COMMIT error = %v", err)
|
||||
}
|
||||
assertMigrationVersionCount(t, pool, 0)
|
||||
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('migration_partial') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("rejected migration left a table behind")
|
||||
}
|
||||
files = fstest.MapFS{"sql/000001_broken.up.sql": {Data: []byte(`CREATE TABLE foo$tag$ (id int); COMMIT; -- $tag$`)}}
|
||||
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err == nil || !strings.Contains(err.Error(), "unsupported non-transactional SQL") {
|
||||
t.Fatalf("identifier-embedded dollar tag error = %v", err)
|
||||
}
|
||||
assertMigrationVersionCount(t, pool, 0)
|
||||
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('foo$tag$') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("identifier-embedded dollar tag left a table behind")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationIdempotentIntegration(t *testing.T) {
|
||||
pool := isolatedMigrationPool(t)
|
||||
files := fstest.MapFS{"sql/000001_once.up.sql": {Data: []byte(`CREATE TABLE migration_once (id integer); INSERT INTO migration_once VALUES (1);`)}}
|
||||
for i := 0; i < 2; i++ {
|
||||
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
assertMigrationVersionCount(t, pool, 1)
|
||||
var rows int
|
||||
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM migration_once`).Scan(&rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rows != 1 {
|
||||
t.Fatalf("migration rows = %d, want 1", rows)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresMigrationLockCancellationIntegration(t *testing.T) {
|
||||
pool := isolatedMigrationPool(t)
|
||||
locker, err := pool.Acquire(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer locker.Release()
|
||||
if _, err := locker.Exec(context.Background(), `SELECT pg_advisory_lock($1)`, postgresMigrationLockKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
_, _ = locker.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey)
|
||||
}()
|
||||
files := fstest.MapFS{"sql/000001_probe.up.sql": {Data: []byte(`CREATE TABLE lock_probe (id integer);`)}}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond)
|
||||
defer cancel()
|
||||
err = ApplyPostgresMigrations(ctx, pool, files, "sql")
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("lock wait error = %v, want context deadline exceeded", err)
|
||||
}
|
||||
var exists bool
|
||||
if err := pool.QueryRow(context.Background(), `SELECT to_regclass('schema_migrations') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if exists {
|
||||
t.Fatal("schema_migrations was created before migration lock acquisition")
|
||||
}
|
||||
var unlocked bool
|
||||
if err := locker.QueryRow(context.Background(), `SELECT pg_advisory_unlock($1)`, postgresMigrationLockKey).Scan(&unlocked); err != nil || !unlocked {
|
||||
t.Fatalf("release held lock: unlocked=%v err=%v", unlocked, err)
|
||||
}
|
||||
if err := ApplyPostgresMigrations(context.Background(), pool, files, "sql"); err != nil {
|
||||
t.Fatalf("apply after lock release: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func isolatedMigrationPool(t *testing.T) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("EVOBGP_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("EVOBGP_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
admin, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var nonce [8]byte
|
||||
if _, err := rand.Read(nonce[:]); err != nil {
|
||||
admin.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
schema := "evobgp_mig_test_" + hex.EncodeToString(nonce[:])
|
||||
if _, err := admin.Exec(ctx, `CREATE SCHEMA `+schema); err != nil {
|
||||
admin.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
var pool *pgxpool.Pool
|
||||
t.Cleanup(func() {
|
||||
if pool != nil {
|
||||
pool.Close()
|
||||
}
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cleanupCancel()
|
||||
if _, err := admin.Exec(cleanupCtx, `DROP SCHEMA `+schema+` CASCADE`); err != nil {
|
||||
t.Errorf("drop isolated schema %s: %v", schema, err)
|
||||
}
|
||||
admin.Close()
|
||||
})
|
||||
cfg, err := pgxpool.ParseConfig(dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.ConnConfig.RuntimeParams == nil {
|
||||
cfg.ConnConfig.RuntimeParams = make(map[string]string)
|
||||
}
|
||||
cfg.ConnConfig.RuntimeParams["search_path"] = schema
|
||||
cfg.MaxConns = 3
|
||||
pool, err = pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return pool
|
||||
}
|
||||
|
||||
func assertMigrationVersionCount(t *testing.T, pool *pgxpool.Pool, want int) {
|
||||
t.Helper()
|
||||
var got int
|
||||
if err := pool.QueryRow(context.Background(), `SELECT count(*) FROM schema_migrations`).Scan(&got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got != want {
|
||||
t.Fatalf("migration versions = %d, want %d", got, want)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// These regressions ensure unobserved live state cannot inherit persisted health.
|
||||
func TestStage0ReproPeerWithoutObservationIsNotEstablished(t *testing.T) {
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||
cases := []struct {
|
||||
name string
|
||||
views []speakerBGPLive
|
||||
}{
|
||||
{name: "no_collector"},
|
||||
{name: "all_timeout", views: []speakerBGPLive{{SpeakerID: "replica", Error: "timeout"}}},
|
||||
{name: "session_absent", views: []speakerBGPLive{{SpeakerID: "replica", Label: "replica"}}},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, peer, tc.views)
|
||||
if row["session_state"] == "Established" {
|
||||
t.Fatal("S01: persisted Established is exposed without a current session observation")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestStage0ReproUnpolledMasterIsNotHealthy(t *testing.T) {
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
// collectSpeakerLiveStatus constructs this view when no poll result exists.
|
||||
view := speakerBGPLive{SpeakerID: sp.ID, Label: "master"}
|
||||
row := speakerLiveStatusJSON(sp, view, nil)
|
||||
if row["bgp_poll_ok"] == true || row["agent_ok"] == true {
|
||||
t.Fatal("S01: absent poll is exposed as successful BGP/agent health")
|
||||
}
|
||||
}
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"time"
|
||||
|
||||
"evobgp/internal/db"
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/repository"
|
||||
"evobgp/internal/store"
|
||||
|
||||
@@ -18,7 +18,7 @@ import (
|
||||
|
||||
// NewCDNHTTPClient returns the shared HTTP client for CDN and preview fetches (PERF-02 / ERR-03).
|
||||
func NewCDNHTTPClient() *http.Client {
|
||||
return httpclient.New(httpclient.DefaultTimeout)
|
||||
return pipeline.NewCDNHTTPClient()
|
||||
}
|
||||
|
||||
// BootstrapWorkers opens the same store.Backend and jobs.Registry as New (without HTTP or bundle keys).
|
||||
@@ -50,46 +50,15 @@ func BootstrapWorkers(ctx context.Context, opts Options) (store.Backend, *jobs.R
|
||||
backend = mem
|
||||
}
|
||||
|
||||
cdnHTTP := NewCDNHTTPClient()
|
||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: cdnHTTP}
|
||||
// Configured DoH profiles and RIPE service overrides retain the previous
|
||||
// general HTTP/HTTPS transport, including explicitly configured private
|
||||
// endpoints. CDN calls independently secure this client in UpstreamHTTPDo.
|
||||
workerHTTP := pipeline.NewPipelineHTTPClient()
|
||||
wk := &jobs.Worker{Store: backend, PgPool: pool, HTTPClient: workerHTTP}
|
||||
reg := jobs.NewRegistry(wk.Process)
|
||||
wk.Registry = reg
|
||||
if pool != nil {
|
||||
audit := repository.NewJobAuditWriter(pool)
|
||||
jobMeta := func(j *jobs.Job) map[string]any {
|
||||
if j == nil {
|
||||
return nil
|
||||
}
|
||||
st := j.Snapshot()
|
||||
meta, _ := st["meta"].(map[string]any)
|
||||
return meta
|
||||
}
|
||||
reg.SetPersistHooks(
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
audit.UpsertQueued(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, j.ModuleID, jobMeta(j))
|
||||
},
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
audit.UpsertRunning(context.Background(), j.TenantID, j.ID, j.Kind, j.IdempotencyKey, jobMeta(j))
|
||||
},
|
||||
func(j *jobs.Job) {
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
st := j.Snapshot()
|
||||
status, _ := st["status"].(string)
|
||||
var errMsg *string
|
||||
if e, ok := st["error"].(string); ok && e != "" {
|
||||
errMsg = &e
|
||||
}
|
||||
audit.MarkTerminal(context.Background(), j.TenantID, j.ID, status, errMsg, time.Now().UTC())
|
||||
},
|
||||
)
|
||||
reg.UseDurableQueue(backend)
|
||||
}
|
||||
observability.RegisterStoreBackend(backend)
|
||||
return backend, reg, pool, nil
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/store"
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
func TestBootstrapWorkerPreservesExplicitPrivateDOH(t *testing.T) {
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||
t.Setenv("EVOBGP_BIRDC_SOCKET", "")
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "0")
|
||||
var calls atomic.Int32
|
||||
resolver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
raw, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns"))
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
query := new(dns.Msg)
|
||||
if err = query.Unpack(raw); err != nil || len(query.Question) == 0 {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
answer := new(dns.Msg)
|
||||
answer.SetReply(query)
|
||||
if query.Question[0].Qtype == dns.TypeA {
|
||||
answer.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: query.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.IPv4(192, 0, 2, 20)}}
|
||||
}
|
||||
body, err := answer.Pack()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/dns-message")
|
||||
_, _ = w.Write(body)
|
||||
}))
|
||||
defer resolver.Close()
|
||||
st, reg, _, err := BootstrapWorkers(context.Background(), Options{SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
reg.RequestCancelAll()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if e := reg.Drain(ctx); e != nil {
|
||||
t.Error(e)
|
||||
}
|
||||
}()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
profile, err := st.CreateDohProfile(tenant, &store.DohProfile{Name: "explicit private resolver", URL: resolver.URL})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Name: "private-doh", Type: "DOMAINS", Enabled: true, DohProfileIDs: []string{profile.ID}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = st.CreateDomainEntry(tenant, mod.ID, &store.DomainEntry{FQDN: "private-resolver.example.test"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j, _, err := reg.Enqueue(tenant, jobs.KindModuleRefresh, nil, &mod.ID, map[string]any{"module_id": mod.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
deadline := time.After(5 * time.Second)
|
||||
ticker := time.NewTicker(5 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
snap := j.Snapshot()
|
||||
status := snap["status"]
|
||||
if status == jobs.StatusSucceeded {
|
||||
break
|
||||
}
|
||||
if status == jobs.StatusFailed || status == jobs.StatusCancelled {
|
||||
t.Fatalf("worker refresh failed: %+v", snap)
|
||||
}
|
||||
select {
|
||||
case <-deadline:
|
||||
t.Fatal("worker did not finish")
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
if calls.Load() < 2 {
|
||||
t.Fatalf("bootstrap worker blocked private DoH: %d requests", calls.Load())
|
||||
}
|
||||
// The same general-client choice must not remove CDN's independent ACL.
|
||||
cdn := httptest.NewTLSServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Error("private CDN reached") }))
|
||||
defer cdn.Close()
|
||||
req, err := http.NewRequest(http.MethodGet, cdn.URL, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = pipeline.UpstreamHTTPDo(context.Background(), httpclient.New(httpclient.DefaultTimeout), req); err == nil {
|
||||
t.Fatal("private CDN accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestPreviewCDNSourceRejectsOversizedBody(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", (8<<20)+1)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
st := store.NewMemory()
|
||||
st.SeedDemo()
|
||||
tenant, _, _, _, _ := st.DemoIDs()
|
||||
mod, err := st.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "oversized-preview", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Server{store: st, cdnHTTP: srv.Client()}
|
||||
body, err := json.Marshal(map[string]string{"url": srv.URL, "source_kind": "plain"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+mod.ID+"/cdn-sources/preview", strings.NewReader(string(body)))
|
||||
req.SetPathValue("module_id", mod.ID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||
recorder := httptest.NewRecorder()
|
||||
s.handlePreviewCDNSource(recorder, req)
|
||||
if recorder.Code != http.StatusBadGateway {
|
||||
t.Fatalf("oversized preview status=%d body=%s", recorder.Code, recorder.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreviewCDNErrorBodyBoundAndQuotaBeforeFetch(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
t.Setenv("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, module, _, _, _ := m.DemoIDs()
|
||||
var calls atomic.Int32
|
||||
upstream := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
|
||||
w.(http.Flusher).Flush()
|
||||
// No EOF: an unbounded drain would wait until the caller deadline.
|
||||
<-r.Context().Done()
|
||||
}))
|
||||
defer upstream.Close()
|
||||
s := &Server{store: m, cdnHTTP: upstream.Client()}
|
||||
request := func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/cdn-sources/preview", strings.NewReader(`{"url":"`+upstream.URL+`","source_kind":"plain"}`))
|
||||
r.SetPathValue("module_id", module)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "editor"}))
|
||||
}
|
||||
first := httptest.NewRecorder()
|
||||
firstRequest := request()
|
||||
ctx, cancel := context.WithTimeout(firstRequest.Context(), 5*time.Second)
|
||||
defer cancel()
|
||||
s.handlePreviewCDNSource(first, firstRequest.WithContext(ctx))
|
||||
if first.Code != http.StatusBadGateway || calls.Load() != 1 || ctx.Err() != nil {
|
||||
t.Fatalf("unbounded drain status=%d calls=%d ctx=%v", first.Code, calls.Load(), ctx.Err())
|
||||
}
|
||||
second := httptest.NewRecorder()
|
||||
s.handlePreviewCDNSource(second, request())
|
||||
if second.Code != http.StatusTooManyRequests || calls.Load() != 1 || second.Header().Get("Retry-After") == "" {
|
||||
t.Fatalf("quota fetch bypass status=%d calls=%d", second.Code, calls.Load())
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ func (s *Server) withCORS(h http.Handler) http.Handler {
|
||||
}
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, Idempotency-Key, Accept, X-Tenant-Id")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PATCH, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "Location, X-EvoBGP-Job-Status, Retry-After")
|
||||
if r.Method == http.MethodOptions {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
return
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func (s *Server) replayJobResponse(w http.ResponseWriter, r *http.Request, in store.JobEnqueue) bool {
|
||||
j, found, err := s.jobs.ReplayContext(r.Context(), in)
|
||||
if err != nil {
|
||||
writeJobEnqueueError(w, err)
|
||||
return true
|
||||
}
|
||||
if !found {
|
||||
return false
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
snap := j.Snapshot()
|
||||
body := map[string]any{"job_id": j.ID, "status": snap["status"]}
|
||||
if j.Kind == jobs.KindTenantRefresh {
|
||||
ids := snap["meta"].(map[string]any)["module_ids"]
|
||||
n := 0
|
||||
switch v := ids.(type) {
|
||||
case []any:
|
||||
n = len(v)
|
||||
case []string:
|
||||
n = len(v)
|
||||
}
|
||||
body["module_ids"] = ids
|
||||
body["modules_due"] = n
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, body)
|
||||
return true
|
||||
}
|
||||
|
||||
func writeJobEnqueueError(w http.ResponseWriter, err error) {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrJobConflict):
|
||||
writeProblem(w, http.StatusConflict, "Conflict", "idempotency key belongs to a different job request")
|
||||
case errors.Is(err, store.ErrJobQueueFull):
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "job queue is full")
|
||||
default:
|
||||
writeInternalError(w, "job_enqueue", err)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJobReadError(w http.ResponseWriter, err error) {
|
||||
if errors.Is(err, jobs.ErrNotFound) || errors.Is(err, store.ErrJobNotFound) {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
return
|
||||
}
|
||||
writeInternalError(w, "job_read", err)
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type unavailableJobQueue struct{ store.JobQueue }
|
||||
|
||||
func (q unavailableJobQueue) EnqueueJob(context.Context, store.JobEnqueue) (store.JobRecord, bool, error) {
|
||||
return store.JobRecord{}, false, errors.New("database unavailable")
|
||||
}
|
||||
|
||||
func TestJobEnqueueFailureNeverAccepts(t *testing.T) {
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(unavailableJobQueue{})
|
||||
s := &Server{jobs: r}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/bird/reload", nil)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleBirdReload(w, req)
|
||||
if w.Code == http.StatusAccepted || w.Code < 500 {
|
||||
t.Fatalf("DB failure returned %d: %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAutomaticEnqueueFailureIsVisible(t *testing.T) {
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(unavailableJobQueue{})
|
||||
s := &Server{jobs: r}
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/peers", nil)
|
||||
s.enqueuePeerReconcile(w, req, "tenant", "peer_create")
|
||||
if got := w.Header().Get("X-EvoBGP-Job-Status"); got != "enqueue-failed" {
|
||||
t.Fatalf("scheduling failure hidden: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
type selectorBackend struct {
|
||||
store.Backend
|
||||
modules []*store.Module
|
||||
reads int
|
||||
}
|
||||
|
||||
func (b *selectorBackend) ListModules(string) []*store.Module { b.reads++; return b.modules }
|
||||
|
||||
func TestTenantRefreshReplayBeforeDynamicSelection(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
b := &selectorBackend{Backend: m, modules: []*store.Module{{ID: "module-a", Enabled: true, RefreshIntervalSec: 1}}}
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
s := &Server{store: b, jobs: r}
|
||||
request := func() *httptest.ResponseRecorder {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/tenant/refresh", strings.NewReader(`{}`))
|
||||
req.Header.Set("Idempotency-Key", "same-request")
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: "tenant", Role: "editor"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleTenantRefresh(w, req)
|
||||
return w
|
||||
}
|
||||
first := request()
|
||||
if first.Code != http.StatusAccepted {
|
||||
t.Fatalf("first=%d %s", first.Code, first.Body.String())
|
||||
}
|
||||
var accepted map[string]any
|
||||
if err := json.Unmarshal(first.Body.Bytes(), &accepted); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := m.ClaimJobs(context.Background(), "worker", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claim %+v %v", claimed, err)
|
||||
}
|
||||
claimed[0].Status = "succeeded"
|
||||
if err = m.FinishJob(context.Background(), claimed[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.modules = nil
|
||||
reads := b.reads
|
||||
replay := request()
|
||||
if replay.Code != http.StatusAccepted {
|
||||
t.Fatalf("replay=%d %s", replay.Code, replay.Body.String())
|
||||
}
|
||||
var got map[string]any
|
||||
if err = json.Unmarshal(replay.Body.Bytes(), &got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got["job_id"] != accepted["job_id"] || got["status"] != "succeeded" || b.reads != reads {
|
||||
t.Fatalf("replay re-resolved selector: %+v reads %d->%d", got, reads, b.reads)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestJobReportDurableProgressAndTenantScope(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
j, _, err := m.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: jobs.KindDeployApply})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claim: %v %v", claimed, err)
|
||||
}
|
||||
f := store.JobFence{JobID: j.ID, TenantID: tenant, OwnerID: "worker", Token: claimed[0].FencingToken}
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unknown", "unconfirmed", "response lost"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// A fresh registry has no in-process job or progress metadata.
|
||||
r := jobs.NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
s := &Server{store: m, jobs: r}
|
||||
request := func(tenantID, jobID string) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/jobs/"+jobID+"/report", nil)
|
||||
req.SetPathValue("job_id", jobID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenantID, Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleGetJobReport(w, req)
|
||||
return w
|
||||
}
|
||||
check := func(id string) {
|
||||
t.Helper()
|
||||
w := request(tenant, id)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("report: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
var body struct {
|
||||
Progress []struct {
|
||||
State, Confirmation string
|
||||
JobID string `json:"job_id"`
|
||||
} `json:"deployment_progress"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(body.Progress) != 1 || body.Progress[0].JobID != j.ID || body.Progress[0].State != "unknown" || body.Progress[0].Confirmation != "unconfirmed" {
|
||||
t.Fatalf("lost persisted progress: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
check(j.ID)
|
||||
recovery, _, err := r.EnqueueContext(ctx, tenant, jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
check(recovery.ID)
|
||||
// A later explicit completion releases the delivery reservation. Once recovery
|
||||
// has its own persisted row, its report must not show the parent's old state.
|
||||
if err := m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recoveryRows, err := m.ClaimJobs(ctx, "recovery-worker", 1, time.Minute)
|
||||
if err != nil || len(recoveryRows) != 1 || recoveryRows[0].ID != recovery.ID {
|
||||
t.Fatalf("recovery claim: %v %v", recoveryRows, err)
|
||||
}
|
||||
rf := store.JobFence{JobID: recovery.ID, TenantID: tenant, OwnerID: "recovery-worker", Token: recoveryRows[0].FencingToken}
|
||||
rd, err := m.BeginDeploymentSpeaker(ctx, rf, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.RecordDeploymentSpeaker(ctx, rf, rd, "applied", "agent_reported", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w := request(tenant, recovery.ID)
|
||||
var report struct {
|
||||
Progress []struct {
|
||||
State string
|
||||
JobID string `json:"job_id"`
|
||||
} `json:"deployment_progress"`
|
||||
}
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &report); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w.Code != http.StatusOK || len(report.Progress) != 1 || report.Progress[0].JobID != recovery.ID || report.Progress[0].State != "applied" {
|
||||
t.Fatalf("recovery own progress hidden: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
if w := request("other-tenant", j.ID); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("cross-tenant report: %d", w.Code)
|
||||
}
|
||||
foreign, _, err := r.EnqueueContext(ctx, "other-tenant", jobs.KindDeployApply, nil, nil, map[string]any{"source_deployment_job_id": j.ID})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if w := request("other-tenant", foreign.ID); w.Code != http.StatusNotFound {
|
||||
t.Fatalf("cross-tenant source: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"net/http/httptest"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
)
|
||||
|
||||
// TestLegacyNodeBinaryCurrentControlPlane exercises an independently built old
|
||||
// CLI against today's HTTP handlers. It never calls apply or touches host BIRD.
|
||||
func TestLegacyNodeBinaryCurrentControlPlane(t *testing.T) {
|
||||
binary := os.Getenv("EVOBGP_TEST_LEGACY_NODE_BIN")
|
||||
if binary == "" {
|
||||
t.Skip("EVOBGP_TEST_LEGACY_NODE_BIN not set")
|
||||
}
|
||||
binary, err := filepath.Abs(binary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(binary); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,other-node|other-tenant|node")
|
||||
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server := httptest.NewServer(srv.Handler())
|
||||
defer server.Close()
|
||||
run := func(wantSuccess bool, args ...string) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
output, err := exec.CommandContext(ctx, binary, args...).CombinedOutput()
|
||||
if ctx.Err() != nil {
|
||||
t.Fatalf("legacy CLI timed out: %v", ctx.Err())
|
||||
}
|
||||
if wantSuccess && err != nil {
|
||||
t.Fatalf("legacy CLI failed: %v: %s", err, output)
|
||||
}
|
||||
if !wantSuccess && err == nil {
|
||||
t.Fatalf("legacy CLI unexpectedly accepted invalid request: %s", output)
|
||||
}
|
||||
}
|
||||
path := filepath.Join(t.TempDir(), "current-bundle.tar.gz")
|
||||
run(true, "pull-bundle", "-base-url", server.URL, "-token", "legacy-node", "-speaker-id", speaker, "-o", path)
|
||||
run(true, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||
if err != nil || verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker {
|
||||
t.Fatalf("legacy CLI fetched the wrong revision/speaker or an invalid bundle: %v", err)
|
||||
}
|
||||
// The archive is valid, but verification with an unrelated key must fail.
|
||||
run(false, "verify-bundle", "-f", path, "-pubkey-hex", strings.Repeat("00", ed25519.PublicKeySize))
|
||||
for _, token := range []string{"invalid-node", "other-node"} {
|
||||
denied := filepath.Join(t.TempDir(), "denied.tar.gz")
|
||||
run(false, "pull-bundle", "-base-url", server.URL, "-token", token, "-speaker-id", speaker, "-o", denied)
|
||||
if _, err := os.Stat(denied); !os.IsNotExist(err) {
|
||||
t.Fatalf("unauthorized CLI wrote a bundle: %v", err)
|
||||
}
|
||||
}
|
||||
// A damaged payload must not pass the old verifier with the correct key.
|
||||
if err := os.WriteFile(path, []byte("not a signed bundle"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
run(false, "verify-bundle", "-f", path, "-pubkey-base64", srv.BundlePublicKeyBase64())
|
||||
}
|
||||
@@ -0,0 +1,310 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/pipeline"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
// nodeBundleCacheMaxEntries is the maximum number of finished node bundles retained.
|
||||
nodeBundleCacheMaxEntries = 32
|
||||
// nodeBundleCacheMaxBytes is the maximum total size of cached bundle bodies (64 MiB).
|
||||
nodeBundleCacheMaxBytes = 64 << 20
|
||||
)
|
||||
|
||||
// errNodeBundleRevisionMissing is returned when the revision cannot be loaded.
|
||||
// It is not cached: the next pull tries again.
|
||||
var errNodeBundleRevisionMissing = errors.New("httpapi: revision not found")
|
||||
|
||||
// nodeBundleOpError carries the writeInternalError operation name for a failed build.
|
||||
type nodeBundleOpError struct {
|
||||
op string
|
||||
err error
|
||||
}
|
||||
|
||||
func (e *nodeBundleOpError) Error() string {
|
||||
if e == nil || e.err == nil {
|
||||
return "httpapi: node bundle"
|
||||
}
|
||||
return fmt.Sprintf("httpapi: %s: %v", e.op, e.err)
|
||||
}
|
||||
|
||||
func (e *nodeBundleOpError) Unwrap() error { return e.err }
|
||||
|
||||
// nodeBundleBuilder produces the gzip body served to evobgp-node.
|
||||
// The default implementation is bundle.BuildGzippedTar, whose archive contains
|
||||
// manifest.json and bundle.sig. Tests replace it to count rebuilds.
|
||||
type nodeBundleBuilder func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error)
|
||||
|
||||
// nodeBundleCache stores finished gzip bodies. A hit returns the bytes of the
|
||||
// build that filled the entry, signature included. Errors are not stored.
|
||||
// Concurrent loads of one key share a single build.
|
||||
type nodeBundleCache struct {
|
||||
mu sync.Mutex
|
||||
maxN int
|
||||
maxBytes int
|
||||
total int
|
||||
items map[string]*nodeBundleEntry
|
||||
lru []string // index 0 is the oldest idle entry
|
||||
flights map[string]*nodeBundleFlight
|
||||
}
|
||||
|
||||
type nodeBundleEntry struct {
|
||||
body []byte
|
||||
}
|
||||
|
||||
type nodeBundleFlight struct {
|
||||
done chan struct{}
|
||||
body []byte
|
||||
err error
|
||||
joined int
|
||||
}
|
||||
|
||||
func newNodeBundleCache(maxEntries, maxBytes int) *nodeBundleCache {
|
||||
return &nodeBundleCache{
|
||||
maxN: maxEntries,
|
||||
maxBytes: maxBytes,
|
||||
items: make(map[string]*nodeBundleEntry),
|
||||
flights: make(map[string]*nodeBundleFlight),
|
||||
}
|
||||
}
|
||||
|
||||
// nodeBundleCacheKey identifies one servable bundle.
|
||||
// overlayFP is a digest of the speaker overlay output (router id / source addresses,
|
||||
// node IPv4 fallback, peer and discovery render). Revision content itself is immutable,
|
||||
// so the revision id stands in for the shared prefix files.
|
||||
func nodeBundleCacheKey(tenantID, revisionID, speakerID, overlayFP string) string {
|
||||
var b strings.Builder
|
||||
b.Grow(len(tenantID) + len(revisionID) + len(speakerID) + len(overlayFP) + 3)
|
||||
b.WriteString(tenantID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(revisionID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(speakerID)
|
||||
b.WriteByte(0)
|
||||
b.WriteString(overlayFP)
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// nodeBundleOverlayFingerprint hashes the speaker overlay that handleNodeBundle
|
||||
// would apply on top of an immutable revision. Empty base fragments are enough:
|
||||
// bird.conf and the peers fragment are fully re-rendered from tenant settings,
|
||||
// speaker source addresses, and the live peer set. Revision file bytes are not
|
||||
// part of this digest.
|
||||
func nodeBundleOverlayFingerprint(st store.Backend, tenantID, speakerID, revisionID string) (string, error) {
|
||||
frags, err := pipeline.OverlayFragmentsForSpeaker(st, tenantID, speakerID, revisionID, nil)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
keys := make([]string, 0, len(frags))
|
||||
for k := range frags {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
h := sha256.New()
|
||||
for _, k := range keys {
|
||||
if err := hashWriteString(h, k); err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
if err := hashWriteString(h, frags[k]); err != nil {
|
||||
return "", fmt.Errorf("httpapi: bundle overlay: %w", err)
|
||||
}
|
||||
}
|
||||
return hex.EncodeToString(h.Sum(nil)), nil
|
||||
}
|
||||
|
||||
func hashWriteString(h interface{ Write([]byte) (int, error) }, s string) error {
|
||||
var buf [8]byte
|
||||
binary.BigEndian.PutUint64(buf[:], uint64(len(s)))
|
||||
if _, err := h.Write(buf[:]); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := h.Write([]byte(s))
|
||||
return err
|
||||
}
|
||||
|
||||
// do returns the cached body or runs build once for key.
|
||||
// build's bool is false when the result must not be stored (failure is also not stored).
|
||||
// Callers must not mutate the returned slice.
|
||||
func (c *nodeBundleCache) do(ctx context.Context, key string, build func(context.Context) ([]byte, bool, error)) (body []byte, err error) {
|
||||
if c == nil {
|
||||
return nil, errors.New("httpapi: nil node bundle cache")
|
||||
}
|
||||
if ctx == nil {
|
||||
ctx = context.Background()
|
||||
}
|
||||
c.mu.Lock()
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.touchLocked(key)
|
||||
body = e.body
|
||||
c.mu.Unlock()
|
||||
return body, nil
|
||||
}
|
||||
if f, ok := c.flights[key]; ok {
|
||||
f.joined++
|
||||
c.mu.Unlock()
|
||||
return waitNodeBundleFlight(ctx, f)
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
c.mu.Unlock()
|
||||
return nil, err
|
||||
}
|
||||
f := &nodeBundleFlight{done: make(chan struct{})}
|
||||
c.flights[key] = f
|
||||
c.mu.Unlock()
|
||||
|
||||
var cacheOK bool
|
||||
var finished bool
|
||||
finish := func() {
|
||||
if finished {
|
||||
return
|
||||
}
|
||||
finished = true
|
||||
out := body
|
||||
c.mu.Lock()
|
||||
delete(c.flights, key)
|
||||
if err == nil && cacheOK {
|
||||
if stored, ok := c.storeLocked(key, out); ok {
|
||||
out = stored
|
||||
}
|
||||
}
|
||||
f.body = out
|
||||
f.err = err
|
||||
close(f.done)
|
||||
c.mu.Unlock()
|
||||
body = out
|
||||
}
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
body = nil
|
||||
cacheOK = false
|
||||
err = fmt.Errorf("httpapi: node bundle build: %v", rec)
|
||||
}
|
||||
finish()
|
||||
}()
|
||||
|
||||
// Detach from the leader's cancel so one disconnected agent does not fail the shared build.
|
||||
body, cacheOK, err = build(context.WithoutCancel(ctx))
|
||||
return
|
||||
}
|
||||
|
||||
func waitNodeBundleFlight(ctx context.Context, f *nodeBundleFlight) ([]byte, error) {
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.body, f.err
|
||||
case <-ctx.Done():
|
||||
select {
|
||||
case <-f.done:
|
||||
return f.body, f.err
|
||||
default:
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// storeLocked retains body when it fits under both caps, evicting the oldest idle
|
||||
// entries first. body is copied. The stored slice is returned.
|
||||
func (c *nodeBundleCache) storeLocked(key string, body []byte) ([]byte, bool) {
|
||||
n := len(body)
|
||||
if n == 0 || c.maxN <= 0 || c.maxBytes <= 0 || n > c.maxBytes {
|
||||
return nil, false
|
||||
}
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.total -= len(e.body)
|
||||
delete(c.items, key)
|
||||
c.removeLRULocked(key)
|
||||
}
|
||||
for len(c.items) > 0 && (len(c.items) >= c.maxN || c.total+n > c.maxBytes) {
|
||||
if len(c.lru) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
c.deleteOldestLocked()
|
||||
}
|
||||
if len(c.items) >= c.maxN || c.total+n > c.maxBytes {
|
||||
return nil, false
|
||||
}
|
||||
stored := make([]byte, n)
|
||||
copy(stored, body)
|
||||
c.items[key] = &nodeBundleEntry{body: stored}
|
||||
c.total += n
|
||||
c.lru = append(c.lru, key)
|
||||
return stored, true
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) touchLocked(key string) {
|
||||
c.removeLRULocked(key)
|
||||
c.lru = append(c.lru, key)
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) deleteOldestLocked() {
|
||||
if len(c.lru) == 0 {
|
||||
return
|
||||
}
|
||||
key := c.lru[0]
|
||||
c.lru = c.lru[1:]
|
||||
if e, ok := c.items[key]; ok {
|
||||
c.total -= len(e.body)
|
||||
delete(c.items, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *nodeBundleCache) removeLRULocked(key string) {
|
||||
for i, k := range c.lru {
|
||||
if k != key {
|
||||
continue
|
||||
}
|
||||
copy(c.lru[i:], c.lru[i+1:])
|
||||
c.lru = c.lru[:len(c.lru)-1]
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) buildNodeBundle(revisionID, speakerID string, fragments map[string]string) ([]byte, error) {
|
||||
if s != nil && s.nodeBundleBuild != nil {
|
||||
return s.nodeBundleBuild(revisionID, speakerID, fragments, s.bundlePriv)
|
||||
}
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, s.bundlePriv)
|
||||
}
|
||||
|
||||
// loadNodeBundle reads the revision, applies the speaker overlay, and signs the gzip.
|
||||
// The bool is false when overlay inputs moved during the load, so the caller must not
|
||||
// cache the body under the key captured before the load. Failed loads are not cacheable.
|
||||
func (s *Server) loadNodeBundle(ctx context.Context, tenantID, speakerID, revisionID, overlayFP string) ([]byte, bool, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
fpNow, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
rev, err := s.store.GetRevision(tenantID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, errNodeBundleRevisionMissing
|
||||
}
|
||||
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, tenantID, speakerID, revisionID, rev.PreviewFragments)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
fpAfter, err := nodeBundleOverlayFingerprint(s.store, tenantID, speakerID, revisionID)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "bundle overlay", err: err}
|
||||
}
|
||||
body, err := s.buildNodeBundle(revisionID, speakerID, overlaid)
|
||||
if err != nil {
|
||||
return nil, false, &nodeBundleOpError{op: "internal", err: err}
|
||||
}
|
||||
cacheable := fpNow == overlayFP && fpAfter == fpNow && len(body) > 0
|
||||
return body, cacheable, nil
|
||||
}
|
||||
@@ -0,0 +1,791 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/signing"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestNodeBundleCacheKeySeparatesTenantsRevisionsSpeakers(t *testing.T) {
|
||||
a := nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay")
|
||||
if a == nodeBundleCacheKey("tenant-b", "rev", "spk", "overlay") {
|
||||
t.Fatal("tenants collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev-2", "spk", "overlay") {
|
||||
t.Fatal("revisions collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev", "spk-2", "overlay") {
|
||||
t.Fatal("speakers collided")
|
||||
}
|
||||
if a == nodeBundleCacheKey("tenant-a", "rev", "spk", "overlay-2") {
|
||||
t.Fatal("overlay inputs collided")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodeBundleCacheHitSingleflightAndErrors(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 1<<20)
|
||||
var builds atomic.Int32
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("signed-body"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||
}
|
||||
if !bytes.Equal(got, []byte("signed-body")) {
|
||||
t.Fatalf("hit body = %q", got)
|
||||
}
|
||||
|
||||
t.Run("singleflight", func(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 1<<20)
|
||||
var builds atomic.Int32
|
||||
release := make(chan struct{})
|
||||
const n = 8
|
||||
var wg sync.WaitGroup
|
||||
bodies := make([][]byte, n)
|
||||
errs := make([]error, n)
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
bodies[i], errs[i] = c.do(context.Background(), "same", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return []byte("signed-bundle"), true, nil
|
||||
})
|
||||
}(i)
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
joined := 0
|
||||
if f := c.flights["same"]; f != nil {
|
||||
joined = f.joined
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if builds.Load() > 1 {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("builds = %d before followers joined", builds.Load())
|
||||
}
|
||||
if builds.Load() == 1 && joined == n-1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("builds=%d joined=%d", builds.Load(), joined)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if errs[i] != nil {
|
||||
t.Fatalf("waiter %d: %v", i, errs[i])
|
||||
}
|
||||
if !bytes.Equal(bodies[i], []byte("signed-bundle")) {
|
||||
t.Fatalf("waiter %d body = %q", i, bodies[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("canceled waiter does not drop the build", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
release := make(chan struct{})
|
||||
var builds atomic.Int32
|
||||
leaderErr := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return []byte("body"), true, nil
|
||||
})
|
||||
leaderErr <- err
|
||||
}()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for {
|
||||
c.mu.Lock()
|
||||
_, ok := c.flights["k"]
|
||||
c.mu.Unlock()
|
||||
if ok || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err := c.do(ctx, "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("nope"), true, nil
|
||||
})
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("follower err = %v", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-leaderErr; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d", builds.Load())
|
||||
}
|
||||
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
builds.Add(1)
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if builds.Load() != 1 {
|
||||
t.Fatal("successful build was not cached after a canceled waiter")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("errors are not cached", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
calls := 0
|
||||
boom := errors.New("httpapi: bundle build failed")
|
||||
_, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return nil, true, boom
|
||||
})
|
||||
if !errors.Is(err, boom) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
_, err = c.do(context.Background(), "miss", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return nil, false, errNodeBundleRevisionMissing
|
||||
})
|
||||
if !errors.Is(err, errNodeBundleRevisionMissing) {
|
||||
t.Fatalf("err = %v", err)
|
||||
}
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("ok"), true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got) != "ok" || calls != 3 {
|
||||
t.Fatalf("calls=%d body=%q", calls, got)
|
||||
}
|
||||
_, err = c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("other"), true, nil
|
||||
})
|
||||
if err != nil || calls != 3 {
|
||||
t.Fatalf("cached error path rebuilt: calls=%d err=%v", calls, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("unstorable success is not cached", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 1024)
|
||||
calls := 0
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := c.do(context.Background(), "k", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return []byte("fresh"), false, nil
|
||||
})
|
||||
if err != nil || string(got) != "fresh" {
|
||||
t.Fatalf("i=%d err=%v body=%q", i, err, got)
|
||||
}
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("calls = %d, want 2", calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNodeBundleCacheEvictsIdleAndRespectsCaps(t *testing.T) {
|
||||
t.Run("entry cap evicts oldest idle", func(t *testing.T) {
|
||||
c := newNodeBundleCache(2, 100)
|
||||
put := func(key, body string) {
|
||||
t.Helper()
|
||||
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||
return []byte(body), true, nil
|
||||
})
|
||||
if err != nil || string(got) != body {
|
||||
t.Fatalf("put %s: %v %q", key, err, got)
|
||||
}
|
||||
}
|
||||
put("a", "A")
|
||||
put("b", "B")
|
||||
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||
t.Fatal("recent entry was rebuilt")
|
||||
return nil, false, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("c", "C")
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if _, ok := c.items["b"]; ok {
|
||||
t.Fatal("idle entry b was kept")
|
||||
}
|
||||
if _, ok := c.items["a"]; !ok || c.items["c"] == nil {
|
||||
t.Fatalf("kept %v", cacheKeys(c))
|
||||
}
|
||||
if len(c.items) > 2 {
|
||||
t.Fatalf("entry cap exceeded: %d", len(c.items))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("byte cap", func(t *testing.T) {
|
||||
c := newNodeBundleCache(8, 10)
|
||||
put := func(key string, n int) {
|
||||
t.Helper()
|
||||
body := bytes.Repeat([]byte("x"), n)
|
||||
got, err := c.do(context.Background(), key, func(context.Context) ([]byte, bool, error) {
|
||||
return body, true, nil
|
||||
})
|
||||
if err != nil || !bytes.Equal(got, body) {
|
||||
t.Fatalf("put %s: %v len=%d", key, err, len(got))
|
||||
}
|
||||
assertBundleCacheAccounting(t, c)
|
||||
}
|
||||
put("a", 4)
|
||||
put("b", 4)
|
||||
if _, err := c.do(context.Background(), "a", func(context.Context) ([]byte, bool, error) {
|
||||
t.Fatal("rebuilt a")
|
||||
return nil, false, nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put("c", 4)
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if _, ok := c.items["b"]; ok {
|
||||
t.Fatal("byte cap kept idle entry b")
|
||||
}
|
||||
if len(c.items) != 2 || c.total != 8 {
|
||||
t.Fatalf("n=%d total=%d", len(c.items), c.total)
|
||||
}
|
||||
put("d", 6)
|
||||
put("e", 6)
|
||||
assertBundleCacheAccounting(t, c)
|
||||
if len(c.items) != 1 || c.total != 6 {
|
||||
t.Fatalf("after oversized pair n=%d total=%d keys=%v", len(c.items), c.total, cacheKeys(c))
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("entry larger than the byte cap is not stored", func(t *testing.T) {
|
||||
c := newNodeBundleCache(4, 8)
|
||||
calls := 0
|
||||
big := bytes.Repeat([]byte("y"), 9)
|
||||
for i := 0; i < 2; i++ {
|
||||
got, err := c.do(context.Background(), "big", func(context.Context) ([]byte, bool, error) {
|
||||
calls++
|
||||
return big, true, nil
|
||||
})
|
||||
if err != nil || !bytes.Equal(got, big) {
|
||||
t.Fatalf("i=%d err=%v", i, err)
|
||||
}
|
||||
}
|
||||
if calls != 2 {
|
||||
t.Fatalf("oversize body was cached, calls=%d", calls)
|
||||
}
|
||||
if len(c.items) != 0 || c.total != 0 {
|
||||
t.Fatalf("stored oversize n=%d total=%d", len(c.items), c.total)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestNodeBundleOverlayFingerprintInputs(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, rev, speaker := m.DemoIDs()
|
||||
|
||||
base, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if base != again {
|
||||
t.Fatal("fingerprint changed without an overlay input change")
|
||||
}
|
||||
|
||||
sp, err := m.GetSpeaker(tenant, speaker)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dispatch := store.MergeSpeakerMetaJSON(sp.MetaJSON, store.SpeakerMeta{
|
||||
LastDispatchAt: "2026-01-01T00:00:00Z",
|
||||
LastDispatchStatus: "ok",
|
||||
SyncStatus: "synced",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &dispatch}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||
t.Fatalf("dispatch meta changed fingerprint: %v %s", err, got)
|
||||
}
|
||||
|
||||
peers := m.ListPeers(tenant)
|
||||
if len(peers) == 0 {
|
||||
t.Fatal("expected demo peers")
|
||||
}
|
||||
state := "Active"
|
||||
if _, err := m.UpdatePeer(tenant, peers[0].ID, &store.PeerPatch{SessionState: &state}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != base {
|
||||
t.Fatalf("session state changed fingerprint: %v %s", err, got)
|
||||
}
|
||||
|
||||
if err := m.PatchGlobalSettings(tenant, map[string]any{"bird_router_id": "203.0.113.7"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
routed, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if routed == base {
|
||||
t.Fatal("router id setting did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
nodeMeta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeMeta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
nodeFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nodeFP == routed {
|
||||
t.Fatal("node ipv4 did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
sourceMeta := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.9",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &sourceMeta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if sourceFP == nodeFP {
|
||||
t.Fatal("source ipv4 did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
nodeOnly := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.8",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &nodeOnly}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev); err != nil || got != sourceFP {
|
||||
t.Fatalf("unused node ipv4 changed fingerprint: %v %s vs %s", err, got, sourceFP)
|
||||
}
|
||||
|
||||
v6 := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.8",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
BirdBgpSourceIPv6: "2001:db8::40",
|
||||
})
|
||||
if _, err := m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &v6}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v6FP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Source v6 changes local v6. When no peer policy overrides it, the rendered
|
||||
// peers fragment can stay the same. A difference means the overlay emitted it;
|
||||
// equality means a rebuild would not change the bundle.
|
||||
_ = v6FP
|
||||
|
||||
if _, err := m.CreatePeer(tenant, &store.BGPPeer{
|
||||
Name: "cache-extra",
|
||||
Neighbor: "203.0.113.50",
|
||||
RemoteASN: 64512,
|
||||
Enabled: true,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
withPeer, err := nodeBundleOverlayFingerprint(m, tenant, speaker, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if withPeer == v6FP {
|
||||
t.Fatal("added peer did not change the overlay fingerprint")
|
||||
}
|
||||
|
||||
sp2, err := m.CreateSpeaker(tenant, &store.Speaker{
|
||||
Role: "replica",
|
||||
Endpoint: "https://203.0.113.2",
|
||||
MetaJSON: v6,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
other, err := nodeBundleOverlayFingerprint(m, tenant, sp2.ID, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if other == withPeer {
|
||||
t.Fatal("speakers with the same addresses shared an overlay fingerprint")
|
||||
}
|
||||
|
||||
otherRev, err := m.CreateRollbackRevision(tenant, rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revFP, err := nodeBundleOverlayFingerprint(m, tenant, speaker, otherRev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if revFP == withPeer {
|
||||
t.Fatal("revision id did not change the overlay fingerprint")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNodeBundleCache(t *testing.T) {
|
||||
srv, ts, ids := startNodeBundleServer(t)
|
||||
if srv.bundleCache.maxN != nodeBundleCacheMaxEntries || srv.bundleCache.maxBytes != nodeBundleCacheMaxBytes {
|
||||
t.Fatalf("caps n=%d bytes=%d", srv.bundleCache.maxN, srv.bundleCache.maxBytes)
|
||||
}
|
||||
var builds atomic.Int32
|
||||
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||
builds.Add(1)
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||
}
|
||||
client := ts.Client()
|
||||
|
||||
status, raw := pullNodeBundle(t, client, ts.URL, ids.speaker, "00000000-0000-0000-0000-000000000099")
|
||||
if status != http.StatusNotFound {
|
||||
t.Fatalf("missing revision status %d: %s", status, raw)
|
||||
}
|
||||
if builds.Load() != 0 {
|
||||
t.Fatal("missing revision built a bundle")
|
||||
}
|
||||
|
||||
body1 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
body2 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("second pull rebuilt: builds=%d", builds.Load())
|
||||
}
|
||||
if !bytes.Equal(body1, body2) {
|
||||
t.Fatal("cache hit was not byte-for-byte identical")
|
||||
}
|
||||
verifyNodeBundle(t, srv, body2)
|
||||
|
||||
sp2, err := srv.Store().CreateSpeaker(ids.tenant, &store.Speaker{
|
||||
Role: "replica",
|
||||
Endpoint: "https://203.0.113.2",
|
||||
MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "203.0.113.2"}),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodyB := mustPullNodeBundle(t, client, ts.URL, sp2.ID, ids.rev)
|
||||
if builds.Load() != 2 {
|
||||
t.Fatalf("second speaker builds=%d", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body1, bodyB) {
|
||||
t.Fatal("different speakers returned the same bundle")
|
||||
}
|
||||
if !strings.Contains(bundleFile(t, bodyB, "bird.conf"), "203.0.113.2") {
|
||||
t.Fatal("speaker B bundle missing its router id")
|
||||
}
|
||||
body3 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 2 || !bytes.Equal(body1, body3) {
|
||||
t.Fatalf("speaker A collided with B: builds=%d equal=%v", builds.Load(), bytes.Equal(body1, body3))
|
||||
}
|
||||
|
||||
meta := store.SpeakerMetaJSON(store.SpeakerMeta{NodeIPv4: "198.51.100.9"})
|
||||
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body4 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 3 {
|
||||
t.Fatalf("changed node ipv4 builds=%d, want 3", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body1, body4) {
|
||||
t.Fatal("changed overlay input reused the previous bundle")
|
||||
}
|
||||
conf := bundleFile(t, body4, "bird.conf")
|
||||
if !strings.Contains(conf, "198.51.100.9") {
|
||||
t.Fatalf("rebuilt bird.conf missing node ipv4: %s", conf)
|
||||
}
|
||||
verifyNodeBundle(t, srv, body4)
|
||||
body5 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 3 || !bytes.Equal(body4, body5) {
|
||||
t.Fatal("rebuilt bundle was not cached")
|
||||
}
|
||||
|
||||
source := store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
NodeIPv4: "198.51.100.9",
|
||||
BirdBgpSourceIPv4: "203.0.113.40",
|
||||
})
|
||||
if _, err := srv.Store().UpdateSpeaker(ids.tenant, ids.speaker, &store.SpeakerPatch{MetaJSON: &source}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body6 := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 4 {
|
||||
t.Fatalf("changed source ipv4 builds=%d, want 4", builds.Load())
|
||||
}
|
||||
conf = bundleFile(t, body6, "bird.conf")
|
||||
if !strings.Contains(conf, "203.0.113.40") {
|
||||
t.Fatalf("source ipv4 not applied: %s", conf)
|
||||
}
|
||||
verifyNodeBundle(t, srv, body6)
|
||||
|
||||
otherRev, err := srv.Store().CreateRollbackRevision(ids.tenant, ids.rev)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bodyRev := mustPullNodeBundle(t, client, ts.URL, ids.speaker, otherRev)
|
||||
if builds.Load() != 5 {
|
||||
t.Fatalf("other revision builds=%d", builds.Load())
|
||||
}
|
||||
if bytes.Equal(body6, bodyRev) {
|
||||
t.Fatal("revisions shared a bundle body")
|
||||
}
|
||||
cached := mustPullNodeBundle(t, client, ts.URL, ids.speaker, ids.rev)
|
||||
if builds.Load() != 5 || !bytes.Equal(body6, cached) {
|
||||
t.Fatal("original revision was evicted or rebuilt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleNodeBundleSingleflight(t *testing.T) {
|
||||
srv, ts, ids := startNodeBundleServer(t)
|
||||
var builds atomic.Int32
|
||||
release := make(chan struct{})
|
||||
srv.nodeBundleBuild = func(revisionID, speakerID string, fragments map[string]string, priv ed25519.PrivateKey) ([]byte, error) {
|
||||
builds.Add(1)
|
||||
<-release
|
||||
return bundle.BuildGzippedTar(revisionID, speakerID, fragments, priv)
|
||||
}
|
||||
client := ts.Client()
|
||||
const n = 2
|
||||
bodies := make([][]byte, n)
|
||||
errs := make([]error, n)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(n)
|
||||
for i := 0; i < n; i++ {
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
bodies[i], errs[i] = pullNodeBundleOK(client, ts.URL, ids.speaker, ids.rev)
|
||||
}(i)
|
||||
}
|
||||
deadline := time.Now().Add(3 * time.Second)
|
||||
joined := 0
|
||||
for {
|
||||
if builds.Load() > 1 {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("concurrent pulls built %d times", builds.Load())
|
||||
}
|
||||
srv.bundleCache.mu.Lock()
|
||||
joined = 0
|
||||
for _, f := range srv.bundleCache.flights {
|
||||
joined += f.joined
|
||||
}
|
||||
srv.bundleCache.mu.Unlock()
|
||||
if builds.Load() == 1 && joined >= 1 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
close(release)
|
||||
wg.Wait()
|
||||
t.Fatalf("timed out builds=%d joined=%d", builds.Load(), joined)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
close(release)
|
||||
wg.Wait()
|
||||
if builds.Load() != 1 {
|
||||
t.Fatalf("builds = %d, want 1", builds.Load())
|
||||
}
|
||||
for i := 0; i < n; i++ {
|
||||
if errs[i] != nil {
|
||||
t.Fatal(errs[i])
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(bodies[0], bodies[1]) {
|
||||
t.Fatal("singleflight returned different bodies")
|
||||
}
|
||||
verifyNodeBundle(t, srv, bodies[0])
|
||||
}
|
||||
|
||||
type nodeBundleIDs struct {
|
||||
tenant, rev, speaker string
|
||||
}
|
||||
|
||||
func startNodeBundleServer(t *testing.T) (*Server, *httptest.Server, nodeBundleIDs) {
|
||||
t.Helper()
|
||||
srv, err := New(Options{
|
||||
InsecureDev: true,
|
||||
SeedDemo: true,
|
||||
BundleSeedHex: testBundleSeed,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(srv.Close)
|
||||
tenant, _, _, rev, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "nodekey|"+tenant+"|node")
|
||||
ts := httptest.NewServer(srv.Handler())
|
||||
t.Cleanup(ts.Close)
|
||||
return srv, ts, nodeBundleIDs{tenant: tenant, rev: rev, speaker: speaker}
|
||||
}
|
||||
|
||||
func mustPullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) []byte {
|
||||
t.Helper()
|
||||
status, body := pullNodeBundle(t, client, base, speaker, rev)
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("status %d: %s", status, body)
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func pullNodeBundle(t *testing.T, client *http.Client, base, speaker, rev string) (int, []byte) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer nodekey")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resp.StatusCode, body
|
||||
}
|
||||
|
||||
func pullNodeBundleOK(client *http.Client, base, speaker, rev string) ([]byte, error) {
|
||||
req, err := http.NewRequest(http.MethodGet, base+"/v1/speakers/"+speaker+"/bundle/"+rev, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer nodekey")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, errors.New(resp.Status + ": " + string(body))
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
func verifyNodeBundle(t *testing.T, srv *Server, raw []byte) {
|
||||
t.Helper()
|
||||
pub, err := base64.StdEncoding.DecodeString(srv.BundlePublicKeyBase64())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := signing.VerifyGzippedTar(raw, ed25519.PublicKey(pub)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func bundleFile(t *testing.T, gz []byte, name string) string {
|
||||
t.Helper()
|
||||
gr, err := gzip.NewReader(bytes.NewReader(gz))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = gr.Close() }()
|
||||
tr := tar.NewReader(gr)
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if errors.Is(err, io.EOF) {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if hdr.Name != name {
|
||||
continue
|
||||
}
|
||||
var b strings.Builder
|
||||
if _, err := io.Copy(&b, tr); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
t.Fatalf("archive missing %s", name)
|
||||
return ""
|
||||
}
|
||||
|
||||
func assertBundleCacheAccounting(t *testing.T, c *nodeBundleCache) {
|
||||
t.Helper()
|
||||
if len(c.items) != len(c.lru) {
|
||||
t.Fatalf("items %d lru %d", len(c.items), len(c.lru))
|
||||
}
|
||||
sum := 0
|
||||
seen := make(map[string]bool, len(c.lru))
|
||||
for _, k := range c.lru {
|
||||
e, ok := c.items[k]
|
||||
if !ok {
|
||||
t.Fatalf("lru key %q missing from items", k)
|
||||
}
|
||||
if seen[k] {
|
||||
t.Fatalf("duplicate lru key %q", k)
|
||||
}
|
||||
seen[k] = true
|
||||
sum += len(e.body)
|
||||
}
|
||||
if sum != c.total {
|
||||
t.Fatalf("total=%d sum=%d", c.total, sum)
|
||||
}
|
||||
if c.maxN > 0 && len(c.items) > c.maxN {
|
||||
t.Fatalf("entries %d over cap %d", len(c.items), c.maxN)
|
||||
}
|
||||
if c.maxBytes > 0 && c.total > c.maxBytes {
|
||||
t.Fatalf("bytes %d over cap %d", c.total, c.maxBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func cacheKeys(c *nodeBundleCache) []string {
|
||||
keys := make([]string, 0, len(c.items))
|
||||
for k := range c.items {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"crypto/ed25519"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
)
|
||||
|
||||
func TestLegacyCompatibilityNodePullAndAuth(t *testing.T) {
|
||||
srv, err := New(Options{SeedDemo: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer srv.Close()
|
||||
tenant, _, _, revision, speaker := srv.Store().DemoIDs()
|
||||
mustSetTestAPIKeys(t, srv, "legacy-node|"+tenant+"|node,operator|"+tenant+"|operator,other-node|other-tenant|node")
|
||||
if err := srv.Store().PublishRevisionForSpeaker(speaker, revision); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := srv.Handler()
|
||||
call := func(path, token string) (int, []byte, string) {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(http.MethodGet, path, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, req)
|
||||
resp := w.Result()
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return resp.StatusCode, body, resp.Header.Get("Content-Type")
|
||||
}
|
||||
latestPath := "/v1/speakers/" + speaker + "/revisions/latest"
|
||||
bundlePath := "/v1/speakers/" + speaker + "/bundle/" + revision
|
||||
for _, path := range []string{latestPath, bundlePath} {
|
||||
for _, tc := range []struct {
|
||||
token string
|
||||
want int
|
||||
}{{"", http.StatusUnauthorized}, {"operator", http.StatusForbidden}, {"other-node", http.StatusForbidden}} {
|
||||
status, _, _ := call(path, tc.token)
|
||||
if status != tc.want {
|
||||
t.Fatalf("%s token %q: status %d, want %d", path, tc.token, status, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
status, raw, _ := call(latestPath, "legacy-node")
|
||||
if status != http.StatusOK {
|
||||
t.Fatalf("legacy latest: status %d: %s", status, raw)
|
||||
}
|
||||
var latest struct {
|
||||
RevisionID string `json:"revision_id"`
|
||||
PublishedAt string `json:"published_at"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &latest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if latest.RevisionID != revision || latest.PublishedAt == "" {
|
||||
t.Fatalf("legacy latest response: %+v", latest)
|
||||
}
|
||||
status, raw, contentType := call("/v1/bundle/signing-public-key", "operator")
|
||||
if status != http.StatusOK || contentType != "application/json" {
|
||||
t.Fatalf("public key status %d, type %q: %s", status, contentType, raw)
|
||||
}
|
||||
var key struct {
|
||||
PublicKeyBase64 string `json:"public_key_base64"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &key); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub, err := base64.StdEncoding.DecodeString(key.PublicKeyBase64)
|
||||
if err != nil || len(pub) != ed25519.PublicKeySize {
|
||||
t.Fatalf("legacy public key: len=%d err=%v", len(pub), err)
|
||||
}
|
||||
status, raw, contentType = call(bundlePath, "legacy-node")
|
||||
if status != http.StatusOK || contentType != "application/gzip" {
|
||||
t.Fatalf("legacy bundle: status %d, type %q: %s", status, contentType, raw)
|
||||
}
|
||||
verified, err := bundle.VerifyGzippedTar(raw, ed25519.PublicKey(pub))
|
||||
if err != nil {
|
||||
t.Fatalf("legacy signed bundle verification: %v", err)
|
||||
}
|
||||
if verified.Manifest.RevisionID != revision || verified.Manifest.SpeakerID != speaker || len(verified.Files) == 0 {
|
||||
t.Fatalf("legacy bundle manifest: %+v", verified.Manifest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type expensiveOperation int
|
||||
|
||||
const (
|
||||
lookupOperation expensiveOperation = iota
|
||||
cdnPreviewOperation
|
||||
revisionPreviewOperation
|
||||
)
|
||||
|
||||
type tenantWork struct {
|
||||
start time.Time
|
||||
counts [3]int
|
||||
active int
|
||||
}
|
||||
|
||||
// operationLimiter is deliberately local to one API process. It never queues
|
||||
// waiters or evicts active tenants to admit new identities.
|
||||
type operationLimiter struct {
|
||||
mu sync.Mutex
|
||||
tenants map[string]*tenantWork
|
||||
active, maxActive, maxTenantActive, maxTenants int
|
||||
rates [3]int
|
||||
nextSweep time.Time
|
||||
}
|
||||
|
||||
func newOperationLimiter() *operationLimiter {
|
||||
return &operationLimiter{
|
||||
tenants: make(map[string]*tenantWork), maxTenants: 4096,
|
||||
maxActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_MAX_CONCURRENT", 4, 64),
|
||||
maxTenantActive: operationEnvLimit("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", 2, 64),
|
||||
rates: [3]int{operationEnvLimit("EVOBGP_API_LOOKUP_PER_MINUTE", 120, 100000), operationEnvLimit("EVOBGP_API_CDN_PREVIEW_PER_MINUTE", 30, 100000), operationEnvLimit("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", 30, 100000)},
|
||||
}
|
||||
}
|
||||
|
||||
func operationEnvLimit(name string, fallback, maximum int) int {
|
||||
n, err := strconv.Atoi(os.Getenv(name))
|
||||
if err != nil || n < 1 || n > maximum {
|
||||
return fallback
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func (l *operationLimiter) acquire(tenant string, op expensiveOperation, now time.Time) (func(), int) {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if !now.Before(l.nextSweep) {
|
||||
for key, entry := range l.tenants {
|
||||
if entry.active == 0 && !now.Before(entry.start.Add(time.Minute)) {
|
||||
delete(l.tenants, key)
|
||||
}
|
||||
}
|
||||
l.nextSweep = now.Add(time.Second)
|
||||
}
|
||||
if l.active >= l.maxActive {
|
||||
return nil, 1
|
||||
}
|
||||
entry := l.tenants[tenant]
|
||||
if entry == nil {
|
||||
if len(l.tenants) >= l.maxTenants {
|
||||
return nil, 1
|
||||
}
|
||||
entry = &tenantWork{start: now}
|
||||
l.tenants[tenant] = entry
|
||||
}
|
||||
if !now.Before(entry.start.Add(time.Minute)) {
|
||||
entry.start = now
|
||||
entry.counts = [3]int{}
|
||||
}
|
||||
if entry.active >= l.maxTenantActive {
|
||||
return nil, 1
|
||||
}
|
||||
if entry.counts[op] >= l.rates[op] {
|
||||
wait := entry.start.Add(time.Minute).Sub(now)
|
||||
return nil, max(1, int((wait+time.Second-1)/time.Second))
|
||||
}
|
||||
entry.counts[op]++
|
||||
entry.active++
|
||||
l.active++
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { l.mu.Lock(); defer l.mu.Unlock(); entry.active--; l.active-- }) }, 0
|
||||
}
|
||||
|
||||
func (s *Server) acquireOperation(w http.ResponseWriter, r *http.Request, tenant string, op expensiveOperation) (func(), bool) {
|
||||
if r.Context().Err() != nil {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request cancelled")
|
||||
return nil, false
|
||||
}
|
||||
s.operationOnce.Do(func() { s.operationLimiter = newOperationLimiter() })
|
||||
release, retry := s.operationLimiter.acquire(tenant, op, time.Now())
|
||||
if release == nil {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(retry))
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "preview/lookup quota exceeded; retry later")
|
||||
return nil, false
|
||||
}
|
||||
return release, true
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestOperationLimitsSharedCapacityAndWindows(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxActive = 2
|
||||
l.maxTenantActive = 1
|
||||
l.rates = [3]int{1, 2, 2}
|
||||
now := time.Now()
|
||||
a, _ := l.acquire("a", lookupOperation, now)
|
||||
if a == nil {
|
||||
t.Fatal("first denied")
|
||||
}
|
||||
if r, _ := l.acquire("a", cdnPreviewOperation, now); r != nil {
|
||||
t.Fatal("tenant concurrency bypass")
|
||||
}
|
||||
b, _ := l.acquire("b", revisionPreviewOperation, now)
|
||||
if b == nil {
|
||||
t.Fatal("other tenant denied")
|
||||
}
|
||||
if r, _ := l.acquire("c", lookupOperation, now); r != nil {
|
||||
t.Fatal("global concurrency bypass")
|
||||
}
|
||||
a()
|
||||
a()
|
||||
b()
|
||||
if l.active != 0 {
|
||||
t.Fatal("release not idempotent")
|
||||
}
|
||||
if r, retry := l.acquire("a", lookupOperation, now.Add(time.Second)); r != nil || retry != 59 {
|
||||
t.Fatalf("rate bypass/retry admitted=%t seconds=%d", r != nil, retry)
|
||||
}
|
||||
c, _ := l.acquire("a", cdnPreviewOperation, now.Add(time.Second))
|
||||
if c == nil {
|
||||
t.Fatal("independent operation denied")
|
||||
}
|
||||
c()
|
||||
d, _ := l.acquire("a", lookupOperation, now.Add(time.Minute))
|
||||
if d == nil {
|
||||
t.Fatal("expired window not reset")
|
||||
}
|
||||
d()
|
||||
}
|
||||
|
||||
func TestOperationTenantMapBoundWithoutActiveEviction(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxTenants = 1
|
||||
now := time.Now()
|
||||
a, _ := l.acquire("a", lookupOperation, now)
|
||||
if r, _ := l.acquire("b", lookupOperation, now.Add(2*time.Minute)); r != nil {
|
||||
t.Fatal("active entry evicted")
|
||||
}
|
||||
a()
|
||||
b, _ := l.acquire("b", lookupOperation, now.Add(3*time.Minute))
|
||||
if b == nil || len(l.tenants) != 1 {
|
||||
t.Fatal("idle expiry did not free bounded state")
|
||||
}
|
||||
b()
|
||||
}
|
||||
|
||||
func TestOperationConcurrentAdmissionBound(t *testing.T) {
|
||||
l := newOperationLimiter()
|
||||
l.maxActive = 4
|
||||
l.maxTenantActive = 4
|
||||
l.rates = [3]int{100, 100, 100}
|
||||
results := make(chan func(), 32)
|
||||
var wg sync.WaitGroup
|
||||
for range 32 {
|
||||
wg.Add(1)
|
||||
go func() { defer wg.Done(); r, _ := l.acquire("tenant", lookupOperation, time.Now()); results <- r }()
|
||||
}
|
||||
wg.Wait()
|
||||
close(results)
|
||||
accepted := 0
|
||||
for r := range results {
|
||||
if r != nil {
|
||||
accepted++
|
||||
r()
|
||||
}
|
||||
}
|
||||
if accepted != 4 || l.active != 0 {
|
||||
t.Fatalf("accepted=%d active=%d", accepted, l.active)
|
||||
}
|
||||
}
|
||||
|
||||
type waitingRevisionStore struct {
|
||||
store.Backend
|
||||
entered chan struct{}
|
||||
unblock chan struct{}
|
||||
}
|
||||
|
||||
func (s waitingRevisionStore) GetRevision(tenant, id string) (*store.Revision, error) {
|
||||
s.entered <- struct{}{}
|
||||
<-s.unblock
|
||||
return s.Backend.GetRevision(tenant, id)
|
||||
}
|
||||
|
||||
func TestRevisionQuotaHeldUntilWorkReturns(t *testing.T) {
|
||||
t.Setenv("EVOBGP_API_EXPENSIVE_TENANT_CONCURRENT", "1")
|
||||
t.Setenv("EVOBGP_API_REVISION_PREVIEW_PER_MINUTE", "30")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, _ := m.DemoIDs()
|
||||
entered := make(chan struct{}, 2)
|
||||
unblock := make(chan struct{})
|
||||
s := &Server{store: waitingRevisionStore{Backend: m, entered: entered, unblock: unblock}}
|
||||
request := func() *http.Request {
|
||||
r := httptest.NewRequest(http.MethodGet, "/v1/revisions/"+revision+"/preview", nil)
|
||||
r.SetPathValue("revision_id", revision)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||
}
|
||||
ctx, cancel := context.WithCancel(request().Context())
|
||||
defer cancel()
|
||||
first := httptest.NewRecorder()
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); s.handleRevisionPreview(first, request().WithContext(ctx)) }()
|
||||
<-entered
|
||||
cancel() // Backend deliberately ignores cancellation: capacity stays reserved.
|
||||
second := httptest.NewRecorder()
|
||||
s.handleRevisionPreview(second, request())
|
||||
if second.Code != http.StatusTooManyRequests || second.Header().Get("Retry-After") == "" {
|
||||
close(unblock)
|
||||
<-done
|
||||
t.Fatalf("quota: %d %s", second.Code, second.Body.String())
|
||||
}
|
||||
close(unblock)
|
||||
<-done
|
||||
third := httptest.NewRecorder()
|
||||
s.handleRevisionPreview(third, request())
|
||||
if third.Code != http.StatusOK {
|
||||
t.Fatalf("release failed: %d %s", third.Code, third.Body.String())
|
||||
}
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
@@ -44,46 +43,6 @@ func peerDiscoveryJSON(d *store.BGPPeerDiscovery) map[string]any {
|
||||
return m
|
||||
}
|
||||
|
||||
// syncPeerDiscoveriesFromLive upserts pending discoveries from dynamic BGP sessions.
|
||||
func (s *Server) syncPeerDiscoveriesFromLive(tenantID string, views []speakerBGPLive) {
|
||||
if s.store == nil || tenantID == "" {
|
||||
return
|
||||
}
|
||||
knownNeighbors := map[string]struct{}{}
|
||||
for _, p := range s.store.ListPeers(tenantID) {
|
||||
if p == nil {
|
||||
continue
|
||||
}
|
||||
if n, ok := store.NormalizePeerNeighborString(p.Neighbor); ok {
|
||||
knownNeighbors[n] = struct{}{}
|
||||
}
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
for _, v := range views {
|
||||
for _, sess := range v.Sessions {
|
||||
if !birdfmt.IsDynamicDiscoverySession(sess.Name) {
|
||||
continue
|
||||
}
|
||||
neighbor, ok := store.NormalizePeerNeighborString(sess.Neighbor)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if _, known := knownNeighbors[neighbor]; known {
|
||||
continue
|
||||
}
|
||||
_, _ = s.store.UpsertPeerDiscovery(tenantID, &store.PeerDiscoveryUpsert{
|
||||
SpeakerID: v.SpeakerID,
|
||||
NeighborID: strings.TrimSpace(sess.NeighborID),
|
||||
Neighbor: neighbor,
|
||||
RemoteASN: sess.NeighborAS,
|
||||
ProtocolName: sess.Name,
|
||||
SessionState: sess.State,
|
||||
SeenAt: now,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
@@ -97,11 +56,6 @@ func (s *Server) handleListPeerDiscoveries(w http.ResponseWriter, r *http.Reques
|
||||
if status == "" {
|
||||
status = store.PeerDiscoveryPending
|
||||
}
|
||||
// Refresh live discovery when listing pending.
|
||||
if status == store.PeerDiscoveryPending || status == "all" {
|
||||
views := s.collectSpeakerBGPLive(r.Context(), a.TenantID, true)
|
||||
s.syncPeerDiscoveriesFromLive(a.TenantID, views)
|
||||
}
|
||||
listStatus := status
|
||||
if status == "all" {
|
||||
listStatus = ""
|
||||
@@ -138,7 +92,7 @@ func (s *Server) handleApprovePeerDiscovery(w http.ResponseWriter, r *http.Reque
|
||||
writeStoreErr(w, err)
|
||||
return
|
||||
}
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_approve")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_approve")
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.discovery.approve", "Approved discovered peer "+peer.Neighbor, peer.ID, map[string]any{
|
||||
"peer_id": peer.ID,
|
||||
"discovery_id": disc.ID,
|
||||
|
||||
+185
-116
@@ -1,47 +1,98 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
const peerLiveCacheTTL = 15 * time.Second
|
||||
|
||||
type speakerBGPLive struct {
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
SpeakerID string
|
||||
Label string
|
||||
Sessions []birdfmt.BGPSession
|
||||
Error string
|
||||
Polled bool
|
||||
ObservedAt time.Time
|
||||
ReceivedAt time.Time
|
||||
LastSuccessAt time.Time
|
||||
Source string
|
||||
ObservationOverride *liveObservation
|
||||
}
|
||||
|
||||
type peerLiveCacheEntry struct {
|
||||
at time.Time
|
||||
views []speakerBGPLive
|
||||
type liveObservation struct {
|
||||
ObservedAt *string `json:"observed_at"`
|
||||
ReceivedAt *string `json:"received_at"`
|
||||
LastSuccessAt *string `json:"last_success_at"`
|
||||
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||
Source string `json:"source"`
|
||||
Quality string `json:"quality"`
|
||||
ErrorCode string `json:"error_code,omitempty"`
|
||||
}
|
||||
|
||||
var peerLiveCache sync.Map // tenantID -> peerLiveCacheEntry
|
||||
func liveTime(t time.Time) *string {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
v := t.UTC().Format(time.RFC3339Nano)
|
||||
return &v
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observed() bool {
|
||||
return v.Polled || v.Error != "" || len(v.Sessions) > 0
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) successful() bool {
|
||||
return v.observed() && v.Error == ""
|
||||
}
|
||||
|
||||
func (v speakerBGPLive) observation() liveObservation {
|
||||
if v.ObservationOverride != nil {
|
||||
return *v.ObservationOverride
|
||||
}
|
||||
o := liveObservation{ObservedAt: liveTime(v.ObservedAt), ReceivedAt: liveTime(v.ReceivedAt), LastSuccessAt: liveTime(v.LastSuccessAt), Source: v.Source, Quality: "unknown"}
|
||||
if o.Source == "" {
|
||||
o.Source = "none"
|
||||
}
|
||||
switch {
|
||||
case v.successful():
|
||||
o.Quality = "ok"
|
||||
case v.Error != "" && !v.LastSuccessAt.IsZero():
|
||||
o.Quality = "stale"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
case v.Error != "":
|
||||
o.Quality = "unreachable"
|
||||
o.ErrorCode = liveErrorCode(v.Error)
|
||||
default:
|
||||
o.ErrorCode = "not_polled"
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func liveErrorCode(err string) string {
|
||||
if strings.Contains(strings.ToLower(err), "not configured") || strings.Contains(strings.ToLower(err), "missing") {
|
||||
return "not_configured"
|
||||
}
|
||||
return "poll_failed"
|
||||
}
|
||||
|
||||
type peerSessionOnSpeaker struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
State string `json:"state"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
type liveSpeakerPoll struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Label string `json:"label"`
|
||||
OK bool `json:"ok"`
|
||||
SessionCount *int `json:"session_count"`
|
||||
PollError string `json:"poll_error,omitempty"`
|
||||
Observation liveObservation `json:"observation"`
|
||||
}
|
||||
|
||||
func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
@@ -65,95 +116,21 @@ func speakerDisplayLabel(sp *store.Speaker) string {
|
||||
return sp.ID
|
||||
}
|
||||
|
||||
func masterSpeakerID(speakers []*store.Speaker) string {
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
return sp.ID
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (s *Server) collectSpeakerBGPLive(ctx context.Context, tenantID string, fresh bool) []speakerBGPLive {
|
||||
if !fresh {
|
||||
if v, ok := peerLiveCache.Load(tenantID); ok {
|
||||
ent := v.(peerLiveCacheEntry)
|
||||
if time.Since(ent.at) < peerLiveCacheTTL {
|
||||
return ent.views
|
||||
}
|
||||
}
|
||||
}
|
||||
speakers := s.store.ListSpeakersForTenant(tenantID)
|
||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||
|
||||
if sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET")); sock != "" {
|
||||
v := speakerBGPLive{Label: "CP (local BIRD)"}
|
||||
if mid := masterSpeakerID(speakers); mid != "" {
|
||||
v.SpeakerID = mid
|
||||
for _, sp := range speakers {
|
||||
if sp != nil && sp.ID == mid {
|
||||
v.Label = speakerDisplayLabel(sp)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
||||
if err != nil {
|
||||
v.Error = err.Error()
|
||||
} else {
|
||||
v.Sessions = birdfmt.ParseBGPSessions(out)
|
||||
}
|
||||
views = append(views, v)
|
||||
}
|
||||
|
||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
||||
type resWrap struct {
|
||||
sp *store.Speaker
|
||||
res nodedispatch.BirdProtocolsResult
|
||||
}
|
||||
ch := make(chan resWrap, len(speakers))
|
||||
var wg sync.WaitGroup
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(speaker *store.Speaker) {
|
||||
defer wg.Done()
|
||||
ch <- resWrap{
|
||||
sp: speaker,
|
||||
res: nodedispatch.FetchBirdProtocols(ctx, speaker, opts),
|
||||
}
|
||||
}(sp)
|
||||
}
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
for rw := range ch {
|
||||
views = append(views, speakerBGPLive{
|
||||
SpeakerID: rw.sp.ID,
|
||||
Label: speakerDisplayLabel(rw.sp),
|
||||
Sessions: rw.res.Sessions,
|
||||
Error: rw.res.Error,
|
||||
})
|
||||
}
|
||||
|
||||
peerLiveCache.Store(tenantID, peerLiveCacheEntry{at: time.Now(), views: views})
|
||||
return views
|
||||
}
|
||||
|
||||
func liveSpeakerPollJSON(views []speakerBGPLive) []liveSpeakerPoll {
|
||||
out := make([]liveSpeakerPoll, 0, len(views))
|
||||
for _, v := range views {
|
||||
var sessionCount *int
|
||||
if v.successful() {
|
||||
count := len(v.Sessions)
|
||||
sessionCount = &count
|
||||
}
|
||||
out = append(out, liveSpeakerPoll{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
OK: v.Error == "",
|
||||
SessionCount: len(v.Sessions),
|
||||
OK: v.successful(),
|
||||
SessionCount: sessionCount,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
}
|
||||
return out
|
||||
@@ -183,27 +160,30 @@ func matchPeerOnSpeakers(peer *store.BGPPeer, views []speakerBGPLive) (
|
||||
protoName := birdfmt.PeerProtocolName(peer.ID)
|
||||
|
||||
for _, v := range views {
|
||||
if v.Error != "" && len(v.Sessions) == 0 {
|
||||
if !v.successful() {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
PollError: v.Error,
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
sess := findPeerSession(v.Sessions, protoName, neighbor, hasNeighbor)
|
||||
if sess == nil {
|
||||
on = append(on, peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: "absent",
|
||||
Observation: v.observation(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
hit := peerSessionOnSpeaker{
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
SpeakerID: v.SpeakerID,
|
||||
Label: v.Label,
|
||||
State: sess.State,
|
||||
Observation: v.observation(),
|
||||
}
|
||||
on = append(on, hit)
|
||||
if strings.EqualFold(strings.TrimSpace(sess.State), "Established") {
|
||||
@@ -273,20 +253,109 @@ func sessionStateRank(state string) int {
|
||||
}
|
||||
|
||||
func applyPeerLiveFields(row map[string]any, peer *store.BGPPeer, views []speakerBGPLive) {
|
||||
state, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
expected := expectedPeerViews(peer, views)
|
||||
state, _, _, _, _, _ := matchPeerOnSpeakers(peer, expected)
|
||||
_, connID, connLabel, establishedOn, on, mismatch := matchPeerOnSpeakers(peer, views)
|
||||
row["session_on_speakers"] = on
|
||||
row["established_on_speakers"] = establishedOn
|
||||
row["session_conflict"] = false
|
||||
row["session_mismatch"] = mismatch
|
||||
row["last_known_session_state"] = row["session_state"]
|
||||
row["session_state"] = nil
|
||||
if state != "" {
|
||||
row["session_state"] = state
|
||||
}
|
||||
if state == "" && len(expected) > 0 && allSpeakerPollsSuccessful(expected) {
|
||||
row["session_state"] = "absent"
|
||||
}
|
||||
row["live_observation"] = peerLiveObservation(expected)
|
||||
if connLabel != "" {
|
||||
row["connected_speaker_label"] = connLabel
|
||||
}
|
||||
row["connected_speaker_id"] = peerLiveSpeakerIDOrNull(connID)
|
||||
}
|
||||
|
||||
func expectedPeerViews(peer *store.BGPPeer, views []speakerBGPLive) []speakerBGPLive {
|
||||
if peer == nil || peer.SpeakerID == nil || strings.TrimSpace(*peer.SpeakerID) == "" {
|
||||
return views
|
||||
}
|
||||
want := strings.TrimSpace(*peer.SpeakerID)
|
||||
expected := make([]speakerBGPLive, 0, 1)
|
||||
for _, v := range views {
|
||||
if strings.EqualFold(v.SpeakerID, want) {
|
||||
expected = append(expected, v)
|
||||
}
|
||||
}
|
||||
return expected
|
||||
}
|
||||
|
||||
func allSpeakerPollsSuccessful(views []speakerBGPLive) bool {
|
||||
for _, v := range views {
|
||||
if !v.successful() {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func peerLiveObservation(views []speakerBGPLive) liveObservation {
|
||||
if len(views) == 0 {
|
||||
return speakerBGPLive{}.observation()
|
||||
}
|
||||
result := liveObservation{Source: views[0].Source, Quality: "unknown"}
|
||||
if result.Source == "" {
|
||||
result.Source = "none"
|
||||
}
|
||||
var latestObserved, latestReceived, latestSuccess time.Time
|
||||
allOK, allFailed, anyStale := true, true, false
|
||||
for _, v := range views {
|
||||
o := v.observation()
|
||||
if v.ObservedAt.After(latestObserved) {
|
||||
latestObserved = v.ObservedAt
|
||||
}
|
||||
if v.ReceivedAt.After(latestReceived) {
|
||||
latestReceived = v.ReceivedAt
|
||||
}
|
||||
if v.LastSuccessAt.After(latestSuccess) {
|
||||
latestSuccess = v.LastSuccessAt
|
||||
}
|
||||
allOK = allOK && v.successful()
|
||||
allFailed = allFailed && o.Quality == "unreachable"
|
||||
anyStale = anyStale || o.Quality == "stale"
|
||||
if result.ErrorCode == "" && o.ErrorCode != "" {
|
||||
result.ErrorCode = o.ErrorCode
|
||||
}
|
||||
if o.ExpiresAt != nil {
|
||||
expires, err := time.Parse(time.RFC3339Nano, *o.ExpiresAt)
|
||||
if err == nil {
|
||||
previous := time.Time{}
|
||||
if result.ExpiresAt != nil {
|
||||
previous, _ = time.Parse(time.RFC3339Nano, *result.ExpiresAt)
|
||||
}
|
||||
if previous.IsZero() || expires.Before(previous) {
|
||||
result.ExpiresAt = o.ExpiresAt
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.ObservedAt = liveTime(latestObserved)
|
||||
result.ReceivedAt = liveTime(latestReceived)
|
||||
result.LastSuccessAt = liveTime(latestSuccess)
|
||||
if len(views) > 1 {
|
||||
result.Source = "mixed"
|
||||
}
|
||||
switch {
|
||||
case allOK:
|
||||
result.Quality = "ok"
|
||||
result.ErrorCode = ""
|
||||
case anyStale:
|
||||
result.Quality = "stale"
|
||||
case allFailed:
|
||||
result.Quality = "unreachable"
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func peerLiveSpeakerIDOrNull(id string) any {
|
||||
if strings.TrimSpace(id) == "" {
|
||||
return nil
|
||||
|
||||
@@ -2,11 +2,84 @@ package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestApplyPeerLiveFieldsObservationStates(t *testing.T) {
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2"}
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
cases := []struct {
|
||||
name string
|
||||
views []speakerBGPLive
|
||||
wantState any
|
||||
wantQuality string
|
||||
}{
|
||||
{"no collector", nil, nil, "unknown"},
|
||||
{"successful empty poll", []speakerBGPLive{{Polled: true, Source: "agent_bird_protocols", ObservedAt: now, ReceivedAt: now, LastSuccessAt: now}}, "absent", "ok"},
|
||||
{"first failure", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now}}, nil, "unreachable"},
|
||||
{"failure after success", []speakerBGPLive{{Polled: true, Error: "timeout", Source: "agent_bird_protocols", ObservedAt: now, LastSuccessAt: now.Add(-time.Minute)}}, nil, "stale"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, peer, tc.views)
|
||||
if row["session_state"] != tc.wantState || row["last_known_session_state"] != "Established" {
|
||||
t.Fatalf("session_state=%v last_known=%v", row["session_state"], row["last_known_session_state"])
|
||||
}
|
||||
observation := row["live_observation"].(liveObservation)
|
||||
if observation.Quality != tc.wantQuality {
|
||||
t.Fatalf("observation=%+v", observation)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPeerLiveFieldsExpectedSpeakerCoverage(t *testing.T) {
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
assigned := "speaker-a"
|
||||
peer := &store.BGPPeer{ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee", Neighbor: "198.51.100.2", SpeakerID: &assigned}
|
||||
viewAError := speakerBGPLive{SpeakerID: assigned, Polled: true, Error: "timeout", ObservedAt: now, Source: "agent_bird_protocols"}
|
||||
viewAEmpty := speakerBGPLive{SpeakerID: assigned, Polled: true, ObservedAt: now, LastSuccessAt: now, Source: "agent_bird_protocols"}
|
||||
viewBEmpty := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, ObservedAt: now.Add(time.Minute), LastSuccessAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||
viewBError := speakerBGPLive{SpeakerID: "speaker-b", Polled: true, Error: "timeout", ObservedAt: now.Add(time.Minute), Source: "agent_bird_protocols"}
|
||||
cases := []struct {
|
||||
name string
|
||||
assigned bool
|
||||
views []speakerBGPLive
|
||||
wantState any
|
||||
wantQuality string
|
||||
wantObserved string
|
||||
}{
|
||||
{"assigned timeout unrelated success", true, []speakerBGPLive{viewAError, viewBEmpty}, nil, "unreachable", now.Format(time.RFC3339Nano)},
|
||||
{"assigned success unrelated failure", true, []speakerBGPLive{viewAEmpty, viewBError}, "absent", "ok", now.Format(time.RFC3339Nano)},
|
||||
{"global mixed outcome", false, []speakerBGPLive{viewAEmpty, viewBError}, nil, "unknown", now.Add(time.Minute).Format(time.RFC3339Nano)},
|
||||
{"assigned speaker not observed", true, []speakerBGPLive{viewBEmpty}, nil, "unknown", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
candidate := *peer
|
||||
if !tc.assigned {
|
||||
candidate.SpeakerID = nil
|
||||
}
|
||||
row := map[string]any{"session_state": "Established"}
|
||||
applyPeerLiveFields(row, &candidate, tc.views)
|
||||
obs := row["live_observation"].(liveObservation)
|
||||
if row["session_state"] != tc.wantState || obs.Quality != tc.wantQuality {
|
||||
t.Fatalf("state=%v observation=%+v", row["session_state"], obs)
|
||||
}
|
||||
if tc.wantObserved == "" && obs.ObservedAt != nil {
|
||||
t.Fatalf("unexpected timestamp %v", *obs.ObservedAt)
|
||||
}
|
||||
if tc.wantObserved != "" && (obs.ObservedAt == nil || *obs.ObservedAt != tc.wantObserved) {
|
||||
t.Fatalf("observed_at=%v want=%q", obs.ObservedAt, tc.wantObserved)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchPeerOnSpeakers_establishedOnReplica(t *testing.T) {
|
||||
peer := &store.BGPPeer{
|
||||
ID: "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
|
||||
|
||||
@@ -9,23 +9,33 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// authRateLimiter is a simple per-IP token bucket for public auth-ish endpoints.
|
||||
const maxAuthRateLimitClients = 4096
|
||||
|
||||
type authRateHit struct {
|
||||
start time.Time
|
||||
count int
|
||||
}
|
||||
|
||||
// authRateLimiter keeps a bounded fixed-window counter per client IP.
|
||||
type authRateLimiter struct {
|
||||
mu sync.Mutex
|
||||
hits map[string][]time.Time
|
||||
limit int
|
||||
window time.Duration
|
||||
mu sync.Mutex
|
||||
hits map[string]authRateHit
|
||||
limit int
|
||||
window time.Duration
|
||||
maxClients int
|
||||
nextSweep time.Time
|
||||
}
|
||||
|
||||
func newAuthRateLimiter() *authRateLimiter {
|
||||
limit := 60
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_AUTH_RATE_LIMIT"))); err == nil && n > 0 && n <= 1000 {
|
||||
limit = n
|
||||
}
|
||||
return &authRateLimiter{
|
||||
hits: make(map[string][]time.Time),
|
||||
limit: limit,
|
||||
window: time.Minute,
|
||||
hits: make(map[string]authRateHit),
|
||||
limit: limit,
|
||||
window: time.Minute,
|
||||
maxClients: maxAuthRateLimitClients,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,23 +43,40 @@ func (l *authRateLimiter) allow(ip string) bool {
|
||||
if l == nil {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
return l.allowAt(ip, time.Now())
|
||||
}
|
||||
|
||||
func (l *authRateLimiter) allowAt(ip string, now time.Time) bool {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
cut := now.Add(-l.window)
|
||||
arr := l.hits[ip]
|
||||
kept := arr[:0]
|
||||
for _, t := range arr {
|
||||
if t.After(cut) {
|
||||
kept = append(kept, t)
|
||||
if !l.nextSweep.IsZero() && !now.Before(l.nextSweep) {
|
||||
l.nextSweep = time.Time{}
|
||||
for key, hit := range l.hits {
|
||||
expiry := hit.start.Add(l.window)
|
||||
if !now.Before(expiry) {
|
||||
delete(l.hits, key)
|
||||
} else if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||
l.nextSweep = expiry
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(kept) >= l.limit {
|
||||
l.hits[ip] = kept
|
||||
hit, exists := l.hits[ip]
|
||||
if !exists && len(l.hits) >= l.maxClients {
|
||||
// Never evict an active counter: a new identity must wait for expiry.
|
||||
return false
|
||||
}
|
||||
kept = append(kept, now)
|
||||
l.hits[ip] = kept
|
||||
if exists && hit.count >= l.limit {
|
||||
return false
|
||||
}
|
||||
if !exists {
|
||||
hit.start = now
|
||||
expiry := now.Add(l.window)
|
||||
if l.nextSweep.IsZero() || expiry.Before(l.nextSweep) {
|
||||
l.nextSweep = expiry
|
||||
}
|
||||
}
|
||||
hit.count++
|
||||
l.hits[ip] = hit
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const requestBodyLimit int64 = 8 << 20
|
||||
|
||||
// withRequestBodyLimit must run inside authMiddleware. Reading the complete
|
||||
// bounded body before dispatch prevents a valid JSON prefix from committing a
|
||||
// mutation before a later size/read error is discovered. Content-Type is not a
|
||||
// security boundary; CSV retains its additional row/field checks in its handler.
|
||||
func withRequestBodyLimit(next http.Handler) http.Handler {
|
||||
return withBodyLimitAndConcurrency(requestBodyLimit, requestBodyConcurrency(), next)
|
||||
}
|
||||
|
||||
func withBodyLimit(limit int64, next http.Handler) http.Handler {
|
||||
return withBodyLimitAndConcurrency(limit, 4, next)
|
||||
}
|
||||
|
||||
func requestBodyConcurrency() int {
|
||||
n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_API_BODY_MAX_CONCURRENT")))
|
||||
if err == nil && n >= 1 && n <= 64 {
|
||||
return n
|
||||
}
|
||||
return 4
|
||||
}
|
||||
|
||||
func withBodyLimitAndConcurrency(limit int64, concurrency int, next http.Handler) http.Handler {
|
||||
slots := make(chan struct{}, concurrency)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
original := r.Body
|
||||
if original != nil {
|
||||
defer func() { _ = original.Close() }()
|
||||
}
|
||||
if r.ContentLength > limit {
|
||||
writeRequestBodyTooLarge(w)
|
||||
return
|
||||
}
|
||||
if r.Context().Err() != nil {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||
return
|
||||
}
|
||||
if r.Body == nil || r.Body == http.NoBody {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
defer func() { <-slots }()
|
||||
default:
|
||||
w.Header().Set("Retry-After", "1")
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "request body processing capacity is full")
|
||||
return
|
||||
}
|
||||
bounded := http.MaxBytesReader(w, r.Body, limit)
|
||||
data, err := io.ReadAll(requestContextReader{ctx: r.Context(), reader: bounded})
|
||||
var tooLarge *http.MaxBytesError
|
||||
if errors.As(err, &tooLarge) {
|
||||
writeRequestBodyTooLarge(w)
|
||||
return
|
||||
}
|
||||
var timeout net.Error
|
||||
if r.Context().Err() != nil || errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &timeout) && timeout.Timeout()) {
|
||||
writeProblem(w, http.StatusRequestTimeout, "Request Timeout", "request body read cancelled")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "could not read request body")
|
||||
return
|
||||
}
|
||||
r.Body = io.NopCloser(bytes.NewReader(data))
|
||||
r.ContentLength = int64(len(data))
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func writeRequestBodyTooLarge(w http.ResponseWriter) {
|
||||
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "request body exceeds the 8 MiB limit")
|
||||
}
|
||||
|
||||
type requestContextReader struct {
|
||||
ctx context.Context
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (r requestContextReader) Read(p []byte) (int, error) {
|
||||
if err := r.ctx.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.reader.Read(p)
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRequestBodyKnownAndUnknownBoundsBeforeMutation(t *testing.T) {
|
||||
const limit = 32
|
||||
for _, unknown := range []bool{false, true} {
|
||||
for _, contentType := range []string{"", "application/json", "text/plain"} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
label := "known"
|
||||
if unknown {
|
||||
label = "unknown"
|
||||
}
|
||||
t.Run(label+"/"+contentType+"/"+string(rune('0'+extra)), func(t *testing.T) {
|
||||
mutations := 0
|
||||
h := withBodyLimit(limit, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var input map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&input); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
mutations++
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
body := `{"x":1}` + strings.Repeat(" ", limit-len(`{"x":1}`)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
if unknown {
|
||||
req.ContentLength = -1
|
||||
}
|
||||
if contentType != "" {
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 0 {
|
||||
if w.Code != http.StatusCreated || mutations != 1 {
|
||||
t.Fatalf("exact limit rejected: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
} else if w.Code != http.StatusRequestEntityTooLarge || mutations != 0 {
|
||||
t.Fatalf("oversized tail mutated: %d mutations=%d", w.Code, mutations)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type checkedRequestBody struct {
|
||||
reads int
|
||||
closed bool
|
||||
reader io.Reader
|
||||
}
|
||||
|
||||
func (b *checkedRequestBody) Read(p []byte) (int, error) { b.reads++; return b.reader.Read(p) }
|
||||
func (b *checkedRequestBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
type failingRequestReader struct{}
|
||||
|
||||
func (failingRequestReader) Read([]byte) (int, error) {
|
||||
return 0, errors.New("untrusted detailed read failure")
|
||||
}
|
||||
|
||||
func TestRequestBodyErrorsDoNotMutate(t *testing.T) {
|
||||
for _, cancelled := range []bool{false, true} {
|
||||
calls := 0
|
||||
b := &checkedRequestBody{reader: io.MultiReader(strings.NewReader(`{"x":1}`), failingRequestReader{})}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = -1
|
||||
if cancelled {
|
||||
ctx, cancel := context.WithCancel(req.Context())
|
||||
cancel()
|
||||
req = req.WithContext(ctx)
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { calls++ })).ServeHTTP(w, req)
|
||||
want := http.StatusBadRequest
|
||||
if cancelled {
|
||||
want = http.StatusRequestTimeout
|
||||
}
|
||||
if w.Code != want || calls != 0 || !b.closed || strings.Contains(w.Body.String(), "untrusted detailed") {
|
||||
t.Fatalf("error handling status=%d calls=%d body=%s", w.Code, calls, w.Body.String())
|
||||
}
|
||||
if cancelled && b.reads != 0 {
|
||||
t.Fatal("read body after cancellation")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAuthenticationRunsFirst(t *testing.T) {
|
||||
b := &checkedRequestBody{reader: failingRequestReader{}}
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", nil)
|
||||
req.Body = b
|
||||
req.ContentLength = requestBodyLimit + 1
|
||||
s := &Server{}
|
||||
called := false
|
||||
h := s.authMiddleware(withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })))
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusUnauthorized || b.reads != 0 || called {
|
||||
t.Fatalf("body processed before auth: %d reads=%d", w.Code, b.reads)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAppliesToGETAndIgnoresClaimedZeroLength(t *testing.T) {
|
||||
for _, length := range []int64{-1, 0, requestBodyLimit + 1} {
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/jobs", strings.NewReader(strings.Repeat("x", int(requestBodyLimit)+1)))
|
||||
req.ContentLength = length
|
||||
called := false
|
||||
w := httptest.NewRecorder()
|
||||
withRequestBodyLimit(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { called = true })).ServeHTTP(w, req)
|
||||
if w.Code != http.StatusRequestEntityTooLarge || called {
|
||||
t.Fatalf("GET size bypass length=%d code=%d", length, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyHandlerWiringAndNoMutation(t *testing.T) {
|
||||
s, err := New(Options{SeedDemo: true, InsecureDev: true, BundleSeedHex: testBundleSeed})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
tenant, _, _, _, _ := s.store.DemoIDs()
|
||||
before := len(s.store.ListModules(tenant))
|
||||
h := s.Handler()
|
||||
prefix := `{"type":"IP_RANGES","name":"bounded-create","enabled":false}`
|
||||
for _, extra := range []int{1, 0} {
|
||||
body := prefix + strings.Repeat(" ", int(requestBodyLimit)-len(prefix)+extra)
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules", strings.NewReader(body))
|
||||
req.ContentLength = -1
|
||||
req.Header.Set("Authorization", "Bearer dev")
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if extra == 1 {
|
||||
if w.Code != http.StatusRequestEntityTooLarge || len(s.store.ListModules(tenant)) != before {
|
||||
t.Fatalf("oversized JSON committed: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
} else if w.Code != http.StatusCreated || len(s.store.ListModules(tenant)) != before+1 {
|
||||
t.Fatalf("8MiB body rejected: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyLeavesStreamingResponseUnbuffered(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1/runtime-logs/stream", nil)
|
||||
withRequestBodyLimit(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f, ok := w.(http.Flusher)
|
||||
if !ok {
|
||||
t.Fatal("response lost Flusher")
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
_, _ = io.WriteString(w, "data: ready\n\n")
|
||||
f.Flush()
|
||||
})).ServeHTTP(w, req)
|
||||
if !w.Flushed || w.Body.String() != "data: ready\n\n" {
|
||||
t.Fatal("stream response buffered")
|
||||
}
|
||||
}
|
||||
|
||||
type blockedAdmissionBody struct {
|
||||
entered chan struct{}
|
||||
resume chan struct{}
|
||||
sent bool
|
||||
}
|
||||
|
||||
func (b *blockedAdmissionBody) Read(p []byte) (int, error) {
|
||||
if b.sent {
|
||||
return 0, io.EOF
|
||||
}
|
||||
b.sent = true
|
||||
close(b.entered)
|
||||
<-b.resume
|
||||
p[0] = 'x'
|
||||
return 1, nil
|
||||
}
|
||||
func (*blockedAdmissionBody) Close() error { return nil }
|
||||
|
||||
func TestRequestBodyAdmissionRejectsWithoutWaitingAndHoldsThroughHandler(t *testing.T) {
|
||||
readEntered, readResume := make(chan struct{}), make(chan struct{})
|
||||
handlerEntered, handlerResume := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/empty" {
|
||||
once.Do(func() { close(handlerEntered) })
|
||||
<-handlerResume
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
first := httptest.NewRequest(http.MethodPost, "/first", nil)
|
||||
first.Body = &blockedAdmissionBody{entered: readEntered, resume: readResume}
|
||||
first.ContentLength = 0 // A claimed zero must not bypass body admission.
|
||||
done := make(chan struct{})
|
||||
go func() { defer close(done); h.ServeHTTP(httptest.NewRecorder(), first) }()
|
||||
select {
|
||||
case <-readEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first body did not start")
|
||||
}
|
||||
reject := func() {
|
||||
t.Helper()
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/second", strings.NewReader("x"))
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusTooManyRequests || w.Header().Get("Retry-After") != "1" {
|
||||
t.Fatalf("admission did not reject: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
reject()
|
||||
// An empty legacy GET remains available while all body slots are occupied.
|
||||
empty := httptest.NewRecorder()
|
||||
h.ServeHTTP(empty, httptest.NewRequest(http.MethodGet, "/empty", nil))
|
||||
if empty.Code != http.StatusNoContent {
|
||||
t.Fatal("empty GET consumed body slot")
|
||||
}
|
||||
close(readResume)
|
||||
select {
|
||||
case <-handlerEntered:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("handler did not start")
|
||||
}
|
||||
reject() // The fully read buffer is still retained by the running handler.
|
||||
close(handlerResume)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("first request did not finish")
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, httptest.NewRequest(http.MethodPost, "/third", strings.NewReader("x")))
|
||||
if w.Code != http.StatusNoContent {
|
||||
t.Fatalf("body slot not released: %d", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionReleasesOnReadError(t *testing.T) {
|
||||
h := withBodyLimitAndConcurrency(32, 1, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("read error reached handler") }))
|
||||
for range 2 {
|
||||
req := httptest.NewRequest(http.MethodPost, "/bad", nil)
|
||||
req.Body = io.NopCloser(failingRequestReader{})
|
||||
req.ContentLength = -1
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, req)
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("read failure retained slot: %d", w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequestBodyAdmissionEnvironment(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
raw string
|
||||
want int
|
||||
}{{"", 4}, {"1", 1}, {"64", 64}, {"65", 4}, {"0", 4}, {"-1", 4}, {"invalid", 4}} {
|
||||
t.Run(tc.raw, func(t *testing.T) {
|
||||
t.Setenv("EVOBGP_API_BODY_MAX_CONCURRENT", tc.raw)
|
||||
if got := requestBodyConcurrency(); got != tc.want {
|
||||
t.Fatalf("concurrency=%d want%d", got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+147
-79
@@ -8,14 +8,12 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/pipeline"
|
||||
@@ -39,7 +37,7 @@ func (s *Server) Handler() http.Handler {
|
||||
// Firewall subsystem moved to the standalone EvoFirewall service; see docs/firewall.md.
|
||||
// Registered on the public mux so it wins over the "/v1/" subtree below regardless of auth.
|
||||
s.mux.HandleFunc("/v1/firewall/", s.handleFirewallGone)
|
||||
s.mux.Handle("/v1/", s.authMiddleware(wrappedV1))
|
||||
s.mux.Handle("/v1/", s.authMiddleware(withRequestBodyLimit(wrappedV1)))
|
||||
return s.withCORS(observability.HTTPMiddleware(s.mux))
|
||||
}
|
||||
|
||||
@@ -60,6 +58,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /modules/{module_id}", s.handleGetModule)
|
||||
m.HandleFunc("GET /peers", s.handleListPeers)
|
||||
m.HandleFunc("GET /speakers", s.handleListSpeakers)
|
||||
m.HandleFunc("POST /telemetry/refresh", s.handleTelemetryRefresh)
|
||||
m.HandleFunc("GET /bundle/signing-public-key", s.handleBundleSigningPublicKey)
|
||||
m.HandleFunc("POST /modules/{module_id}/refresh", s.handleModuleRefresh)
|
||||
m.HandleFunc("POST /tenant/refresh", s.handleTenantRefresh)
|
||||
@@ -79,6 +78,7 @@ func (s *Server) registerV1(m *http.ServeMux) {
|
||||
m.HandleFunc("GET /jobs/{job_id}", s.handleGetJob)
|
||||
m.HandleFunc("GET /jobs/{job_id}/report", s.handleGetJobReport)
|
||||
m.HandleFunc("POST /jobs/{job_id}/cancel", s.handleCancelJob)
|
||||
m.HandleFunc("POST /jobs/{job_id}/retry", s.handleRetryJob)
|
||||
m.HandleFunc("GET /speakers/{speaker_id}/revisions/latest", s.handleNodeLatestRevision)
|
||||
m.HandleFunc("GET /speakers/{speaker_id}/bundle/{revision_id}", s.handleNodeBundle)
|
||||
m.HandleFunc("POST /nodes/enroll", s.handleNodeEnroll)
|
||||
@@ -295,12 +295,10 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
||||
allPeers := s.store.ListPeers(a.TenantID)
|
||||
allPeers = store.FilterOwned(allPeers, func(p *store.BGPPeer) string { return p.CreatedByUserID }, a.Kind, a.IsAdmin, a.UserID)
|
||||
page, next, more := store.PaginateOffset(allPeers, r.URL.Query().Get("cursor"), parseListLimit(r))
|
||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
||||
defer cancel()
|
||||
liveViews := s.collectSpeakerBGPLive(ctx, a.TenantID, fresh)
|
||||
if fresh {
|
||||
s.syncPeerDiscoveriesFromLive(a.TenantID, liveViews)
|
||||
liveViews, _, err := s.loadTelemetry(r.Context(), a.TenantID, s.store.ListSpeakersForTenant(a.TenantID))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||
return
|
||||
}
|
||||
items := make([]map[string]any, 0, len(page))
|
||||
for _, p := range page {
|
||||
@@ -316,30 +314,6 @@ func (s *Server) handleListPeers(w http.ResponseWriter, r *http.Request) {
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) liveBGPProtocolStates(r *http.Request) map[string]string {
|
||||
if r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1") {
|
||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
||||
}
|
||||
if cached, ok := observability.CachedBirdProtocolStates(90 * time.Second); ok {
|
||||
return cached
|
||||
}
|
||||
return s.liveBGPProtocolStatesFresh(r.Context())
|
||||
}
|
||||
|
||||
func (s *Server) liveBGPProtocolStatesFresh(ctx context.Context) map[string]string {
|
||||
sock := strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_SOCKET"))
|
||||
if sock == "" {
|
||||
return map[string]string{}
|
||||
}
|
||||
out, err := birdfmt.ShowProtocols(ctx, sock, strings.TrimSpace(os.Getenv("EVOBGP_BIRDC_BIN")))
|
||||
if err != nil {
|
||||
return map[string]string{}
|
||||
}
|
||||
states := birdfmt.ParseBGPProtocolStates(out)
|
||||
observability.SetBirdProtocolStates(states)
|
||||
return states
|
||||
}
|
||||
|
||||
// parseBGPProtocolStates parses `birdc show protocols all` summary rows into protocol_name -> state.
|
||||
func parseBGPProtocolStates(output string) map[string]string {
|
||||
out := make(map[string]string)
|
||||
@@ -403,9 +377,12 @@ func (s *Server) handleListSpeakers(w http.ResponseWriter, r *http.Request) {
|
||||
fresh := r != nil && strings.EqualFold(strings.TrimSpace(r.URL.Query().Get("live")), "1")
|
||||
var liveByID map[string]map[string]any
|
||||
if fresh {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 12*time.Second)
|
||||
defer cancel()
|
||||
liveByID = s.collectSpeakerLiveStatus(ctx, a.TenantID, true, speakers)
|
||||
var err error
|
||||
_, liveByID, err = s.loadTelemetry(r.Context(), a.TenantID, speakers)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry snapshots unavailable")
|
||||
return
|
||||
}
|
||||
}
|
||||
items := make([]map[string]any, 0, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
@@ -448,12 +425,12 @@ func (s *Server) handleModuleRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
idemPtr = &idem
|
||||
}
|
||||
mid := mod.ID
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindModuleRefresh, idemPtr, &mid, map[string]any{
|
||||
"module_id": moduleID,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -480,6 +457,17 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
moduleIDs := body.ModuleIDs
|
||||
sort.Strings(body.ModuleIDs)
|
||||
requestIDs := append([]string{}, body.ModuleIDs...)
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
request := map[string]any{"module_ids": requestIDs, "trigger": "api"}
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindTenantRefresh, IdempotencyKey: idemPtr, Request: request}) {
|
||||
return
|
||||
}
|
||||
if len(moduleIDs) == 0 {
|
||||
now := time.Now().UTC()
|
||||
for _, mod := range s.store.ListModules(a.TenantID) {
|
||||
@@ -488,21 +476,16 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(moduleIDs) == 0 {
|
||||
if len(moduleIDs) == 0 && idem == "" {
|
||||
writeJSON(w, http.StatusNoContent, map[string]any{"message": "no modules due for refresh"})
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueRequestContext(r.Context(), a.TenantID, jobs.KindTenantRefresh, idemPtr, nil, map[string]any{
|
||||
"module_ids": moduleIDs,
|
||||
"trigger": "api",
|
||||
})
|
||||
}, request)
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -510,7 +493,7 @@ func (s *Server) handleTenantRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{
|
||||
"job_id": snap["job_id"],
|
||||
"status": snap["status"],
|
||||
"module_ids": moduleIDs,
|
||||
"module_ids": j.Snapshot()["meta"].(map[string]any)["module_ids"],
|
||||
"modules_due": len(moduleIDs),
|
||||
})
|
||||
}
|
||||
@@ -560,8 +543,8 @@ func revisionJSON(rev *store.Revision) map[string]any {
|
||||
return m
|
||||
}
|
||||
|
||||
// enqueueModuleRefreshIfEnabled queues module_refresh when the module exists and is enabled (best-effort, no HTTP error).
|
||||
func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger string) {
|
||||
// enqueueModuleRefreshIfEnabled reports scheduling failure after a committed CRUD mutation.
|
||||
func (s *Server) enqueueModuleRefreshIfEnabled(w http.ResponseWriter, r *http.Request, tenantID, moduleID, trigger string) {
|
||||
if s.jobs == nil {
|
||||
return
|
||||
}
|
||||
@@ -570,22 +553,35 @@ func (s *Server) enqueueModuleRefreshIfEnabled(tenantID, moduleID, trigger strin
|
||||
return
|
||||
}
|
||||
mid := moduleID
|
||||
key := "module_refresh:" + moduleID
|
||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindModuleRefresh, &key, &mid, map[string]any{
|
||||
// Each mutation is a new logical refresh. Reusing a module-wide key would
|
||||
// incorrectly replay an older terminal job after later configuration changes.
|
||||
_, _, err = s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindModuleRefresh, nil, &mid, map[string]any{
|
||||
"module_id": moduleID,
|
||||
"trigger": trigger,
|
||||
})
|
||||
s.reportAutomaticEnqueue(w, r, trigger, moduleID, err)
|
||||
}
|
||||
|
||||
// enqueuePeerReconcile queues fast peer-only reconcile/render (best-effort, no HTTP error).
|
||||
func (s *Server) enqueuePeerReconcile(tenantID, trigger string) {
|
||||
func (s *Server) enqueuePeerReconcile(w http.ResponseWriter, r *http.Request, tenantID, trigger string) {
|
||||
if s.jobs == nil {
|
||||
return
|
||||
}
|
||||
_, _, _ = s.jobs.Enqueue(tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
||||
_, _, err := s.jobs.EnqueueContext(r.Context(), tenantID, jobs.KindPeerReconcile, nil, nil, map[string]any{
|
||||
"trigger": trigger,
|
||||
"job_title": "Обновление BGP пиров",
|
||||
})
|
||||
s.reportAutomaticEnqueue(w, r, trigger, "", err)
|
||||
}
|
||||
|
||||
func (s *Server) reportAutomaticEnqueue(w http.ResponseWriter, r *http.Request, trigger, target string, err error) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-EvoBGP-Job-Status", "enqueue-failed")
|
||||
if a, ok := authFromContext(r.Context()); ok {
|
||||
s.recordCRUDAudit(r, a, "bgp.job.enqueue_failed", "Configuration saved; automatic job was not queued", target, map[string]any{"trigger": trigger, "error": err.Error()})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) handleGetRevision(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -614,6 +610,11 @@ func (s *Server) handleRevisionPreview(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, revisionPreviewOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
rev, err := s.store.GetRevision(a.TenantID, r.PathValue("revision_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
@@ -706,11 +707,11 @@ func (s *Server) handleRevisionRollback(w http.ResponseWriter, r *http.Request)
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindRevisionRollback, idemPtr, nil, map[string]any{
|
||||
"source_revision_id": revID,
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -751,13 +752,13 @@ func (s *Server) handleApply(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
"revision_id": revID,
|
||||
"strategy": body.Strategy,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -802,13 +803,13 @@ func (s *Server) handleSpeakerApply(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindDeployApply, idemPtr, nil, map[string]any{
|
||||
"revision_id": revID,
|
||||
"speaker_id": spkID,
|
||||
"trigger": "api",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -830,9 +831,9 @@ func (s *Server) handleBirdReload(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindBirdReload, idemPtr, nil, nil)
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -867,7 +868,11 @@ func (s *Server) handleListJobs(w http.ResponseWriter, r *http.Request) {
|
||||
status := r.URL.Query().Get("status")
|
||||
kind := r.URL.Query().Get("kind")
|
||||
cursor := r.URL.Query().Get("cursor")
|
||||
list, next, more := s.jobs.List(a.TenantID, status, kind, cursor, limit)
|
||||
list, next, more, err := s.jobs.ListContext(r.Context(), a.TenantID, status, kind, cursor, limit)
|
||||
if err != nil {
|
||||
writeInternalError(w, "jobs_list", err)
|
||||
return
|
||||
}
|
||||
items := make([]map[string]any, 0, len(list))
|
||||
for _, j := range list {
|
||||
items = append(items, j.Snapshot())
|
||||
@@ -886,9 +891,9 @@ func (s *Server) handleGetJob(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, j.Snapshot())
|
||||
@@ -903,9 +908,9 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:read") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.Get(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.GetContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
snap := j.Snapshot()
|
||||
@@ -929,6 +934,35 @@ func (s *Server) handleGetJobReport(w http.ResponseWriter, r *http.Request) {
|
||||
out["revision_id"] = v
|
||||
}
|
||||
}
|
||||
sourceJobID := ""
|
||||
if source, ok := meta["source_deployment_job_id"].(string); ok && source != "" {
|
||||
// Resolve through the tenant-scoped registry before exposing linked progress.
|
||||
if _, err := s.jobs.GetContext(r.Context(), a.TenantID, source); err != nil {
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
sourceJobID = source
|
||||
out["source_deployment_job_id"] = source
|
||||
}
|
||||
rows, err := s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, j.ID)
|
||||
if err == nil && len(rows) == 0 && sourceJobID != "" {
|
||||
rows, err = s.store.ListDeploymentSpeakers(r.Context(), a.TenantID, sourceJobID)
|
||||
}
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "deployment progress unavailable")
|
||||
return
|
||||
}
|
||||
progress := make([]map[string]any, 0, len(rows))
|
||||
for _, d := range rows {
|
||||
progress = append(progress, map[string]any{
|
||||
"job_id": d.JobID, "speaker_id": d.SpeakerID, "revision_id": d.RevisionID,
|
||||
"generation": d.Generation, "state": d.State, "confirmation": d.Confirmation,
|
||||
"error": d.Error, "dispatch_attempts": d.DispatchAttempts, "retry_round": d.RetryRound,
|
||||
"health_received_at": d.HealthReceivedAt, "health_revision_id": d.HealthRevisionID,
|
||||
"health_source_at": d.HealthSourceAt, "health_error": d.HealthError,
|
||||
})
|
||||
}
|
||||
out["deployment_progress"] = progress
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
@@ -941,9 +975,34 @@ func (s *Server) handleCancelJob(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.RequestCancel(a.TenantID, r.PathValue("job_id"))
|
||||
j, err := s.jobs.RequestCancelContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "job not found")
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||
}
|
||||
|
||||
func (s *Server) handleRetryJob(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
if !s.requirePerm(w, a, "bgp:operations:admin") {
|
||||
return
|
||||
}
|
||||
j, err := s.jobs.RetryContext(r.Context(), a.TenantID, r.PathValue("job_id"))
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrJobQueueFull) {
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrJobConflict) {
|
||||
writeProblem(w, http.StatusConflict, "Conflict", "job is not failed")
|
||||
return
|
||||
}
|
||||
writeJobReadError(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, j.Snapshot())
|
||||
@@ -998,21 +1057,30 @@ func (s *Server) handleNodeBundle(w http.ResponseWriter, r *http.Request) {
|
||||
writeProblem(w, http.StatusForbidden, "Forbidden", "speaker not in tenant scope")
|
||||
return
|
||||
}
|
||||
rev, err := s.store.GetRevision(a.TenantID, rid)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
return
|
||||
}
|
||||
frags := rev.PreviewFragments
|
||||
overlaid, err := pipeline.OverlayFragmentsForSpeaker(s.store, a.TenantID, sid, rid, frags)
|
||||
overlayFP, err := nodeBundleOverlayFingerprint(s.store, a.TenantID, sid, rid)
|
||||
if err != nil {
|
||||
writeInternalError(w, "bundle overlay", err)
|
||||
return
|
||||
}
|
||||
frags = overlaid
|
||||
tgz, err := bundle.BuildGzippedTar(rid, sid, frags, s.bundlePriv)
|
||||
// Finished gzip, including manifest and bundle.sig. Revision bytes are immutable;
|
||||
// the overlay fingerprint keeps speaker-specific renders from colliding.
|
||||
tgz, err := s.bundleCache.do(r.Context(), nodeBundleCacheKey(a.TenantID, rid, sid, overlayFP), func(ctx context.Context) ([]byte, bool, error) {
|
||||
return s.loadNodeBundle(ctx, a.TenantID, sid, rid, overlayFP)
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "internal", err)
|
||||
if errors.Is(err, errNodeBundleRevisionMissing) {
|
||||
writeProblem(w, http.StatusNotFound, "Not Found", "revision not found")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return
|
||||
}
|
||||
op := "internal"
|
||||
var opErr *nodeBundleOpError
|
||||
if errors.As(err, &opErr) && opErr.op != "" {
|
||||
op = opErr.op
|
||||
}
|
||||
writeInternalError(w, op, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
|
||||
@@ -3,7 +3,6 @@ package httpapi
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -132,23 +131,7 @@ func (s *Server) pushAuditToPortal(entry *store.AuditEntry) {
|
||||
}
|
||||
|
||||
func clientIP(r *http.Request) string {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
if xff := strings.TrimSpace(r.Header.Get("X-Forwarded-For")); xff != "" {
|
||||
parts := strings.Split(xff, ",")
|
||||
if len(parts) > 0 {
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
}
|
||||
if xrip := strings.TrimSpace(r.Header.Get("X-Real-IP")); xrip != "" {
|
||||
return xrip
|
||||
}
|
||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||
if err != nil {
|
||||
return strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
return host
|
||||
return clientIPWithTrustedProxies(r, trustedProxyPrefixes())
|
||||
}
|
||||
|
||||
// initAuditPusher wires portal push when URL and secret are configured.
|
||||
|
||||
@@ -296,10 +296,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, cdnPreviewOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
mod, err := s.store.GetModule(a.TenantID, r.PathValue("module_id"))
|
||||
if err != nil {
|
||||
writeStoreErr(w, err)
|
||||
@@ -309,6 +310,10 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "module type must be CDN_CIDRS")
|
||||
return
|
||||
}
|
||||
if err := pipeline.ResolveCDNURLHost(r.Context(), u); err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", invalidInputDetail)
|
||||
return
|
||||
}
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "invalid url")
|
||||
@@ -321,11 +326,11 @@ func (s *Server) handlePreviewCDNSource(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||
writeBadGateway(w, "cdn preview fetch", fmt.Errorf("upstream status: %s", resp.Status))
|
||||
return
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
raw, err := pipeline.ReadCDNBody(resp.Body)
|
||||
if err != nil {
|
||||
writeBadGateway(w, "cdn preview read body", err)
|
||||
return
|
||||
@@ -379,7 +384,7 @@ func (s *Server) handlePostCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.create", "Created CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID, "url": x.URL})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_create")
|
||||
writeJSON(w, http.StatusCreated, cdnSourceJSON(x))
|
||||
}
|
||||
|
||||
@@ -410,7 +415,7 @@ func (s *Server) handlePatchCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.update", "Updated CDN source", x.ID, map[string]any{"module_id": mid, "source_id": x.ID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_patch")
|
||||
writeJSON(w, http.StatusOK, cdnSourceJSON(x))
|
||||
}
|
||||
|
||||
@@ -426,7 +431,7 @@ func (s *Server) handleDeleteCDNSource(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.cdn_source.delete", "Deleted CDN source", sourceID, map[string]any{"module_id": mid, "source_id": sourceID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "cdn_source_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "cdn_source_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -485,7 +490,7 @@ func (s *Server) handlePostAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.create", "Created AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_create")
|
||||
writeJSON(w, http.StatusCreated, asEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -506,7 +511,7 @@ func (s *Server) handlePatchAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.update", "Updated AS entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "asn": x.ASN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_patch")
|
||||
writeJSON(w, http.StatusOK, asEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -522,7 +527,7 @@ func (s *Server) handleDeleteAS(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.as_entry.delete", "Deleted AS entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "as_entry_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "as_entry_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -566,7 +571,7 @@ func (s *Server) handlePostDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.create", "Created domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_create")
|
||||
writeJSON(w, http.StatusCreated, domainEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -587,7 +592,7 @@ func (s *Server) handlePatchDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.update", "Updated domain entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "fqdn": x.FQDN})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_patch")
|
||||
writeJSON(w, http.StatusOK, domainEntryJSON(x))
|
||||
}
|
||||
|
||||
@@ -603,7 +608,7 @@ func (s *Server) handleDeleteDomain(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.domain_entry.delete", "Deleted domain entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "domain_entry_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "domain_entry_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -647,7 +652,7 @@ func (s *Server) handlePostIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.create", "Created IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_create")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_create")
|
||||
writeJSON(w, http.StatusCreated, ipRangeJSON(x))
|
||||
}
|
||||
|
||||
@@ -668,7 +673,7 @@ func (s *Server) handlePatchIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.update", "Updated IP range entry", x.ID, map[string]any{"module_id": mid, "entry_id": x.ID, "prefix": x.Prefix})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_patch")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_patch")
|
||||
writeJSON(w, http.StatusOK, ipRangeJSON(x))
|
||||
}
|
||||
|
||||
@@ -684,7 +689,7 @@ func (s *Server) handleDeleteIPRange(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.ip_range.delete", "Deleted IP range entry", entryID, map[string]any{"module_id": mid, "entry_id": entryID})
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, mid, "ip_range_delete")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, mid, "ip_range_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -777,8 +782,12 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
moduleID := r.PathValue("module_id")
|
||||
res, err := importer.ImportModuleEntriesCSV(s.store, a.TenantID, moduleID, r.Body)
|
||||
res, err := importer.ImportModuleEntriesCSV(r.Context(), s.store, a.TenantID, moduleID, r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, importer.ErrCSVTooLarge) {
|
||||
writeProblem(w, http.StatusRequestEntityTooLarge, "Content Too Large", "csv exceeds 8 MiB, 10000 data rows or 64 fields per record; split the import into smaller requests")
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrInvalidInput) {
|
||||
writeProblem(w, http.StatusUnprocessableEntity, "Unprocessable Entity", "csv is empty")
|
||||
return
|
||||
@@ -802,11 +811,11 @@ func (s *Server) handleImportModuleEntriesCSV(w http.ResponseWriter, r *http.Req
|
||||
if res.Imported > 0 {
|
||||
switch res.ModuleType {
|
||||
case "AS_PREFIXES":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "as_entry_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "as_entry_import_csv")
|
||||
case "DOMAINS":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "domain_entry_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "domain_entry_import_csv")
|
||||
case "IP_RANGES":
|
||||
s.enqueueModuleRefreshIfEnabled(a.TenantID, moduleID, "ip_range_import_csv")
|
||||
s.enqueueModuleRefreshIfEnabled(w, r, a.TenantID, moduleID, "ip_range_import_csv")
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{
|
||||
@@ -1055,7 +1064,7 @@ func (s *Server) handlePostPeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.create", "Created BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_create")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_create")
|
||||
writeJSON(w, http.StatusCreated, peerJSON(x))
|
||||
}
|
||||
|
||||
@@ -1099,7 +1108,7 @@ func (s *Server) handlePatchPeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.update", "Updated BGP peer "+x.Name, x.ID, map[string]any{"peer_id": x.ID, "neighbor": x.Neighbor})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_patch")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_patch")
|
||||
writeJSON(w, http.StatusOK, peerJSON(x))
|
||||
}
|
||||
|
||||
@@ -1120,7 +1129,7 @@ func (s *Server) handleDeletePeer(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.peer.delete", "Deleted BGP peer", peerID, map[string]any{"peer_id": peerID})
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_delete")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_delete")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
@@ -1295,7 +1304,7 @@ func (s *Server) handlePatchSettings(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
if peerDiscoverySettingsChanged(body) {
|
||||
s.enqueuePeerReconcile(a.TenantID, "peer_discovery_settings")
|
||||
s.enqueuePeerReconcile(w, r, a.TenantID, "peer_discovery_settings")
|
||||
}
|
||||
s.recordCRUDAudit(r, a, "bgp.settings.update", "Updated tenant settings", a.TenantID, map[string]any{"keys": settingsAuditKeys(body)})
|
||||
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"io"
|
||||
@@ -8,8 +9,60 @@ import (
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/importer"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestCSVImportLimitsBeforeMutations(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "AS_PREFIXES", Name: "csv-limits"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
module := mod.ID
|
||||
s := &Server{store: m}
|
||||
before, err := m.ListASEntries(tenant, module)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
header := "asn,community\n"
|
||||
validPrefix := header + "64512,\n"
|
||||
oversized := validPrefix + strings.Repeat("\n", importer.MaxCSVBytes-len(validPrefix)) + "64513,\n"
|
||||
for _, tc := range []struct {
|
||||
name, body string
|
||||
unknownLength bool
|
||||
status int
|
||||
}{
|
||||
{"exact", header + strings.Repeat("\n", importer.MaxCSVBytes-len(header)), false, http.StatusOK},
|
||||
{"oversize", oversized, false, http.StatusRequestEntityTooLarge},
|
||||
{"unknown_length", oversized, true, http.StatusRequestEntityTooLarge},
|
||||
{"rows", header + strings.Repeat("64512,\n", importer.MaxCSVRows+1), true, http.StatusRequestEntityTooLarge},
|
||||
{"fields", header + "64512," + strings.Repeat(",", importer.MaxCSVFields), false, http.StatusRequestEntityTooLarge},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/v1/modules/"+module+"/entries.csv", strings.NewReader(tc.body))
|
||||
if tc.unknownLength {
|
||||
req.ContentLength = -1
|
||||
req.TransferEncoding = []string{"chunked"}
|
||||
}
|
||||
req.SetPathValue("module_id", module)
|
||||
req = req.WithContext(context.WithValue(req.Context(), authCtxKey, Auth{TenantID: tenant, Role: "operator"}))
|
||||
w := httptest.NewRecorder()
|
||||
s.handleImportModuleEntriesCSV(w, req)
|
||||
if w.Code != tc.status {
|
||||
t.Fatalf("status=%d body=%s", w.Code, w.Body.String())
|
||||
}
|
||||
after, e := m.ListASEntries(tenant, module)
|
||||
if e != nil || len(after) != len(before) {
|
||||
t.Fatalf("mutations on rejected/header-only request: count=%d err=%v", len(after), e)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestModuleEntriesCSVImportExportIPRanges(t *testing.T) {
|
||||
srv, err := New(Options{
|
||||
InsecureDev: true,
|
||||
|
||||
@@ -24,6 +24,11 @@ func (s *Server) handleLookup(w http.ResponseWriter, r *http.Request) {
|
||||
writeProblem(w, http.StatusBadRequest, "Bad Request", "query parameter q is required")
|
||||
return
|
||||
}
|
||||
release, ok := s.acquireOperation(w, r, a.TenantID, lookupOperation)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
res, err := lookup.Lookup(r.Context(), s.store, a.TenantID, q)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrInvalidInput) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -31,7 +32,7 @@ func TestLookupMembershipHTTP(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := srv.Store().SetModulePrefixSnapshot(tenant, modIP, "t", []store.PrefixRow{
|
||||
if err := srv.Store().SetModulePrefixSnapshot(context.Background(), tenant, modIP, "t", []store.PrefixRow{
|
||||
{Prefix: "198.51.100.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -242,10 +242,6 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
kind := "maintenance_policy_run"
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
@@ -255,11 +251,18 @@ func (s *Server) enqueueMaintenancePolicy(w http.ResponseWriter, r *http.Request
|
||||
if dryRun {
|
||||
title = "Maintenance policy dry-run"
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: jobs.KindMaintenancePolicyRun, IdempotencyKey: idemPtr, Meta: map[string]any{"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title}}) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind+":"+policyID) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
"policy_id": policyID, "dry_run": dryRun, "actor_prefix": actorPrefix(a), "job_title": title,
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "maintenance_policy_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -68,19 +69,22 @@ func (s *Server) enqueuePostgresMaint(w http.ResponseWriter, r *http.Request, a
|
||||
if !s.requirePostgres(w) || !s.requireOperatorStrict(w, a) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
idem := strings.TrimSpace(r.Header.Get("Idempotency-Key"))
|
||||
var idemPtr *string
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
meta["actor_prefix"] = actorPrefix(a)
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, kind, idemPtr, nil, meta)
|
||||
if s.replayJobResponse(w, r, store.JobEnqueue{TenantID: a.TenantID, Kind: kind, IdempotencyKey: idemPtr, Meta: meta}) {
|
||||
return
|
||||
}
|
||||
if !s.checkPgMaintRateLimit(a.TenantID, kind) {
|
||||
writeProblem(w, http.StatusTooManyRequests, "Too Many Requests", "wait before repeating this maintenance operation")
|
||||
return
|
||||
}
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, kind, idemPtr, nil, meta)
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
@@ -184,12 +188,12 @@ func (s *Server) handlePostgresCleanup(w http.ResponseWriter, r *http.Request) {
|
||||
if idem != "" {
|
||||
idemPtr = &idem
|
||||
}
|
||||
j, _, err := s.jobs.Enqueue(a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
j, _, err := s.jobs.EnqueueContext(r.Context(), a.TenantID, jobs.KindMaintenancePolicyRun, idemPtr, nil, map[string]any{
|
||||
"policy_id": policyID, "dry_run": body.DryRun, "actor_prefix": actorPrefix(a),
|
||||
"job_title": "PostgreSQL cleanup (deprecated path)",
|
||||
})
|
||||
if err != nil {
|
||||
writeInternalError(w, "postgres_maint_enqueue", err)
|
||||
writeJobEnqueueError(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Location", "/v1/jobs/"+j.ID)
|
||||
|
||||
@@ -8,15 +8,17 @@ import (
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/audit"
|
||||
"evobgp/internal/jobs"
|
||||
"evobgp/internal/logging"
|
||||
"evobgp/internal/maintenance"
|
||||
"evobgp/internal/pgmonitor"
|
||||
"evobgp/internal/repository"
|
||||
"evobgp/internal/runtimelogs"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/internal/telemetry"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
@@ -30,7 +32,11 @@ type Server struct {
|
||||
maintStats *maintenance.DBStatsProvider
|
||||
jobs *jobs.Registry
|
||||
authLimiter *authRateLimiter
|
||||
operationOnce sync.Once
|
||||
operationLimiter *operationLimiter
|
||||
bundlePriv ed25519.PrivateKey
|
||||
bundleCache *nodeBundleCache
|
||||
nodeBundleBuild nodeBundleBuilder
|
||||
keyResolver *apiKeyResolver
|
||||
firewallResolver *firewallTokenResolver
|
||||
bundleSeedHex string
|
||||
@@ -39,6 +45,11 @@ type Server struct {
|
||||
runtimeLogs *runtimelogs.Service
|
||||
runtimeLogsPolicyTenant string
|
||||
mux *http.ServeMux
|
||||
backgroundMu sync.Mutex
|
||||
backgroundStarted bool
|
||||
backgroundClosed bool
|
||||
backgroundCancel context.CancelFunc
|
||||
telemetryCollector *telemetry.Collector
|
||||
|
||||
// Portal / dual-auth (JWT) configuration.
|
||||
jwtSecret string
|
||||
@@ -118,6 +129,7 @@ func New(opts Options) (*Server, error) {
|
||||
maintStats: maintStats,
|
||||
jobs: reg,
|
||||
bundlePriv: priv,
|
||||
bundleCache: newNodeBundleCache(nodeBundleCacheMaxEntries, nodeBundleCacheMaxBytes),
|
||||
keyResolver: resolver,
|
||||
firewallResolver: fwResolver,
|
||||
bundleSeedHex: strings.TrimSpace(opts.BundleSeedHex),
|
||||
@@ -143,6 +155,29 @@ func New(opts Options) (*Server, error) {
|
||||
|
||||
// Close releases database resources.
|
||||
func (s *Server) Close() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.backgroundMu.Lock()
|
||||
if s.backgroundClosed {
|
||||
s.backgroundMu.Unlock()
|
||||
return
|
||||
}
|
||||
s.backgroundClosed = true
|
||||
cancel := s.backgroundCancel
|
||||
collector := s.telemetryCollector
|
||||
s.backgroundMu.Unlock()
|
||||
if cancel != nil {
|
||||
cancel()
|
||||
}
|
||||
if collector != nil {
|
||||
_ = collector.Close(context.Background())
|
||||
}
|
||||
// Do not close the pool while a worker is still persisting its final state.
|
||||
if s.jobs != nil {
|
||||
s.jobs.RequestCancelAll()
|
||||
_ = s.jobs.Drain(context.Background())
|
||||
}
|
||||
if s.pgPool != nil {
|
||||
s.pgPool.Close()
|
||||
}
|
||||
@@ -154,26 +189,44 @@ func (s *Server) Store() store.Backend { return s.store }
|
||||
// Jobs exposes the in-process async job registry (for scheduler / evobgp-all).
|
||||
func (s *Server) Jobs() *jobs.Registry { return s.jobs }
|
||||
|
||||
// StartBackground starts PostgreSQL monitoring and maintenance schedulers until ctx is cancelled.
|
||||
// StartBackground starts shared schedulers and telemetry polling once until ctx is cancelled.
|
||||
func (s *Server) StartBackground(ctx context.Context) {
|
||||
if s != nil && s.pgPool != nil {
|
||||
pgmonitor.StartScheduler(ctx, s.pgPool)
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.backgroundMu.Lock()
|
||||
defer s.backgroundMu.Unlock()
|
||||
if s.backgroundClosed || s.backgroundStarted {
|
||||
return
|
||||
}
|
||||
s.backgroundStarted = true
|
||||
backgroundCtx, cancel := context.WithCancel(ctx)
|
||||
s.backgroundCancel = cancel
|
||||
if s.store != nil {
|
||||
s.telemetryCollector = telemetry.Start(backgroundCtx, s.store, telemetry.ConfigFromEnv())
|
||||
}
|
||||
if s.pgPool != nil {
|
||||
pgmonitor.StartScheduler(backgroundCtx, s.pgPool)
|
||||
if s.jobs != nil {
|
||||
jobs.StartDurableQueueLoop(ctx, s.jobs, repository.NewJobAuditWriter(s.pgPool))
|
||||
jobs.StartDurableQueueLoop(backgroundCtx, s.jobs)
|
||||
jobs.StartDeploymentReconciliationLoop(backgroundCtx, s.jobs, s.store)
|
||||
}
|
||||
}
|
||||
if s != nil && s.maintConfig != nil && s.jobs != nil {
|
||||
maintenance.StartScheduler(ctx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
||||
maintenance.StartScheduler(backgroundCtx, s.maintConfig, func(policyID string, dryRun bool, idem string) {
|
||||
key := idem
|
||||
_, _, _ = s.jobs.Enqueue("", jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
||||
_, _, err := s.jobs.EnqueueContext(backgroundCtx, store.SystemJobTenantID, jobs.KindMaintenancePolicyRun, &key, nil, map[string]any{
|
||||
"policy_id": policyID,
|
||||
"dry_run": dryRun,
|
||||
"trigger": "scheduler",
|
||||
})
|
||||
if err != nil {
|
||||
logging.With("component", "maintenance.scheduler").Error("enqueue failed", "policy_id", policyID, "err", err)
|
||||
}
|
||||
}, 30*time.Second)
|
||||
}
|
||||
if s != nil && s.runtimeLogs != nil && s.store != nil {
|
||||
runtimelogs.StartAutoCleanupScheduler(ctx, runtimelogs.SchedulerDeps{
|
||||
runtimelogs.StartAutoCleanupScheduler(backgroundCtx, runtimelogs.SchedulerDeps{
|
||||
Service: s.runtimeLogs,
|
||||
Store: s.store,
|
||||
PolicyTenant: s.runtimeLogsPolicyTenant,
|
||||
|
||||
@@ -6,7 +6,11 @@ import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -165,24 +169,65 @@ func publicControlPlaneURL(r *http.Request, override string) string {
|
||||
if s := strings.TrimSpace(override); s != "" {
|
||||
return strings.TrimRight(s, "/")
|
||||
}
|
||||
if origin := strings.TrimSpace(r.Header.Get("Origin")); strings.HasPrefix(origin, "http://") || strings.HasPrefix(origin, "https://") {
|
||||
return strings.TrimRight(origin, "/")
|
||||
}
|
||||
proto := strings.TrimSpace(r.Header.Get("X-Forwarded-Proto"))
|
||||
if proto == "" {
|
||||
proto = "https"
|
||||
}
|
||||
host := strings.TrimSpace(r.Header.Get("X-Forwarded-Host"))
|
||||
if i := strings.Index(host, ","); i >= 0 {
|
||||
host = strings.TrimSpace(host[:i])
|
||||
}
|
||||
if host == "" {
|
||||
host = strings.TrimSpace(r.Host)
|
||||
}
|
||||
if host == "" {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return proto + "://" + host
|
||||
if peer, ok := peerAddress(r); ok && isTrustedProxy(peer, trustedProxyPrefixes()) {
|
||||
protos := r.Header.Values("X-Forwarded-Proto")
|
||||
hosts := r.Header.Values("X-Forwarded-Host")
|
||||
if len(protos) == 1 && len(hosts) == 1 {
|
||||
proto := strings.TrimSpace(protos[0])
|
||||
host := strings.TrimSpace(hosts[0])
|
||||
if (proto == "http" || proto == "https") && validControlPlaneHost(host) {
|
||||
return proto + "://" + host
|
||||
}
|
||||
}
|
||||
}
|
||||
if !validControlPlaneHost(r.Host) {
|
||||
return ""
|
||||
}
|
||||
proto := "http"
|
||||
if r.TLS != nil {
|
||||
proto = "https"
|
||||
}
|
||||
return proto + "://" + r.Host
|
||||
}
|
||||
|
||||
func validControlPlaneHost(host string) bool {
|
||||
if host == "" || strings.TrimSpace(host) != host || strings.HasSuffix(host, ":") || strings.ContainsAny(host, " ,/@?#\\\t\r\n") {
|
||||
return false
|
||||
}
|
||||
u, err := url.Parse("http://" + host)
|
||||
if err != nil || u.Host != host || u.User != nil || u.Path != "" || u.RawQuery != "" || u.Fragment != "" {
|
||||
return false
|
||||
}
|
||||
name := u.Hostname()
|
||||
if name == "" {
|
||||
return false
|
||||
}
|
||||
if strings.HasPrefix(host, "[") {
|
||||
if _, err := netip.ParseAddr(name); err != nil {
|
||||
return false
|
||||
}
|
||||
} else if net.ParseIP(name) == nil {
|
||||
for _, label := range strings.Split(name, ".") {
|
||||
if label == "" || label[0] == '-' || label[len(label)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for _, c := range label {
|
||||
if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if port := u.Port(); port != "" {
|
||||
n, err := strconv.Atoi(port)
|
||||
if err != nil || n < 1 || n > 65535 {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) attachReplicaInstall(resp map[string]any, sp *store.Speaker, nodeToken string, req speakerCreateRequest, r *http.Request) {
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
@@ -11,6 +9,34 @@ import (
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type agentHealthPoll struct {
|
||||
result nodedispatch.AgentHealthResult
|
||||
at time.Time
|
||||
lastSuccessAt time.Time
|
||||
observationOverride *liveObservation
|
||||
}
|
||||
|
||||
func agentHealthObservation(poll *agentHealthPoll) liveObservation {
|
||||
if poll == nil {
|
||||
return liveObservation{Source: "none", Quality: "unknown", ErrorCode: "not_polled"}
|
||||
}
|
||||
if poll.observationOverride != nil {
|
||||
return *poll.observationOverride
|
||||
}
|
||||
o := liveObservation{ObservedAt: liveTime(poll.at), ReceivedAt: liveTime(poll.at), LastSuccessAt: liveTime(poll.lastSuccessAt), Source: "agent_health", Quality: "ok"}
|
||||
if !poll.result.OK {
|
||||
o.Quality = "unreachable"
|
||||
if !poll.lastSuccessAt.IsZero() {
|
||||
o.Quality = "stale"
|
||||
}
|
||||
o.ErrorCode = liveErrorCode(poll.result.Error)
|
||||
if poll.result.Error == "agent reported ok=false" {
|
||||
o.ErrorCode = "reported_unhealthy"
|
||||
}
|
||||
}
|
||||
return o
|
||||
}
|
||||
|
||||
func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
||||
total = len(sessions)
|
||||
for _, s := range sessions {
|
||||
@@ -22,29 +48,48 @@ func countBGPSessions(sessions []birdfmt.BGPSession) (total, established int) {
|
||||
}
|
||||
|
||||
func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *nodedispatch.AgentHealthResult) map[string]any {
|
||||
var poll *agentHealthPoll
|
||||
if health != nil {
|
||||
poll = &agentHealthPoll{result: *health}
|
||||
}
|
||||
return speakerLiveStatusWithPoll(sp, view, poll)
|
||||
}
|
||||
|
||||
func speakerLiveStatusWithPoll(sp *store.Speaker, view speakerBGPLive, health *agentHealthPoll) map[string]any {
|
||||
total, established := countBGPSessions(view.Sessions)
|
||||
bgpObservation := view.observation()
|
||||
agentObservation := agentHealthObservation(health)
|
||||
if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") && health == nil {
|
||||
agentObservation = bgpObservation
|
||||
}
|
||||
m := map[string]any{
|
||||
"label": view.Label,
|
||||
"bgp_poll_ok": view.Error == "",
|
||||
"bgp_sessions_total": total,
|
||||
"bgp_established": established,
|
||||
"bgp_poll_ok": view.successful(),
|
||||
"bgp_sessions_total": nil,
|
||||
"bgp_established": nil,
|
||||
"bgp_observation": bgpObservation,
|
||||
"agent_observation": agentObservation,
|
||||
}
|
||||
if view.successful() {
|
||||
m["bgp_sessions_total"] = total
|
||||
m["bgp_established"] = established
|
||||
}
|
||||
if view.Error != "" {
|
||||
m["bgp_poll_error"] = view.Error
|
||||
}
|
||||
if health != nil {
|
||||
m["agent_ok"] = health.OK
|
||||
if health.Error != "" {
|
||||
m["agent_error"] = health.Error
|
||||
m["agent_ok"] = health.result.OK
|
||||
if health.result.Error != "" {
|
||||
m["agent_error"] = health.result.Error
|
||||
}
|
||||
if health.LastSyncAt != "" {
|
||||
m["agent_last_sync_at"] = health.LastSyncAt
|
||||
if health.result.LastSyncAt != "" {
|
||||
m["agent_last_sync_at"] = health.result.LastSyncAt
|
||||
}
|
||||
if health.LastAppliedRevisionID != "" {
|
||||
m["agent_last_applied_revision_id"] = health.LastAppliedRevisionID
|
||||
if health.result.LastAppliedRevisionID != "" {
|
||||
m["agent_last_applied_revision_id"] = health.result.LastAppliedRevisionID
|
||||
}
|
||||
} else if sp != nil && strings.EqualFold(strings.TrimSpace(sp.Role), "master") {
|
||||
m["agent_ok"] = view.Error == ""
|
||||
m["agent_ok"] = view.successful()
|
||||
if view.Error != "" {
|
||||
m["agent_error"] = view.Error
|
||||
}
|
||||
@@ -52,7 +97,7 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
||||
m["agent_ok"] = false
|
||||
m["agent_error"] = "agent health not polled"
|
||||
}
|
||||
if len(view.Sessions) > 0 {
|
||||
if view.successful() {
|
||||
sess := make([]map[string]any, 0, len(view.Sessions))
|
||||
for _, s := range view.Sessions {
|
||||
row := map[string]any{
|
||||
@@ -68,62 +113,3 @@ func speakerLiveStatusJSON(sp *store.Speaker, view speakerBGPLive, health *noded
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (s *Server) collectSpeakerLiveStatus(ctx context.Context, tenantID string, fresh bool, speakers []*store.Speaker) map[string]map[string]any {
|
||||
views := s.collectSpeakerBGPLive(ctx, tenantID, fresh)
|
||||
viewByID := make(map[string]speakerBGPLive, len(views))
|
||||
for _, v := range views {
|
||||
if v.SpeakerID != "" {
|
||||
viewByID[v.SpeakerID] = v
|
||||
}
|
||||
}
|
||||
|
||||
opts := nodedispatch.Options{Timeout: 8 * time.Second}
|
||||
type healthWrap struct {
|
||||
id string
|
||||
h nodedispatch.AgentHealthResult
|
||||
}
|
||||
healthCh := make(chan healthWrap, len(speakers))
|
||||
var wg sync.WaitGroup
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(speaker *store.Speaker) {
|
||||
defer wg.Done()
|
||||
healthCh <- healthWrap{
|
||||
id: speaker.ID,
|
||||
h: nodedispatch.FetchAgentHealth(ctx, speaker, opts),
|
||||
}
|
||||
}(sp)
|
||||
}
|
||||
wg.Wait()
|
||||
close(healthCh)
|
||||
healthByID := make(map[string]nodedispatch.AgentHealthResult, len(speakers))
|
||||
for hw := range healthCh {
|
||||
healthByID[hw.id] = hw.h
|
||||
}
|
||||
|
||||
out := make(map[string]map[string]any, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
view, ok := viewByID[sp.ID]
|
||||
if !ok {
|
||||
view = speakerBGPLive{SpeakerID: sp.ID, Label: speakerDisplayLabel(sp)}
|
||||
}
|
||||
var hp *nodedispatch.AgentHealthResult
|
||||
if h, ok := healthByID[sp.ID]; ok {
|
||||
hCopy := h
|
||||
hp = &hCopy
|
||||
}
|
||||
out[sp.ID] = speakerLiveStatusJSON(sp, view, hp)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -2,12 +2,29 @@ package httpapi
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestSpeakerLiveStatusJSONEmptyAndUnpolled(t *testing.T) {
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
unpolled := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID}, nil)
|
||||
if unpolled["agent_ok"] != false || unpolled["bgp_poll_ok"] != false || unpolled["bgp_sessions_total"] != nil {
|
||||
t.Fatalf("unpolled=%#v", unpolled)
|
||||
}
|
||||
if unpolled["bgp_observation"].(liveObservation).Quality != "unknown" {
|
||||
t.Fatalf("unpolled observation=%#v", unpolled["bgp_observation"])
|
||||
}
|
||||
now := time.Date(2026, 10, 26, 10, 0, 0, 0, time.UTC)
|
||||
empty := speakerLiveStatusJSON(sp, speakerBGPLive{SpeakerID: sp.ID, Polled: true, ObservedAt: now, ReceivedAt: now, LastSuccessAt: now, Source: "cp_birdc"}, nil)
|
||||
if empty["agent_ok"] != true || empty["bgp_poll_ok"] != true || empty["bgp_sessions_total"] != 0 {
|
||||
t.Fatalf("empty=%#v", empty)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCountBGPSessions(t *testing.T) {
|
||||
total, est := countBGPSessions([]birdfmt.BGPSession{
|
||||
{Name: "p1", State: "Established"},
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
"evobgp/internal/telemetry"
|
||||
)
|
||||
|
||||
func (s *Server) loadTelemetry(ctx context.Context, tenant string, speakers []*store.Speaker) ([]speakerBGPLive, map[string]map[string]any, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||
defer cancel()
|
||||
snapshots, err := s.store.ListTelemetry(ctx, tenant)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
views, statuses := hydrateTelemetry(speakers, snapshots, time.Now(), telemetry.ConfigFromEnv().StaleAfter)
|
||||
return views, statuses, nil
|
||||
}
|
||||
|
||||
// snapshotObservation uses control-plane receipt time, never the remote clock.
|
||||
func snapshotObservation(o store.TelemetryObservation, now time.Time, staleAfter time.Duration, invalid string) liveObservation {
|
||||
v := liveObservation{ObservedAt: liveTime(o.ObservedAt), ReceivedAt: liveTime(o.ReceivedAt), LastSuccessAt: liveTime(o.LastSuccessAt), Source: o.Source, Quality: "unknown"}
|
||||
if !o.ReceivedAt.IsZero() {
|
||||
v.ExpiresAt = liveTime(o.ReceivedAt.Add(staleAfter))
|
||||
}
|
||||
if v.Source == "" {
|
||||
v.Source = "none"
|
||||
}
|
||||
switch {
|
||||
case invalid != "":
|
||||
v.ErrorCode = invalid
|
||||
case o.ReceivedAt.IsZero():
|
||||
v.ErrorCode = "not_polled"
|
||||
case o.ReceivedAt.After(now) || now.Sub(o.ReceivedAt) > staleAfter:
|
||||
v.Quality, v.ErrorCode = "stale", "snapshot_expired"
|
||||
case o.Error != "":
|
||||
v.Quality, v.ErrorCode = "unreachable", liveErrorCode(o.Error)
|
||||
if !o.LastSuccessAt.IsZero() {
|
||||
v.Quality = "stale"
|
||||
}
|
||||
default:
|
||||
v.Quality = "ok"
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func hydrateTelemetry(speakers []*store.Speaker, snapshots []store.TelemetrySnapshot, now time.Time, staleAfter time.Duration) ([]speakerBGPLive, map[string]map[string]any) {
|
||||
byID := make(map[string]*store.Speaker, len(speakers))
|
||||
for _, sp := range speakers {
|
||||
if sp != nil {
|
||||
byID[sp.ID] = sp
|
||||
}
|
||||
}
|
||||
grouped := make(map[string][]store.TelemetrySnapshot)
|
||||
for _, snapshot := range snapshots {
|
||||
if snapshot.SpeakerID != "" && byID[snapshot.SpeakerID] == nil {
|
||||
continue
|
||||
}
|
||||
if snapshot.SpeakerID == "" && snapshot.Kind != "local_bird" {
|
||||
continue
|
||||
}
|
||||
grouped[snapshot.SpeakerID] = append(grouped[snapshot.SpeakerID], snapshot)
|
||||
}
|
||||
views := make([]speakerBGPLive, 0, len(speakers)+1)
|
||||
statuses := make(map[string]map[string]any, len(speakers))
|
||||
appendView := func(id string, sp *store.Speaker) {
|
||||
view := speakerBGPLive{SpeakerID: id, Label: speakerDisplayLabel(sp)}
|
||||
if sp == nil {
|
||||
view.Label = "CP (local)"
|
||||
}
|
||||
var health *agentHealthPoll
|
||||
rows := grouped[id]
|
||||
// A restarted local collector may leave an expired authority behind.
|
||||
// Only simultaneously current authorities conflict; never prefer one
|
||||
// of two current BIRD nodes arbitrarily.
|
||||
if len(rows) > 1 {
|
||||
current := make([]store.TelemetrySnapshot, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
if row.Kind == "local_bird" && row.ConfigFingerprint == store.TelemetrySpeakerFingerprint(sp) && !row.BGP.ReceivedAt.IsZero() && !row.BGP.ReceivedAt.After(now) && now.Sub(row.BGP.ReceivedAt) <= staleAfter {
|
||||
current = append(current, row)
|
||||
}
|
||||
}
|
||||
if len(current) > 0 {
|
||||
rows = current
|
||||
}
|
||||
}
|
||||
if len(rows) > 1 {
|
||||
view.Error = "ambiguous telemetry authorities"
|
||||
view.ObservationOverride = &liveObservation{Source: "none", Quality: "unknown", ErrorCode: "ambiguous_authority"}
|
||||
} else if len(rows) == 1 {
|
||||
row := rows[0]
|
||||
invalid := ""
|
||||
if row.ConfigFingerprint != store.TelemetrySpeakerFingerprint(sp) {
|
||||
invalid = "configuration_changed"
|
||||
}
|
||||
bgp := snapshotObservation(row.BGP.TelemetryObservation, now, staleAfter, invalid)
|
||||
view.ObservationOverride = &bgp
|
||||
view.Polled = !row.BGP.ReceivedAt.IsZero()
|
||||
view.ObservedAt, view.ReceivedAt, view.LastSuccessAt, view.Source = row.BGP.ObservedAt, row.BGP.ReceivedAt, row.BGP.LastSuccessAt, row.BGP.Source
|
||||
if bgp.Quality == "ok" {
|
||||
view.Sessions = row.BGP.Sessions
|
||||
} else {
|
||||
view.Error = bgp.ErrorCode
|
||||
}
|
||||
if row.Kind == "remote_agent" {
|
||||
h := snapshotObservation(row.Health.TelemetryObservation, now, staleAfter, invalid)
|
||||
result := nodedispatch.AgentHealthResult{OK: row.Health.OK && h.Quality == "ok", Error: row.Health.Error, LastSyncAt: row.Health.LastSyncAt, LastAppliedRevisionID: row.Health.LastAppliedRevisionID}
|
||||
if h.Quality != "ok" {
|
||||
result.Error = h.ErrorCode
|
||||
}
|
||||
if h.Quality == "ok" && !result.OK {
|
||||
h.Quality, h.ErrorCode = "unreachable", "reported_unhealthy"
|
||||
}
|
||||
health = &agentHealthPoll{result: result, observationOverride: &h}
|
||||
}
|
||||
}
|
||||
views = append(views, view)
|
||||
if sp != nil {
|
||||
statuses[id] = speakerLiveStatusWithPoll(sp, view, health)
|
||||
}
|
||||
}
|
||||
for _, sp := range speakers {
|
||||
if sp != nil {
|
||||
appendView(sp.ID, sp)
|
||||
}
|
||||
}
|
||||
if len(grouped[""]) > 0 {
|
||||
appendView("", nil)
|
||||
}
|
||||
return views, statuses
|
||||
}
|
||||
|
||||
func (s *Server) handleTelemetryRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := authFromContext(r.Context())
|
||||
if !ok {
|
||||
writeProblem(w, http.StatusUnauthorized, "Unauthorized", "missing auth")
|
||||
return
|
||||
}
|
||||
if !s.requirePerm(w, a, "bgp:network:read") {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second)
|
||||
defer cancel()
|
||||
requested, err := s.store.RequestTelemetryRefresh(ctx, a.TenantID, 10*time.Second)
|
||||
if err != nil {
|
||||
writeProblem(w, http.StatusServiceUnavailable, "Service Unavailable", "telemetry refresh unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"requested": requested})
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type telemetryHTTPStore struct {
|
||||
store.Backend
|
||||
discoveryWrites int
|
||||
snapshotErr error
|
||||
refreshCalls int
|
||||
}
|
||||
|
||||
func (s *telemetryHTTPStore) UpsertPeerDiscovery(tenant string, in *store.PeerDiscoveryUpsert) (*store.BGPPeerDiscovery, error) {
|
||||
s.discoveryWrites++
|
||||
return s.Backend.UpsertPeerDiscovery(tenant, in)
|
||||
}
|
||||
func (s *telemetryHTTPStore) ListTelemetry(ctx context.Context, tenant string) ([]store.TelemetrySnapshot, error) {
|
||||
if s.snapshotErr != nil {
|
||||
return nil, s.snapshotErr
|
||||
}
|
||||
return s.Backend.ListTelemetry(ctx, tenant)
|
||||
}
|
||||
func (s *telemetryHTTPStore) RequestTelemetryRefresh(ctx context.Context, tenant string, cooldown time.Duration) (bool, error) {
|
||||
s.refreshCalls++
|
||||
return s.Backend.RequestTelemetryRefresh(ctx, tenant, cooldown)
|
||||
}
|
||||
func telemetryRequest(method, path, tenant string) *http.Request {
|
||||
r := httptest.NewRequest(method, path, nil)
|
||||
return r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Role: "viewer"}))
|
||||
}
|
||||
|
||||
func TestTelemetryGETsAndRefreshNeverPollAgents(t *testing.T) {
|
||||
var calls atomic.Int64
|
||||
agent := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
_, _ = w.Write([]byte(`{"ok":true,"sessions":[]}`))
|
||||
}))
|
||||
defer agent.Close()
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_INSECURE_TLS", "1")
|
||||
mem := store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
tenant, _, _, _, _ := mem.DemoIDs()
|
||||
sp, err := mem.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: strings.TrimPrefix(agent.URL, "https://"), AgentSecret: "test"})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Positive control: the legacy adapter reaches the fake configured agent.
|
||||
if result := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: agent.Client()}); !result.OK {
|
||||
t.Fatal(result)
|
||||
}
|
||||
calls.Store(0)
|
||||
st := &telemetryHTTPStore{Backend: mem}
|
||||
server := &Server{store: st}
|
||||
claims, err := mem.ClaimTelemetry(context.Background(), "collector", 1, time.Minute, "")
|
||||
if err != nil || len(claims) != 1 {
|
||||
t.Fatalf("claim: %v %v", claims, err)
|
||||
}
|
||||
for i := 0; i < 20; i++ {
|
||||
for _, tc := range []struct {
|
||||
path string
|
||||
handler http.HandlerFunc
|
||||
}{
|
||||
{"/v1/peers", server.handleListPeers}, {"/v1/peers?live=1", server.handleListPeers},
|
||||
{"/v1/speakers?live=1", server.handleListSpeakers}, {"/v1/peers/discovered?status=all", server.handleListPeerDiscoveries},
|
||||
} {
|
||||
w := httptest.NewRecorder()
|
||||
tc.handler(w, telemetryRequest(http.MethodGet, tc.path, tenant))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("%s: %d %s", tc.path, w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
w := httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, telemetryRequest(http.MethodPost, "/v1/telemetry/refresh", tenant))
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Fatalf("refresh: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
want := `"requested":false`
|
||||
if i == 0 {
|
||||
want = `"requested":true`
|
||||
}
|
||||
if !strings.Contains(w.Body.String(), want) {
|
||||
t.Fatalf("refresh coalescing: %s", w.Body.String())
|
||||
}
|
||||
}
|
||||
if calls.Load() != 0 || st.discoveryWrites != 0 {
|
||||
t.Fatalf("GET/refresh caused IO: agent=%d discovery=%d", calls.Load(), st.discoveryWrites)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryReadFailureAndRefreshPermission(t *testing.T) {
|
||||
mem := store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
tenant, _, _, _, _ := mem.DemoIDs()
|
||||
st := &telemetryHTTPStore{Backend: mem, snapshotErr: errors.New("private database failure")}
|
||||
server := &Server{store: st}
|
||||
for _, handler := range []http.HandlerFunc{server.handleListPeers, server.handleListSpeakers} {
|
||||
w := httptest.NewRecorder()
|
||||
handler(w, telemetryRequest(http.MethodGet, "/v1/resource?live=1", tenant))
|
||||
if w.Code != http.StatusServiceUnavailable || strings.Contains(w.Body.String(), "private") {
|
||||
t.Fatalf("snapshot failure: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, "/v1/telemetry/refresh", nil)
|
||||
w := httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, r)
|
||||
if w.Code != http.StatusUnauthorized || st.refreshCalls != 0 {
|
||||
t.Fatal("unauthenticated refresh admitted")
|
||||
}
|
||||
r = r.WithContext(context.WithValue(r.Context(), authCtxKey, Auth{TenantID: tenant, Kind: AuthKindJWT}))
|
||||
w = httptest.NewRecorder()
|
||||
server.handleTelemetryRefresh(w, r)
|
||||
if w.Code != http.StatusForbidden || st.refreshCalls != 0 {
|
||||
t.Fatal("unprivileged refresh admitted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestTelemetryBackgroundStartsOnceAndStopsBeforeClose(t *testing.T) {
|
||||
s, err := New(Options{SeedDemo: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s.StartBackground(ctx)
|
||||
first := s.telemetryCollector
|
||||
if first == nil {
|
||||
t.Fatal("telemetry collector not started")
|
||||
}
|
||||
s.StartBackground(ctx)
|
||||
if s.telemetryCollector != first {
|
||||
t.Fatal("duplicate telemetry collector")
|
||||
}
|
||||
s.Close()
|
||||
s.StartBackground(ctx)
|
||||
if s.telemetryCollector != first {
|
||||
t.Fatal("collector started after close")
|
||||
}
|
||||
s.Close()
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/store"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestTelemetryHydrationFreshnessAndConfiguration(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
sp := &store.Speaker{ID: "replica", Role: "replica", Endpoint: "old"}
|
||||
observation := store.TelemetryObservation{ObservedAt: now.Add(-time.Hour), ReceivedAt: now, LastSuccessAt: now, Source: "agent"}
|
||||
base := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: observation, Sessions: []birdfmt.BGPSession{{Name: "peer", State: "Established"}}}, Health: store.TelemetryHealth{TelemetryObservation: observation, OK: true}}
|
||||
cases := []struct {
|
||||
name string
|
||||
mutate func(*store.TelemetrySnapshot)
|
||||
bgp, health bool
|
||||
quality string
|
||||
}{
|
||||
{"fresh receipt despite remote clock", func(*store.TelemetrySnapshot) {}, true, true, "ok"},
|
||||
{"expired BGP independent of health", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(-2 * time.Minute) }, false, true, "stale"},
|
||||
{"expired health independent of BGP", func(s *store.TelemetrySnapshot) { s.Health.ReceivedAt = now.Add(-2 * time.Minute) }, true, false, "ok"},
|
||||
{"future receipt is not fresh", func(s *store.TelemetrySnapshot) { s.BGP.ReceivedAt = now.Add(time.Hour) }, false, true, "stale"},
|
||||
{"configuration changed", func(s *store.TelemetrySnapshot) { s.ConfigFingerprint = "old" }, false, false, "unknown"},
|
||||
{"failed BGP", func(s *store.TelemetrySnapshot) { s.BGP.Error = "timeout" }, false, true, "stale"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
snapshot := base
|
||||
tc.mutate(&snapshot)
|
||||
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{snapshot}, now, time.Minute)
|
||||
status := statuses[sp.ID]
|
||||
if status["bgp_poll_ok"] != tc.bgp || status["agent_ok"] != tc.health || views[0].observation().Quality != tc.quality {
|
||||
t.Fatalf("status=%+v observation=%+v", status, views[0].observation())
|
||||
}
|
||||
if !tc.bgp && (status["bgp_established"] != nil || len(views[0].Sessions) != 0) {
|
||||
t.Fatal("stale BGP exposed live sessions")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryHydrationMissingAndAmbiguousLocal(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
for _, rows := range [][]store.TelemetrySnapshot{nil, {{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "a"}, {SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "b"}}} {
|
||||
views, statuses := hydrateTelemetry([]*store.Speaker{sp}, rows, now, time.Minute)
|
||||
if views[0].observation().Quality != "unknown" || statuses[sp.ID]["agent_ok"] == true || statuses[sp.ID]["bgp_poll_ok"] == true {
|
||||
t.Fatalf("unexpected healthy master: %+v", statuses)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryHydrationLocalAuthorityRestart(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "master", Role: "master"}
|
||||
fresh := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "local_bird", LocalAuthority: "new", ConfigFingerprint: store.TelemetrySpeakerFingerprint(sp), BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now}}}
|
||||
old := fresh
|
||||
old.LocalAuthority = "old"
|
||||
old.BGP.ReceivedAt = now.Add(-2 * time.Minute)
|
||||
_, status := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||
if status[sp.ID]["bgp_poll_ok"] != true {
|
||||
t.Fatal("expired authority hides current local snapshot")
|
||||
}
|
||||
old.BGP.ReceivedAt = now
|
||||
_, status = hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{old, fresh}, now, time.Minute)
|
||||
if status[sp.ID]["bgp_poll_ok"] != false {
|
||||
t.Fatal("two current local authorities are ambiguous")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryPeerObservationPreservesUnknownConfiguration(t *testing.T) {
|
||||
now := time.Now()
|
||||
sp := &store.Speaker{ID: "replica", Role: "replica"}
|
||||
row := store.TelemetrySnapshot{SpeakerID: sp.ID, Kind: "remote_agent", ConfigFingerprint: "previous", BGP: store.TelemetryBGP{TelemetryObservation: store.TelemetryObservation{ReceivedAt: now, LastSuccessAt: now}}}
|
||||
views, _ := hydrateTelemetry([]*store.Speaker{sp}, []store.TelemetrySnapshot{row}, now, time.Minute)
|
||||
observation := peerLiveObservation(views)
|
||||
if observation.Quality != "unknown" || observation.ErrorCode != "configuration_changed" {
|
||||
t.Fatalf("changed config is not a failed current poll: %+v", observation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTelemetryExpiryUsesConfiguredIntervalAndEarliestSource(t *testing.T) {
|
||||
now := time.Now().UTC()
|
||||
old := now.Add(-30 * time.Second)
|
||||
a := snapshotObservation(store.TelemetryObservation{ReceivedAt: old}, now, 5*time.Minute, "")
|
||||
b := snapshotObservation(store.TelemetryObservation{ReceivedAt: now}, now, 5*time.Minute, "")
|
||||
want := old.Add(5 * time.Minute).Format(time.RFC3339Nano)
|
||||
if a.ExpiresAt == nil || *a.ExpiresAt != want {
|
||||
t.Fatalf("expiry ignores configured interval: %+v", a)
|
||||
}
|
||||
views := []speakerBGPLive{{Polled: true, ObservationOverride: &b}, {Polled: true, ObservationOverride: &a}}
|
||||
combined := peerLiveObservation(views)
|
||||
if combined.ExpiresAt == nil || *combined.ExpiresAt != want {
|
||||
t.Fatalf("aggregate extends oldest source: %+v", combined)
|
||||
}
|
||||
missing := snapshotObservation(store.TelemetryObservation{}, now, time.Minute, "")
|
||||
if missing.ExpiresAt != nil {
|
||||
t.Fatal("unobserved source has expiry")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// trustedProxyPrefixes returns no trusted peers when any configured CIDR is invalid.
|
||||
// An empty configuration deliberately treats every connection as direct.
|
||||
func trustedProxyPrefixes() []netip.Prefix {
|
||||
raw := strings.TrimSpace(os.Getenv("EVOBGP_TRUSTED_PROXY_CIDRS"))
|
||||
if raw == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(raw, ",")
|
||||
prefixes := make([]netip.Prefix, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
prefix, err := netip.ParsePrefix(strings.TrimSpace(part))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if prefix.Addr().Is4In6() {
|
||||
if prefix.Bits() < 96 {
|
||||
return nil
|
||||
}
|
||||
prefix = netip.PrefixFrom(prefix.Addr().Unmap(), prefix.Bits()-96)
|
||||
}
|
||||
prefixes = append(prefixes, prefix.Masked())
|
||||
}
|
||||
return prefixes
|
||||
}
|
||||
|
||||
func peerAddress(r *http.Request) (netip.Addr, bool) {
|
||||
if r == nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
host, _, err := net.SplitHostPort(strings.TrimSpace(r.RemoteAddr))
|
||||
if err != nil {
|
||||
host = strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
ip, err := netip.ParseAddr(host)
|
||||
if err != nil {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
return ip.WithZone("").Unmap(), true
|
||||
}
|
||||
|
||||
func isTrustedProxy(ip netip.Addr, prefixes []netip.Prefix) bool {
|
||||
for _, prefix := range prefixes {
|
||||
if prefix.Contains(ip) || prefix.Contains(ip.Unmap()) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func clientIPWithTrustedProxies(r *http.Request, prefixes []netip.Prefix) string {
|
||||
peer, ok := peerAddress(r)
|
||||
if !ok {
|
||||
if r == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(r.RemoteAddr)
|
||||
}
|
||||
if !isTrustedProxy(peer, prefixes) {
|
||||
return peer.String()
|
||||
}
|
||||
values := r.Header.Values("X-Forwarded-For")
|
||||
if len(values) != 1 || strings.TrimSpace(values[0]) == "" {
|
||||
return peer.String()
|
||||
}
|
||||
hops := strings.Split(values[0], ",")
|
||||
current := peer
|
||||
for i := len(hops) - 1; i >= 0; i-- {
|
||||
if !isTrustedProxy(current, prefixes) {
|
||||
return current.String()
|
||||
}
|
||||
next, err := netip.ParseAddr(strings.TrimSpace(hops[i]))
|
||||
if err != nil {
|
||||
return peer.String()
|
||||
}
|
||||
if next.Zone() != "" {
|
||||
return peer.String()
|
||||
}
|
||||
current = next.Unmap()
|
||||
}
|
||||
return current.String()
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package httpapi
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestClientIPTrustedProxyChain(t *testing.T) {
|
||||
prefixes := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
|
||||
tests := []struct {
|
||||
name string
|
||||
remote string
|
||||
xff string
|
||||
want string
|
||||
}{
|
||||
{"untrusted peer ignores spoof", "198.51.100.10:1234", "203.0.113.9", "198.51.100.10"},
|
||||
{"rightmost untrusted hop", "10.0.0.2:1234", "203.0.113.9, 198.51.100.8", "198.51.100.8"},
|
||||
{"trusted chain", "10.0.0.2:1234", "203.0.113.9, 10.0.0.3", "203.0.113.9"},
|
||||
{"mapped peer", "[::ffff:10.0.0.2]:1234", "203.0.113.9", "203.0.113.9"},
|
||||
{"mapped forwarded client", "10.0.0.2:1234", "::ffff:203.0.113.9", "203.0.113.9"},
|
||||
{"malformed nearest hop", "10.0.0.2:1234", "203.0.113.9, bogus", "10.0.0.2"},
|
||||
{"malformed farther hop ignored", "10.0.0.2:1234", "bogus, 198.51.100.8", "198.51.100.8"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||
r.RemoteAddr = tt.remote
|
||||
r.Header.Set("X-Forwarded-For", tt.xff)
|
||||
if got := clientIPWithTrustedProxies(r, prefixes); got != tt.want {
|
||||
t.Fatalf("client IP = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientIPConfigurationFailsClosed(t *testing.T) {
|
||||
r := httptest.NewRequest("GET", "http://example.test/", nil)
|
||||
r.RemoteAddr = "10.0.0.2:1234"
|
||||
r.Header.Set("X-Forwarded-For", "203.0.113.9")
|
||||
r.Header.Set("X-Real-IP", "203.0.113.8")
|
||||
if got := clientIPWithTrustedProxies(r, nil); got != "10.0.0.2" {
|
||||
t.Fatalf("direct client IP = %q", got)
|
||||
}
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8, invalid")
|
||||
if got := clientIP(r); got != "10.0.0.2" {
|
||||
t.Fatalf("invalid CIDR client IP = %q", got)
|
||||
}
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "::ffff:10.0.0.2/128")
|
||||
if got := clientIP(r); got != "203.0.113.9" {
|
||||
t.Fatalf("mapped CIDR client IP = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicControlPlaneURLTrust(t *testing.T) {
|
||||
t.Setenv("EVOBGP_TRUSTED_PROXY_CIDRS", "10.0.0.0/8")
|
||||
r := httptest.NewRequest("POST", "http://internal.local:8080/v1/speakers", nil)
|
||||
r.Header.Set("Origin", "https://evil.example")
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||
r.RemoteAddr = "198.51.100.8:1234"
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Fatalf("direct URL = %q", got)
|
||||
}
|
||||
r.RemoteAddr = "10.0.0.2:1234"
|
||||
if got := publicControlPlaneURL(r, ""); got != "https://bgp.example.test" {
|
||||
t.Fatalf("proxied URL = %q", got)
|
||||
}
|
||||
if got := publicControlPlaneURL(r, "https://explicit.example/"); got != "https://explicit.example" {
|
||||
t.Fatalf("override URL = %q", got)
|
||||
}
|
||||
for _, bad := range []struct{ proto, host string }{
|
||||
{"javascript", "bgp.example.test"},
|
||||
{"https,http", "bgp.example.test"},
|
||||
{"https", "evil.example/path"},
|
||||
{"https", "[email protected]"},
|
||||
{"https", "evil.example,bgp.example.test"},
|
||||
{"https", "bgp.example.test:99999"},
|
||||
{"https", "bgp.example.test:"},
|
||||
} {
|
||||
r.Header.Set("X-Forwarded-Proto", bad.proto)
|
||||
r.Header.Set("X-Forwarded-Host", bad.host)
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Errorf("bad forwarded headers (%q, %q): %q", bad.proto, bad.host, got)
|
||||
}
|
||||
}
|
||||
r.Header.Set("X-Forwarded-Proto", "https")
|
||||
r.Header.Set("X-Forwarded-Host", "bgp.example.test")
|
||||
r.Header.Add("X-Forwarded-Host", "evil.example")
|
||||
if got := publicControlPlaneURL(r, ""); got != "http://internal.local:8080" {
|
||||
t.Fatalf("duplicate forwarded host = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRateLimiterManyIdentities(t *testing.T) {
|
||||
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 1, window: time.Minute, maxClients: 32}
|
||||
now := time.Unix(1000, 0)
|
||||
for i := 0; i < 1000; i++ {
|
||||
allowed := l.allowAt(fmt.Sprintf("ip-%d", i), now)
|
||||
if allowed != (i < l.maxClients) {
|
||||
t.Fatalf("identity %d allowed = %t", i, allowed)
|
||||
}
|
||||
}
|
||||
if len(l.hits) != l.maxClients || l.allowAt("ip-0", now) {
|
||||
t.Fatalf("active counters changed under cardinality pressure: %d", len(l.hits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAuthRateLimiterCapacityAndExpiry(t *testing.T) {
|
||||
l := &authRateLimiter{hits: make(map[string]authRateHit), limit: 2, window: time.Minute, maxClients: 2}
|
||||
now := time.Unix(1000, 0)
|
||||
if !l.allowAt("a", now) || !l.allowAt("b", now) || !l.allowAt("a", now) {
|
||||
t.Fatal("initial requests rejected")
|
||||
}
|
||||
if l.allowAt("a", now) || l.allowAt("c", now) {
|
||||
t.Fatal("active limit or capacity bypassed")
|
||||
}
|
||||
if len(l.hits) != 2 {
|
||||
t.Fatalf("map size = %d", len(l.hits))
|
||||
}
|
||||
if !l.allowAt("c", now.Add(time.Minute)) {
|
||||
t.Fatal("expired entries did not release capacity")
|
||||
}
|
||||
if len(l.hits) != 1 {
|
||||
t.Fatalf("expired entries remained: %d", len(l.hits))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// ErrResponseTooLarge means the complete decoded HTTP response exceeded its budget.
|
||||
var ErrResponseTooLarge = errors.New("httpclient: response body exceeds limit")
|
||||
|
||||
// ReadResponseBody reads a complete bounded body, including one extra byte to
|
||||
// distinguish an exact limit from truncation. Standard Transport gzip decoding
|
||||
// occurs before this reader, so the limit covers decompressed data as well.
|
||||
func ReadResponseBody(resp *http.Response, limit int64) ([]byte, error) {
|
||||
if limit <= 0 || limit == int64(^uint64(0)>>1) {
|
||||
return nil, fmt.Errorf("httpclient: invalid response limit")
|
||||
}
|
||||
if resp.ContentLength > limit {
|
||||
return nil, ErrResponseTooLarge
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, ErrResponseTooLarge
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestReadResponseBodyDoesNotAcceptValidPrefixTruncation(t *testing.T) {
|
||||
for _, length := range []int64{-1, 9} {
|
||||
resp := &http.Response{ContentLength: length, Body: io.NopCloser(strings.NewReader("{} "))}
|
||||
if _, err := ReadResponseBody(resp, 8); !errors.Is(err, ErrResponseTooLarge) {
|
||||
t.Fatalf("length=%d got %v", length, err)
|
||||
}
|
||||
}
|
||||
resp := &http.Response{ContentLength: -1, Body: io.NopCloser(strings.NewReader("{} "))}
|
||||
if b, err := ReadResponseBody(resp, 8); err != nil || len(b) != 8 {
|
||||
t.Fatalf("exact limit failed: %q %v", b, err)
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,13 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -16,14 +21,77 @@ type hostBreaker struct {
|
||||
openUntil time.Time
|
||||
}
|
||||
|
||||
var hostBreakers sync.Map // string -> *hostBreaker
|
||||
// ErrBreakerCapacity rejects new upstream identities when the bounded cache is full.
|
||||
var ErrBreakerCapacity = errors.New("httpclient: upstream circuit capacity exhausted")
|
||||
|
||||
func breakerForHost(host string) *hostBreaker {
|
||||
type breakerEntry struct {
|
||||
breaker *hostBreaker
|
||||
lastUsed time.Time
|
||||
active int
|
||||
}
|
||||
|
||||
type breakerCache struct {
|
||||
mu sync.Mutex
|
||||
entries map[string]*breakerEntry
|
||||
maximum int
|
||||
idle time.Duration
|
||||
nextSweep time.Time
|
||||
}
|
||||
|
||||
var hostBreakers = breakerCache{entries: make(map[string]*breakerEntry), maximum: 1024, idle: 10 * time.Minute}
|
||||
|
||||
func (c *breakerCache) sweep(now time.Time) {
|
||||
for host, e := range c.entries {
|
||||
if e.active != 0 || now.Before(e.lastUsed.Add(c.idle)) {
|
||||
continue
|
||||
}
|
||||
e.breaker.mu.Lock()
|
||||
open := now.Before(e.breaker.openUntil)
|
||||
e.breaker.mu.Unlock()
|
||||
if !open {
|
||||
delete(c.entries, host)
|
||||
observability.RemoveUpstreamBreaker(host)
|
||||
}
|
||||
}
|
||||
c.nextSweep = now.Add(time.Second)
|
||||
}
|
||||
|
||||
func (c *breakerCache) acquire(host string, now time.Time) (*hostBreaker, string, func(), error) {
|
||||
if len(host) > 254 {
|
||||
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
|
||||
}
|
||||
host = strings.TrimSuffix(strings.ToLower(host), ".")
|
||||
if len(host) > 253 {
|
||||
return nil, "", nil, fmt.Errorf("httpclient: upstream hostname too long")
|
||||
}
|
||||
if host == "" {
|
||||
host = "_"
|
||||
}
|
||||
v, _ := hostBreakers.LoadOrStore(host, &hostBreaker{})
|
||||
return v.(*hostBreaker)
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if !now.Before(c.nextSweep) {
|
||||
c.sweep(now)
|
||||
}
|
||||
e := c.entries[host]
|
||||
if e == nil {
|
||||
if len(c.entries) >= c.maximum {
|
||||
return nil, host, nil, ErrBreakerCapacity
|
||||
}
|
||||
e = &breakerEntry{breaker: &hostBreaker{}}
|
||||
c.entries[host] = e
|
||||
}
|
||||
e.active++
|
||||
e.lastUsed = now
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
e.active--
|
||||
e.lastUsed = time.Now()
|
||||
})
|
||||
}
|
||||
return e.breaker, host, release, nil
|
||||
}
|
||||
|
||||
func (b *hostBreaker) allow() bool {
|
||||
@@ -51,23 +119,31 @@ func (b *hostBreaker) recordFailure() {
|
||||
|
||||
// SnapshotBreakers returns whether each known host breaker is currently open.
|
||||
func SnapshotBreakers() map[string]bool {
|
||||
hostBreakers.mu.Lock()
|
||||
defer hostBreakers.mu.Unlock()
|
||||
now := time.Now()
|
||||
hostBreakers.sweep(now)
|
||||
out := map[string]bool{}
|
||||
hostBreakers.Range(func(key, value any) bool {
|
||||
host, _ := key.(string)
|
||||
b, _ := value.(*hostBreaker)
|
||||
if b == nil {
|
||||
return true
|
||||
}
|
||||
for host, e := range hostBreakers.entries {
|
||||
b := e.breaker
|
||||
b.mu.Lock()
|
||||
open := time.Now().Before(b.openUntil)
|
||||
b.mu.Unlock()
|
||||
out[host] = open
|
||||
return true
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ResetHostBreakers clears all circuit breakers (tests only).
|
||||
// ResetHostBreakers clears idle circuit breakers (tests only). In-flight
|
||||
// references are retained so their completions cannot recreate orphan metrics.
|
||||
func ResetHostBreakers() {
|
||||
hostBreakers = sync.Map{}
|
||||
hostBreakers.mu.Lock()
|
||||
defer hostBreakers.mu.Unlock()
|
||||
for host, e := range hostBreakers.entries {
|
||||
if e.active == 0 {
|
||||
delete(hostBreakers.entries, host)
|
||||
observability.RemoveUpstreamBreaker(host)
|
||||
}
|
||||
}
|
||||
hostBreakers.nextSweep = time.Time{}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
func TestBreakerCanonicalHostAndActiveRetention(t *testing.T) {
|
||||
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
|
||||
now := time.Now()
|
||||
first, key, release, err := c.acquire("Cache-Test.Example.", now)
|
||||
if err != nil || key != "cache-test.example" {
|
||||
t.Fatalf("canonical key=%s err=%v", key, err)
|
||||
}
|
||||
second, _, release2, err := c.acquire("cache-test.example", now)
|
||||
if err != nil || first != second || len(c.entries) != 1 {
|
||||
t.Fatal("alias bypassed shared breaker")
|
||||
}
|
||||
observability.SetUpstreamBreakerOpen(key, true)
|
||||
first.openUntil = now.Add(3 * time.Minute)
|
||||
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute)); !errors.Is(err, ErrBreakerCapacity) {
|
||||
t.Fatalf("active entry evicted: %v", err)
|
||||
}
|
||||
release()
|
||||
release()
|
||||
release2()
|
||||
if _, _, _, err := c.acquire("other.example", now.Add(2*time.Minute+2*time.Second)); !errors.Is(err, ErrBreakerCapacity) {
|
||||
t.Fatalf("open breaker evicted: %v", err)
|
||||
}
|
||||
_, _, done, err := c.acquire("other.example", now.Add(4*time.Minute))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer done()
|
||||
if len(c.entries) != 1 || c.entries[key] != nil {
|
||||
t.Fatal("idle closed entry retained")
|
||||
}
|
||||
families, err := prometheus.DefaultGatherer.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, family := range families {
|
||||
if strings.HasSuffix(family.GetName(), "upstream_breaker_open") {
|
||||
for _, metric := range family.Metric {
|
||||
for _, label := range metric.Label {
|
||||
if label.GetName() == "host" && label.GetValue() == key {
|
||||
t.Fatal("evicted host metric retained")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type breakerTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f breakerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestBreakerCapacityRejectsBeforeDial(t *testing.T) {
|
||||
ResetHostBreakers()
|
||||
var releases []func()
|
||||
defer func() {
|
||||
for _, release := range releases {
|
||||
release()
|
||||
}
|
||||
ResetHostBreakers()
|
||||
}()
|
||||
for i := 0; i < hostBreakers.maximum; i++ {
|
||||
_, _, release, err := hostBreakers.acquire(fmt.Sprintf("capacity-%d.example", i), time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
releases = append(releases, release)
|
||||
}
|
||||
called := false
|
||||
hc := &http.Client{Transport: breakerTransport(func(*http.Request) (*http.Response, error) { called = true; return nil, errors.New("unexpected dial") })}
|
||||
req, _ := http.NewRequest(http.MethodGet, "https://overflow.example/", nil)
|
||||
_, err := DoWithBreaker(context.Background(), hc, req, 1)
|
||||
if !errors.Is(err, ErrBreakerCapacity) || called {
|
||||
t.Fatalf("capacity err=%v dial=%t", err, called)
|
||||
}
|
||||
if len(SnapshotBreakers()) != hostBreakers.maximum {
|
||||
t.Fatal("cache capacity changed")
|
||||
}
|
||||
ResetHostBreakers()
|
||||
if len(SnapshotBreakers()) != hostBreakers.maximum {
|
||||
t.Fatal("reset discarded in-flight references")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBreakerRejectsOversizedIdentity(t *testing.T) {
|
||||
c := breakerCache{entries: make(map[string]*breakerEntry), maximum: 1, idle: time.Minute}
|
||||
if _, _, _, err := c.acquire(strings.Repeat("x", 254), time.Now()); err == nil || len(c.entries) != 0 {
|
||||
t.Fatal("oversized identity retained")
|
||||
}
|
||||
name := strings.Repeat("a", 253)
|
||||
first, key, release, err := c.acquire(name, time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release()
|
||||
second, alias, release2, err := c.acquire(strings.ToUpper(name)+".", time.Now())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer release2()
|
||||
if first != second || key != alias {
|
||||
t.Fatal("maximum length trailing-dot alias differs")
|
||||
}
|
||||
}
|
||||
@@ -54,7 +54,9 @@ func DoWithRetry(ctx context.Context, hc *http.Client, req *http.Request, maxAtt
|
||||
continue
|
||||
}
|
||||
if resp.StatusCode >= 500 {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
// Draining is only an optimization for connection reuse, never an
|
||||
// invitation to consume an unbounded error stream before retrying.
|
||||
_, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10))
|
||||
_ = resp.Body.Close()
|
||||
lastErr = fmt.Errorf("httpclient: upstream %s", resp.Status)
|
||||
continue
|
||||
@@ -72,8 +74,11 @@ func DoWithBreaker(ctx context.Context, hc *http.Client, req *http.Request, maxA
|
||||
if req == nil || req.URL == nil {
|
||||
return nil, fmt.Errorf("httpclient: nil request")
|
||||
}
|
||||
host := req.URL.Hostname()
|
||||
br := breakerForHost(host)
|
||||
br, host, release, err := hostBreakers.acquire(req.URL.Hostname(), time.Now())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer release()
|
||||
if !br.allow() {
|
||||
observability.SetUpstreamBreakerOpen(host, true)
|
||||
return nil, fmt.Errorf("httpclient: circuit open for %s", host)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package httpclient
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type endlessRetryBody struct {
|
||||
read int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *endlessRetryBody) Read(p []byte) (int, error) {
|
||||
for i := range p {
|
||||
p[i] = 'x'
|
||||
}
|
||||
b.read += len(p)
|
||||
return len(p), nil
|
||||
}
|
||||
func (b *endlessRetryBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
type retryTransport func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f retryTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestRetryDrainBoundsEndlessErrorBody(t *testing.T) {
|
||||
b := &endlessRetryBody{}
|
||||
c := &http.Client{Transport: retryTransport(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: 503, Status: "503 Service Unavailable", Body: b, Header: make(http.Header)}, nil
|
||||
})}
|
||||
req, err := http.NewRequest(http.MethodGet, "http://agent.example/bundle", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = DoWithRetry(context.Background(), c, req, 1); err == nil {
|
||||
t.Fatal("upstream failure accepted")
|
||||
}
|
||||
if b.read != 64<<10 || !b.closed {
|
||||
t.Fatalf("retry consumed %d bytes, closed=%v", b.read, b.closed)
|
||||
}
|
||||
var _ io.ReadCloser = b
|
||||
}
|
||||
+109
-11
@@ -1,7 +1,10 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
@@ -10,6 +13,18 @@ import (
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// MaxCSVBytes is the complete request limit, including the optional header.
|
||||
const MaxCSVBytes = 8 << 20
|
||||
|
||||
// MaxCSVRows bounds database work for a single import, excluding its header.
|
||||
const MaxCSVRows = 10000
|
||||
|
||||
// MaxCSVFields bounds CSV parser allocations for delimiter-heavy records.
|
||||
const MaxCSVFields = 64
|
||||
|
||||
// ErrCSVTooLarge means the entire import was rejected before creating entries.
|
||||
var ErrCSVTooLarge = errors.New("importer: csv exceeds byte, row or field limit")
|
||||
|
||||
// Result of importing module entries from CSV.
|
||||
type Result struct {
|
||||
Imported int
|
||||
@@ -17,18 +32,50 @@ type Result struct {
|
||||
}
|
||||
|
||||
// ImportModuleEntriesCSV parses and creates entries for AS_PREFIXES, DOMAINS, or IP_RANGES modules.
|
||||
func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
||||
func ImportModuleEntriesCSV(ctx context.Context, st store.Backend, tenantID, moduleID string, body io.Reader) (*Result, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mod, err := st.GetModule(tenantID, moduleID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cr := csv.NewReader(io.LimitReader(body, 8<<20))
|
||||
// Read one byte beyond the cap: a valid prefix must never be imported as if
|
||||
// it were the complete request. Check all resource limits before mutations.
|
||||
raw, err := io.ReadAll(io.LimitReader(body, MaxCSVBytes+1))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: read csv: %w", err)
|
||||
}
|
||||
if len(raw) > MaxCSVBytes {
|
||||
return nil, ErrCSVTooLarge
|
||||
}
|
||||
if err := checkCSVFields(ctx, raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cr := csv.NewReader(bytes.NewReader(raw))
|
||||
cr.TrimLeadingSpace = true
|
||||
cr.FieldsPerRecord = -1
|
||||
rows, err := cr.ReadAll()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
||||
rows := make([][]string, 0)
|
||||
headerRows := 0
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
record, err := cr.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("importer: invalid csv: %w", err)
|
||||
}
|
||||
if len(rows) == 0 && isCSVHeader(record) {
|
||||
headerRows = 1
|
||||
}
|
||||
if len(rows)-headerRows >= MaxCSVRows {
|
||||
return nil, ErrCSVTooLarge
|
||||
}
|
||||
rows = append(rows, record)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, fmt.Errorf("importer: %w", store.ErrInvalidInput)
|
||||
@@ -63,18 +110,17 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
|
||||
start := 0
|
||||
if len(rows[0]) >= 2 {
|
||||
key := strings.ToLower(strings.TrimSpace(rows[0][0]))
|
||||
switch key {
|
||||
case "asn", "domain", "iprange":
|
||||
start = 1
|
||||
}
|
||||
if isCSVHeader(rows[0]) {
|
||||
start = 1
|
||||
}
|
||||
|
||||
imported := 0
|
||||
switch mod.Type {
|
||||
case "AS_PREFIXES":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -97,6 +143,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
case "DOMAINS":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -119,6 +168,9 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
}
|
||||
case "IP_RANGES":
|
||||
for i := start; i < len(rows); i++ {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rec := rows[i]
|
||||
if len(rec) == 0 || (strings.TrimSpace(rec[0]) == "" && (len(rec) < 2 || strings.TrimSpace(rec[1]) == "")) {
|
||||
continue
|
||||
@@ -145,3 +197,49 @@ func ImportModuleEntriesCSV(st store.Backend, tenantID, moduleID string, body io
|
||||
|
||||
return &Result{Imported: imported, ModuleType: mod.Type}, nil
|
||||
}
|
||||
|
||||
func isCSVHeader(record []string) bool {
|
||||
if len(record) < 2 {
|
||||
return false
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(record[0])) {
|
||||
case "asn", "domain", "iprange":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// checkCSVFields runs before encoding/csv allocates a string/index for each field.
|
||||
// Quoted commas, escaped quotes and multiline fields don't start new fields.
|
||||
// Syntax validation remains the responsibility of encoding/csv below.
|
||||
func checkCSVFields(ctx context.Context, raw []byte) error {
|
||||
quoted, fields := false, 1
|
||||
for i := 0; i < len(raw); i++ {
|
||||
if i%4096 == 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
switch raw[i] {
|
||||
case '"':
|
||||
if quoted && i+1 < len(raw) && raw[i+1] == '"' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
quoted = !quoted
|
||||
case ',':
|
||||
if !quoted {
|
||||
fields++
|
||||
if fields > MaxCSVFields {
|
||||
return ErrCSVTooLarge
|
||||
}
|
||||
}
|
||||
case '\n':
|
||||
if !quoted {
|
||||
fields = 1
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type importBackend struct {
|
||||
store.Backend
|
||||
created int
|
||||
onCreate func()
|
||||
}
|
||||
|
||||
func (s *importBackend) GetModule(string, string) (*store.Module, error) {
|
||||
return &store.Module{Type: "AS_PREFIXES"}, nil
|
||||
}
|
||||
func (s *importBackend) ListCommunities(string) ([]*store.Community, error) { return nil, nil }
|
||||
func (s *importBackend) CreateASEntry(_, _ string, e *store.ASEntry) (*store.ASEntry, error) {
|
||||
s.created++
|
||||
if s.onCreate != nil {
|
||||
s.onCreate()
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func TestCSVByteLimitRejectsValidTruncatedPrefix(t *testing.T) {
|
||||
header := "asn,community\n"
|
||||
for _, extra := range []int{0, 1} {
|
||||
s := &importBackend{}
|
||||
body := header + strings.Repeat("\n", MaxCSVBytes-len(header)) + strings.Repeat("\n", extra)
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(body))
|
||||
if extra == 0 && err != nil {
|
||||
t.Fatalf("exact limit: %v", err)
|
||||
}
|
||||
if extra == 1 && !errors.Is(err, ErrCSVTooLarge) {
|
||||
t.Fatalf("limit+1: %v", err)
|
||||
}
|
||||
if s.created != 0 {
|
||||
t.Fatal("oversized import created entries")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVRowLimitCheckedBeforeWrites(t *testing.T) {
|
||||
for _, header := range []string{"", "asn,community\n"} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
s := &importBackend{}
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(header+strings.Repeat("64512,\n", MaxCSVRows+extra)))
|
||||
if extra == 0 && (err != nil || s.created != MaxCSVRows) {
|
||||
t.Fatalf("exact rows: created=%d err=%v", s.created, err)
|
||||
}
|
||||
if extra == 1 && (!errors.Is(err, ErrCSVTooLarge) || s.created != 0) {
|
||||
t.Fatalf("rows+1: created=%d err=%v", s.created, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVCancellationStopsFurtherWrites(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
s := &importBackend{onCreate: cancel}
|
||||
_, err := ImportModuleEntriesCSV(ctx, s, "tenant", "module", strings.NewReader("asn,community\n64512,\n64513,\n"))
|
||||
if !errors.Is(err, context.Canceled) || s.created != 1 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVFieldLimitAndQuotedFields(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, input string
|
||||
oversized bool
|
||||
}{
|
||||
{"exact", "64512," + strings.Repeat(",", MaxCSVFields-2) + "\n", false},
|
||||
{"over", "64512," + strings.Repeat(",", MaxCSVFields-1) + "\n", true},
|
||||
{"many_fields", strings.Repeat(",", 1<<20) + "\n", true},
|
||||
{"quoted_extra", "64512,,\"" + strings.Repeat(",", 128) + "\nquoted \"\"text\"\"\"\n", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
s := &importBackend{}
|
||||
_, err := ImportModuleEntriesCSV(context.Background(), s, "tenant", "module", strings.NewReader(tc.input))
|
||||
if tc.oversized {
|
||||
if !errors.Is(err, ErrCSVTooLarge) || s.created != 0 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
} else if err != nil || s.created != 1 {
|
||||
t.Fatalf("created=%d err=%v", s.created, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type deployRoundTrip func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f deployRoundTrip) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestDeployApplyConfirmation(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, response, confirmation, status string
|
||||
dispatch bool
|
||||
stale bool
|
||||
wantApplied bool
|
||||
}{
|
||||
{name: "dispatch disabled", confirmation: "unconfirmed", status: StatusSucceeded},
|
||||
{name: "bare success", dispatch: true, response: `{"ok":true}`, confirmation: "unconfirmed", status: StatusSucceeded},
|
||||
{name: "matching ACK", dispatch: true, response: "matching", confirmation: "agent_reported", status: StatusSucceeded, wantApplied: true},
|
||||
{name: "mismatched ACK", dispatch: true, response: `{"ok":true,"applied_revision_id":"other"}`, confirmation: "unconfirmed", status: StatusFailed},
|
||||
{name: "stale desired", dispatch: true, response: "matching", confirmation: "unconfirmed", status: StatusFailed, stale: true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||
if tc.dispatch {
|
||||
// A remote-only job must not touch the control plane BIRD tree.
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", t.TempDir())
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||
} else {
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||
}
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, module, previous, speakerID := m.DemoIDs()
|
||||
if err := m.SetLastAppliedRevision(tenant, speakerID, previous); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revision := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newer := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(context.Background(), newer, tenant, module, &revision, "newer-hash", map[string]string{"bird.conf": "# newer\n"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||
if _, err := m.UpdateSpeaker(tenant, speakerID, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" || r.Header.Get("Authorization") != "Bearer secret" {
|
||||
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
var body map[string]string
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil || len(body) != 1 || body["revision_id"] != revision {
|
||||
t.Errorf("legacy sync body: %#v, %v", body, err)
|
||||
}
|
||||
if tc.stale {
|
||||
if err := m.PublishRevisionForSpeaker(speakerID, newer); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
response := tc.response
|
||||
if response == "matching" {
|
||||
response = `{"ok":true,"applied_revision_id":"` + revision + `"}`
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision, "speaker_id": speakerID}}
|
||||
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||
if j.Status != tc.status {
|
||||
t.Fatalf("job status = %s, want %s; error=%v", j.Status, tc.status, j.Error)
|
||||
}
|
||||
sp, err := m.GetSpeaker(tenant, speakerID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := previous
|
||||
if tc.wantApplied {
|
||||
want = revision
|
||||
}
|
||||
if sp.LastAppliedRevisionID == nil || *sp.LastAppliedRevisionID != want {
|
||||
t.Fatalf("applied = %v, want %s", sp.LastAppliedRevisionID, want)
|
||||
}
|
||||
published, _, err := m.LatestPublishedRevision(speakerID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantPublished := revision
|
||||
if tc.stale {
|
||||
wantPublished = newer
|
||||
}
|
||||
if published != wantPublished {
|
||||
t.Fatalf("published = %s, want %s", published, wantPublished)
|
||||
}
|
||||
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||
if len(results) != 1 || results[0].Confirmation != tc.confirmation {
|
||||
t.Fatalf("results = %+v", results)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployApplyPartialResult(t *testing.T) {
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, module, previous, first := m.DemoIDs()
|
||||
revision := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
goodMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "good.example.test", AgentSecret: "secret"})
|
||||
if _, err := m.UpdateSpeaker(tenant, first, &store.SpeakerPatch{MetaJSON: &goodMeta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
badMeta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "bad.example.test", AgentSecret: "secret"})
|
||||
second, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: badMeta})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||
response := `{"ok":true,"applied_revision_id":"` + revision + `"}`
|
||||
if strings.HasPrefix(r.URL.Host, "bad.") {
|
||||
response = `{"ok":false,"error":"apply failed"}`
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
j := &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
|
||||
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||
if j.Status != StatusFailed {
|
||||
t.Fatalf("job status = %s, want failed", j.Status)
|
||||
}
|
||||
summary := j.Meta["apply_summary"].(map[string]any)
|
||||
if summary["applied_count"] != 1 || summary["failed_count"] != 1 || summary["published_count"] != 2 {
|
||||
t.Fatalf("partial summary = %#v", summary)
|
||||
}
|
||||
firstSp, _ := m.GetSpeaker(tenant, first)
|
||||
secondSp, _ := m.GetSpeaker(tenant, second.ID)
|
||||
if firstSp.LastAppliedRevisionID == nil || *firstSp.LastAppliedRevisionID != revision || secondSp.LastAppliedRevisionID != nil {
|
||||
t.Fatalf("partial applied state: first=%v second=%v", firstSp.LastAppliedRevisionID, secondSp.LastAppliedRevisionID)
|
||||
}
|
||||
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||
got := map[string]bool{}
|
||||
for _, res := range results {
|
||||
got[res.SpeakerID] = true
|
||||
}
|
||||
if len(results) != 2 || !got[first] || !got[second.ID] {
|
||||
t.Fatalf("partial results=%+v", results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeployApplyCancellationStopsFurtherPublication(t *testing.T) {
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, module, previous, seeded := m.DemoIDs()
|
||||
if err := m.DeleteSpeaker(tenant, seeded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
revision := uuid.NewString()
|
||||
if err := m.CreateRenderRevision(context.Background(), revision, tenant, module, &previous, "new-hash", map[string]string{"bird.conf": "# new\n"}, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||
speakerCount := speakerWakeLimit + 2
|
||||
for range speakerCount {
|
||||
if _, err := m.CreateSpeaker(tenant, &store.Speaker{Role: "replica", MetaJSON: meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
speakers := m.ListSpeakersForTenant(tenant)
|
||||
if len(speakers) != speakerCount {
|
||||
t.Fatalf("speakers=%d, want %d", len(speakers), speakerCount)
|
||||
}
|
||||
var j *Job
|
||||
var calls atomic.Int32
|
||||
client := &http.Client{Transport: deployRoundTrip(func(r *http.Request) (*http.Response, error) {
|
||||
// Cancel only once the in-flight wake limit is busy, so later speakers stay unpublished.
|
||||
if calls.Add(1) == int32(speakerWakeLimit) {
|
||||
j.RequestCancel()
|
||||
}
|
||||
<-r.Context().Done()
|
||||
return nil, r.Context().Err()
|
||||
})}
|
||||
j = &Job{TenantID: tenant, Kind: KindDeployApply, Status: StatusQueued, Meta: map[string]any{"revision_id": revision}}
|
||||
(&Worker{Store: m, NodeHTTPClient: client}).Process(j)
|
||||
if j.Status != StatusCancelled || int(calls.Load()) != speakerWakeLimit {
|
||||
t.Fatalf("cancelled job status=%s dispatch calls=%d", j.Status, calls.Load())
|
||||
}
|
||||
results := j.Meta["node_dispatch"].(map[string]any)["results"].([]nodedispatch.Result)
|
||||
gotResult := map[string]bool{}
|
||||
for _, res := range results {
|
||||
gotResult[res.SpeakerID] = res.Status == "cancelled"
|
||||
}
|
||||
if len(results) != speakerWakeLimit {
|
||||
t.Fatalf("cancelled results=%+v", results)
|
||||
}
|
||||
for i, sp := range speakers {
|
||||
got, _, err := m.LatestPublishedRevision(sp.ID)
|
||||
if i < speakerWakeLimit {
|
||||
if err != nil || got != revision || !gotResult[sp.ID] {
|
||||
t.Fatalf("speaker %s published=%s err=%v result=%v", sp.ID, got, err, gotResult[sp.ID])
|
||||
}
|
||||
if sp.LastAppliedRevisionID != nil {
|
||||
t.Fatalf("cancelled speaker %s marked applied", sp.ID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == nil {
|
||||
t.Fatalf("speaker %s published %s after cancellation", sp.ID, got)
|
||||
}
|
||||
}
|
||||
if j.Meta["apply_summary"].(map[string]any)["published_count"] != speakerWakeLimit {
|
||||
t.Fatalf("cancelled summary = %#v", j.Meta["apply_summary"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/logging"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func deploymentFenceForJob(j *Job) store.JobFence {
|
||||
generation, _ := strconv.ParseInt(j.metaString("reconcile_generation"), 10, 64)
|
||||
round, _ := strconv.Atoi(j.metaString("reconcile_round"))
|
||||
return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken, ExpectedGeneration: generation, RetryRound: round}
|
||||
}
|
||||
|
||||
func beginDeployment(ctx context.Context, st store.Deployments, f store.JobFence, speaker, revision string) (store.DeploymentSpeaker, error) {
|
||||
for {
|
||||
d, err := st.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if !errors.Is(err, store.ErrDeploymentBusy) {
|
||||
return d, err
|
||||
}
|
||||
timer := time.NewTimer(time.Second)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
timer.Stop()
|
||||
return store.DeploymentSpeaker{}, ctx.Err()
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ReconcileDeployments schedules at most three recovery rounds for current desired
|
||||
// generations. Stable keys coalesce API replicas; the worker CAS checks generation
|
||||
// again before publishing, and preserves the original desired request sequence.
|
||||
func ReconcileDeployments(ctx context.Context, r *Registry, st store.Deployments) error {
|
||||
rows, err := st.ListPendingDeployments(ctx, 100)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range rows {
|
||||
key := fmt.Sprintf("deployment_reconcile:%s:%s:%d", d.JobID, d.SpeakerID, d.RetryRound+1)
|
||||
_, _, err = r.EnqueueContext(ctx, d.TenantID, KindDeployApply, &key, nil, map[string]any{"revision_id": d.RevisionID, "speaker_id": d.SpeakerID, "reconcile_generation": strconv.FormatInt(d.Generation, 10), "reconcile_round": strconv.Itoa(d.RetryRound + 1), "source_deployment_job_id": d.JobID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// StartDeploymentReconciliationLoop belongs to the API server lifecycle. Static
|
||||
// ingest/deploy/scheduler processes must not start this job-producing loop.
|
||||
func StartDeploymentReconciliationLoop(ctx context.Context, r *Registry, st store.Backend) {
|
||||
if r == nil || r.queue == nil || st == nil {
|
||||
return
|
||||
}
|
||||
r.durableMu.Lock()
|
||||
if r.durableStop {
|
||||
r.durableMu.Unlock()
|
||||
return
|
||||
}
|
||||
r.durableLoopWG.Add(1)
|
||||
r.durableMu.Unlock()
|
||||
go func() {
|
||||
defer r.durableLoopWG.Done()
|
||||
ticker := time.NewTicker(10 * time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
r.durableMu.Lock()
|
||||
stopped := r.durableStop
|
||||
r.durableMu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
qctx, cancel := context.WithTimeout(ctx, 40*time.Second)
|
||||
err := ObserveDeployments(qctx, st, nil)
|
||||
if err == nil {
|
||||
err = ReconcileDeployments(qctx, r, st)
|
||||
}
|
||||
cancel()
|
||||
if err != nil && ctx.Err() == nil {
|
||||
logging.With("component", "jobs.deployment_reconcile").Warn("reconciliation failed", "err", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// ObserveDeployments samples compatible health before reconciling drift. The CP
|
||||
// receipt time and optional agent source time are stored separately; missing
|
||||
// legacy revision fields never manufacture an applied/verified confirmation.
|
||||
func ObserveDeployments(ctx context.Context, st store.Backend, client *http.Client) error {
|
||||
rows, err := st.ListDeploymentObservations(ctx, 10)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range rows {
|
||||
sp, e := st.GetSpeaker(d.TenantID, d.SpeakerID)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
observation := store.DeploymentObservation{Error: "remote health unavailable for this speaker"}
|
||||
if nodedispatch.Enabled() && store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON)) {
|
||||
hctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
h := nodedispatch.FetchAgentHealth(hctx, sp, nodedispatch.Options{HTTPClient: client, Timeout: 3 * time.Second})
|
||||
cancel()
|
||||
observation = store.DeploymentObservation{RevisionID: h.LastAppliedRevisionID, SourceAt: h.LastSyncAt, Error: h.Error}
|
||||
if h.OK && observation.RevisionID == "" {
|
||||
observation.Error = "legacy health did not report an applied revision"
|
||||
}
|
||||
}
|
||||
if err = st.ObserveDeployment(ctx, d, observation); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) {
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
r := NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claim := func(owner string) *Job {
|
||||
t.Helper()
|
||||
rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute)
|
||||
if e != nil || len(rows) != 1 {
|
||||
t.Fatalf("claim: %+v %v", rows, e)
|
||||
}
|
||||
return jobFromRecord(rows[0])
|
||||
}
|
||||
first := claim("initial")
|
||||
if first.ID != initial.ID {
|
||||
t.Fatal("wrong initial job")
|
||||
}
|
||||
(&Worker{Store: m}).Process(first)
|
||||
if err = m.FinishJob(ctx, recordFromJob(first)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Enqueue an operator request before the synthetic recovery request. The latter
|
||||
// must retain the earlier desired order despite its own larger queue sequence.
|
||||
operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for range 2 {
|
||||
if err = ReconcileDeployments(ctx, r, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100)
|
||||
if err != nil || len(all) != 3 {
|
||||
t.Fatalf("reconcile dedupe: %d %v", len(all), err)
|
||||
}
|
||||
newer := claim("operator")
|
||||
if newer.ID != operator.ID {
|
||||
t.Fatal("wrong operator job")
|
||||
}
|
||||
recovery := claim("recovery")
|
||||
f := deploymentFenceForJob(recovery)
|
||||
d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if d.RequestSequence != first.RequestSequence {
|
||||
t.Fatal("recovery lost original desired order")
|
||||
}
|
||||
if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision)
|
||||
if err != nil {
|
||||
t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err)
|
||||
}
|
||||
if newD.RequestSequence != newer.RequestSequence {
|
||||
t.Fatal("operator desired sequence changed")
|
||||
}
|
||||
if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil {
|
||||
t.Fatal("stale recovery generation reused")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) {
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, revision, speaker := m.DemoIDs()
|
||||
ctx := context.Background()
|
||||
r := NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
_, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
j := jobFromRecord(rows[0])
|
||||
(&Worker{Store: m}).Process(j)
|
||||
if err = m.FinishJob(ctx, recordFromJob(j)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
|
||||
if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
|
||||
response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}`
|
||||
client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) {
|
||||
if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" {
|
||||
t.Error("health must be authenticated GET")
|
||||
}
|
||||
return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
if err = ObserveDeployments(ctx, m, client); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
|
||||
if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" {
|
||||
t.Fatalf("legacy health confirmation: %+v %v", progress, err)
|
||||
}
|
||||
response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}`
|
||||
if err = ObserveDeployments(ctx, m, client); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID)
|
||||
if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" {
|
||||
t.Fatalf("late drift hidden: %+v", progress)
|
||||
}
|
||||
if err = ReconcileDeployments(ctx, r, m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100)
|
||||
if len(jobs) != 1 {
|
||||
t.Fatalf("drift recovery jobs: %d", len(jobs))
|
||||
}
|
||||
}
|
||||
+195
-115
@@ -2,136 +2,216 @@ package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"errors"
|
||||
"evobgp/internal/logging"
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/repository"
|
||||
"evobgp/internal/store"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Adopt registers a durable job_audit claim into the in-process registry and starts the worker.
|
||||
// Used after SKIP LOCKED claim so two API processes can share the queue without losing work on restart.
|
||||
func (r *Registry) Adopt(j *Job) bool {
|
||||
if r == nil || j == nil || j.ID == "" {
|
||||
return false
|
||||
}
|
||||
r.mu.Lock()
|
||||
if _, exists := r.byID[j.ID]; exists {
|
||||
r.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
if j.Status == "" {
|
||||
j.Status = StatusQueued
|
||||
}
|
||||
if j.Meta == nil {
|
||||
j.Meta = map[string]any{}
|
||||
}
|
||||
r.byID[j.ID] = j
|
||||
if j.IdempotencyKey != nil && *j.IdempotencyKey != "" {
|
||||
r.byIdempo[idempoKey{tenant: j.TenantID, key: *j.IdempotencyKey}] = j
|
||||
}
|
||||
if isRefreshKind(j.Kind) {
|
||||
r.inflightRefresh[j.TenantID]++
|
||||
}
|
||||
workerStart := r.workerStart
|
||||
r.mu.Unlock()
|
||||
const durableLease = time.Minute
|
||||
|
||||
if workerStart != nil {
|
||||
go func() {
|
||||
r.workerSem <- struct{}{}
|
||||
active := len(r.workerSem)
|
||||
capacity := cap(r.workerSem)
|
||||
observability.RecordJobQueueDepth(active, capacity)
|
||||
defer func() {
|
||||
<-r.workerSem
|
||||
observability.RecordJobQueueDepth(len(r.workerSem), capacity)
|
||||
}()
|
||||
workerStart(j)
|
||||
}()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// StartDurableQueueLoop periodically reclaims stale running rows and claims queued job_audit work (PG SKIP LOCKED).
|
||||
// No-op when audit is nil. Interval from EVOBGP_JOB_RECLAIM_INTERVAL (default 20s); stale from EVOBGP_JOB_STALE_AFTER (default 15m).
|
||||
func StartDurableQueueLoop(ctx context.Context, reg *Registry, audit *repository.JobAuditWriter) {
|
||||
if ctx == nil || reg == nil || audit == nil {
|
||||
// StartDurableQueueLoop claims work only after reserving local capacity.
|
||||
func StartDurableQueueLoop(ctx context.Context, r *Registry) {
|
||||
if r == nil || r.queue == nil {
|
||||
return
|
||||
}
|
||||
interval := 20 * time.Second
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_RECLAIM_INTERVAL")); s != "" {
|
||||
if d, err := time.ParseDuration(s); err == nil && d > 0 {
|
||||
interval = d
|
||||
}
|
||||
}
|
||||
staleAfter := 15 * time.Minute
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_STALE_AFTER")); s != "" {
|
||||
if d, err := time.ParseDuration(s); err == nil && d > 0 {
|
||||
staleAfter = d
|
||||
}
|
||||
}
|
||||
limit := 8
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_LIMIT"))); err == nil && n > 0 {
|
||||
limit = n
|
||||
}
|
||||
grace := 30 * time.Second
|
||||
if s := strings.TrimSpace(os.Getenv("EVOBGP_JOB_CLAIM_GRACE")); s != "" {
|
||||
if d, err := time.ParseDuration(s); err == nil && d >= 0 {
|
||||
grace = d
|
||||
}
|
||||
}
|
||||
log := logging.With("component", "jobs.durable")
|
||||
run := func() {
|
||||
cctx, cancel := context.WithTimeout(ctx, 15*time.Second)
|
||||
defer cancel()
|
||||
n, err := audit.ReclaimStaleRunning(cctx, staleAfter)
|
||||
if err != nil {
|
||||
log.Warn("reclaim stale running failed", "err", err)
|
||||
} else if n > 0 {
|
||||
log.Info("reclaimed stale running jobs", "count", n)
|
||||
}
|
||||
claimed, err := audit.ClaimQueued(cctx, limit, grace)
|
||||
if err != nil {
|
||||
log.Warn("claim queued failed", "err", err)
|
||||
return
|
||||
}
|
||||
for i := range claimed {
|
||||
c := claimed[i]
|
||||
j := &Job{
|
||||
ID: c.ID,
|
||||
TenantID: c.TenantID,
|
||||
Kind: c.Kind,
|
||||
Status: StatusQueued,
|
||||
IdempotencyKey: c.IdempotencyKey,
|
||||
ModuleID: c.ModuleID,
|
||||
CreatedAt: c.CreatedAt,
|
||||
Meta: c.Meta,
|
||||
}
|
||||
if !reg.Adopt(j) {
|
||||
// Already local — leave DB running; local worker owns it.
|
||||
continue
|
||||
}
|
||||
log.Info("adopted durable job", "job_id", j.ID, "kind", j.Kind, "tenant_id", j.TenantID)
|
||||
}
|
||||
counts, err := audit.CountByStatus(cctx)
|
||||
if err == nil {
|
||||
observability.RecordJobAuditDepth(counts)
|
||||
}
|
||||
r.durableMu.Lock()
|
||||
if r.durableStop {
|
||||
r.durableMu.Unlock()
|
||||
return
|
||||
}
|
||||
r.durableLoopWG.Add(2)
|
||||
r.durableMu.Unlock()
|
||||
go r.reapExpiredJobsLoop(ctx)
|
||||
go func() {
|
||||
run()
|
||||
t := time.NewTicker(interval)
|
||||
defer t.Stop()
|
||||
defer r.durableLoopWG.Done()
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
r.durableMu.Lock()
|
||||
stopped := r.durableStop
|
||||
r.durableMu.Unlock()
|
||||
if stopped {
|
||||
return
|
||||
}
|
||||
metricCtx, metricCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
if counts, err := r.queue.CountJobsByStatus(metricCtx); err == nil {
|
||||
observability.RecordJobAuditDepth(counts)
|
||||
}
|
||||
metricCancel()
|
||||
free := 0
|
||||
reserve:
|
||||
for free < cap(r.workerSem) {
|
||||
select {
|
||||
case r.workerSem <- struct{}{}:
|
||||
free++
|
||||
default:
|
||||
break reserve
|
||||
}
|
||||
}
|
||||
if free > 0 {
|
||||
qctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
claimed, err := r.queue.ClaimJobs(qctx, r.ownerID, free, durableLease)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Warn("claim failed", "err", err)
|
||||
}
|
||||
for i := len(claimed); i < free; i++ {
|
||||
<-r.workerSem
|
||||
}
|
||||
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
|
||||
for _, c := range claimed {
|
||||
r.durableMu.Lock()
|
||||
stopped = r.durableStop
|
||||
if stopped || ctx.Err() != nil {
|
||||
r.durableMu.Unlock()
|
||||
releaseCtx, releaseCancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
if err := r.queue.ReleaseJob(releaseCtx, c.ID, c.OwnerID, c.FencingToken); err != nil {
|
||||
log.Warn("release during shutdown failed", "job_id", c.ID, "err", err)
|
||||
}
|
||||
releaseCancel()
|
||||
<-r.workerSem
|
||||
continue
|
||||
}
|
||||
j := jobFromRecord(c)
|
||||
r.mu.Lock()
|
||||
r.byID[j.ID] = j
|
||||
if isRefreshKind(j.Kind) {
|
||||
r.inflightRefresh[j.TenantID]++
|
||||
}
|
||||
r.mu.Unlock()
|
||||
r.durableWG.Add(1)
|
||||
go r.runClaimed(j)
|
||||
r.durableMu.Unlock()
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
run()
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// reapExpiredJobsLoop runs the job_audit reaper every 15s, not on each claim.
|
||||
func (r *Registry) reapExpiredJobsLoop(ctx context.Context) {
|
||||
defer r.durableLoopWG.Done()
|
||||
r.reapExpiredJobs(ctx)
|
||||
reap := time.NewTicker(15 * time.Second)
|
||||
stop := time.NewTicker(time.Second)
|
||||
defer reap.Stop()
|
||||
defer stop.Stop()
|
||||
for {
|
||||
if r.durableStopped() || ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-stop.C:
|
||||
if r.durableStopped() {
|
||||
return
|
||||
}
|
||||
case <-reap.C:
|
||||
r.reapExpiredJobs(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) durableStopped() bool {
|
||||
if r == nil {
|
||||
return true
|
||||
}
|
||||
r.durableMu.Lock()
|
||||
defer r.durableMu.Unlock()
|
||||
return r.durableStop
|
||||
}
|
||||
|
||||
func (r *Registry) reapExpiredJobs(ctx context.Context) {
|
||||
if r == nil || r.queue == nil {
|
||||
return
|
||||
}
|
||||
cctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := r.queue.ReapExpiredJobs(cctx); err != nil && ctx.Err() == nil {
|
||||
logging.With("component", "jobs.durable").Warn("reap expired jobs", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Registry) runClaimed(j *Job) {
|
||||
log := logging.With("component", "jobs.durable")
|
||||
var leaseHealthy atomic.Bool
|
||||
leaseHealthy.Store(true)
|
||||
defer func() {
|
||||
r.mu.Lock()
|
||||
delete(r.byID, j.ID)
|
||||
r.mu.Unlock()
|
||||
<-r.workerSem
|
||||
observability.RecordJobQueueDepth(len(r.workerSem), cap(r.workerSem))
|
||||
r.durableWG.Done()
|
||||
}()
|
||||
stop := make(chan struct{})
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
t := time.NewTicker(durableLease / 4)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-t.C:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
requested, err := r.queue.HeartbeatJob(ctx, j.ID, j.OwnerID, j.FencingToken, durableLease)
|
||||
if err == nil {
|
||||
err = r.queue.CheckpointJob(ctx, recordFromJob(j))
|
||||
}
|
||||
cancel()
|
||||
if err != nil {
|
||||
leaseHealthy.Store(false)
|
||||
j.RequestCancel()
|
||||
log.Warn("heartbeat failed", "job_id", j.ID, "err", err)
|
||||
return
|
||||
}
|
||||
if requested {
|
||||
j.RequestCancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
r.workerStart(j)
|
||||
close(stop)
|
||||
<-done
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
r.durableMu.Lock()
|
||||
stopped := r.durableStop
|
||||
r.durableMu.Unlock()
|
||||
c := recordFromJob(j)
|
||||
if stopped && c.Status != StatusSucceeded && c.Status != StatusFailed {
|
||||
if err := r.queue.ReleaseJob(ctx, j.ID, j.OwnerID, j.FencingToken); err != nil && !errors.Is(err, store.ErrJobLeaseLost) {
|
||||
log.Warn("release failed", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !leaseHealthy.Load() {
|
||||
// Leave an uncertain lease for expiry/reclaim. This consumes an attempt,
|
||||
// whereas a deliberate graceful shutdown releases without using the budget.
|
||||
return
|
||||
}
|
||||
if c.Status != StatusSucceeded && c.Status != StatusFailed && c.Status != StatusCancelled {
|
||||
c.Status = StatusFailed
|
||||
msg := "worker returned without terminal status"
|
||||
c.Error = &msg
|
||||
}
|
||||
if err := r.queue.FinishJob(ctx, c); err != nil {
|
||||
log.Warn("finish failed; lease will retry", "job_id", j.ID, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type blockingClaimQueue struct {
|
||||
store.JobQueue
|
||||
entered chan int
|
||||
resume chan struct{}
|
||||
}
|
||||
|
||||
func (q *blockingClaimQueue) ClaimJobs(ctx context.Context, owner string, limit int, lease time.Duration) ([]store.JobRecord, error) {
|
||||
q.entered <- limit
|
||||
<-q.resume
|
||||
return q.JobQueue.ClaimJobs(context.Background(), owner, limit, lease)
|
||||
}
|
||||
|
||||
func TestDurableShutdownWaitsInflightClaimAndReleases(t *testing.T) {
|
||||
t.Setenv("EVOBGP_JOB_MAX_CONCURRENT", "1")
|
||||
m := store.NewMemory()
|
||||
queued, _, err := m.EnqueueJob(context.Background(), store.JobEnqueue{TenantID: "tenant", Kind: "test"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
q := &blockingClaimQueue{JobQueue: m, entered: make(chan int, 1), resume: make(chan struct{})}
|
||||
var runs atomic.Int32
|
||||
r := NewRegistry(func(j *Job) { runs.Add(1); j.Succeed() })
|
||||
r.UseDurableQueue(q)
|
||||
StartDurableQueueLoop(context.Background(), r)
|
||||
select {
|
||||
case limit := <-q.entered:
|
||||
if limit != cap(r.workerSem) {
|
||||
t.Fatalf("claim capacity=%d reserved=%d", limit, cap(r.workerSem))
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("no claim")
|
||||
}
|
||||
r.RequestCancelAll()
|
||||
drained := make(chan error, 1)
|
||||
go func() { drained <- r.Drain(context.Background()) }()
|
||||
select {
|
||||
case <-drained:
|
||||
t.Fatal("drained before in-flight claim returned")
|
||||
case <-time.After(20 * time.Millisecond):
|
||||
}
|
||||
close(q.resume)
|
||||
select {
|
||||
case err := <-drained:
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("shutdown did not drain")
|
||||
}
|
||||
if runs.Load() != 0 {
|
||||
t.Fatal("started worker after stop")
|
||||
}
|
||||
got, err := m.GetJob(context.Background(), "tenant", queued.ID)
|
||||
if err != nil || got.Status != "queued" || got.OwnerID != "" || got.Attempt != 0 {
|
||||
t.Fatalf("not released: %+v %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDurableSharedTerminalAndRequestReplay(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := store.NewMemory()
|
||||
key := "request"
|
||||
a := NewRegistry(nil)
|
||||
a.UseDurableQueue(m)
|
||||
j, _, err := a.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := m.ClaimJobs(ctx, "worker", 1, time.Minute)
|
||||
if err != nil || len(claimed) != 1 {
|
||||
t.Fatalf("claim %+v %v", claimed, err)
|
||||
}
|
||||
claimed[0].Status = StatusSucceeded
|
||||
claimed[0].Meta = map[string]any{"result": "ok"}
|
||||
if err = m.FinishJob(ctx, claimed[0]); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b := NewRegistry(nil)
|
||||
b.UseDurableQueue(m)
|
||||
got, err := b.GetContext(ctx, "tenant", j.ID)
|
||||
if err != nil || got.Status != StatusSucceeded {
|
||||
t.Fatalf("shared get %+v %v", got, err)
|
||||
}
|
||||
list, _, _, err := b.ListContext(ctx, "tenant", StatusSucceeded, "", "", 10)
|
||||
if err != nil || len(list) != 1 {
|
||||
t.Fatalf("shared list %+v %v", list, err)
|
||||
}
|
||||
replay, created, err := b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"nested": map[string]any{"input": "same"}})
|
||||
if err != nil || created || replay.ID != j.ID || replay.Status != StatusSucceeded {
|
||||
t.Fatalf("terminal replay %+v %v %v", replay, created, err)
|
||||
}
|
||||
if _, _, err = b.EnqueueContext(ctx, "tenant", "test", &key, nil, map[string]any{"input": "different"}); !errors.Is(err, store.ErrJobConflict) {
|
||||
t.Fatalf("changed payload accepted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParentRetryRecoversCommittedDeployChild(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
m := store.NewMemory()
|
||||
r := NewRegistry(nil)
|
||||
r.UseDurableQueue(m)
|
||||
parent, _, err := r.EnqueueContext(ctx, "tenant", KindRevisionRollback, nil, nil, map[string]any{"revision_id": "original"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := "deploy_from:" + parent.ID
|
||||
child, _, err := r.EnqueueContext(ctx, "tenant", KindDeployApply, &key, nil, map[string]any{"revision_id": "already-rendered"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
worker := &Worker{Store: m, Registry: r}
|
||||
worker.Process(parent)
|
||||
if parent.statusLocked() != StatusSucceeded || parent.metaString("revision_id") != "already-rendered" || parent.metaString("deploy_apply_job_id") != child.ID {
|
||||
t.Fatalf("did not recover child: %+v", parent.Snapshot())
|
||||
}
|
||||
items, _, _, err := m.ListJobs(ctx, "tenant", "", KindDeployApply, "", 10)
|
||||
if err != nil || len(items) != 1 {
|
||||
t.Fatalf("duplicate child: %+v %v", items, err)
|
||||
}
|
||||
}
|
||||
+102
-5
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"evobgp/internal/observability"
|
||||
"evobgp/internal/store"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
@@ -38,6 +39,11 @@ type Job struct {
|
||||
Error *string
|
||||
ProgressPct *int16
|
||||
Meta map[string]any
|
||||
OwnerID string
|
||||
FencingToken int64
|
||||
RequestSequence int64
|
||||
Attempt int
|
||||
requestHash string
|
||||
cancelRequested bool
|
||||
mu sync.Mutex
|
||||
}
|
||||
@@ -225,6 +231,30 @@ type Registry struct {
|
||||
// have been enqueued but not yet finalized in finishModuleRefreshSuccess. Used for deterministic
|
||||
// deploy coalescing under tenantRefreshMu (instead of polling job statuses).
|
||||
inflightRefresh map[string]int
|
||||
queue store.JobQueue
|
||||
ownerID string
|
||||
durableMu sync.Mutex
|
||||
durableWG sync.WaitGroup
|
||||
durableLoopWG sync.WaitGroup
|
||||
durableStop bool
|
||||
}
|
||||
|
||||
// UseDurableQueue makes the shared store authoritative for new work and job reads.
|
||||
func (r *Registry) UseDurableQueue(q store.JobQueue) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.queue = q
|
||||
r.ownerID = uuid.NewString()
|
||||
}
|
||||
|
||||
func jobFromRecord(c store.JobRecord) *Job {
|
||||
return &Job{ID: c.ID, TenantID: c.TenantID, Kind: c.Kind, Status: c.Status, IdempotencyKey: c.IdempotencyKey, ModuleID: c.ModuleID, Meta: c.Meta, CreatedAt: c.CreatedAt, StartedAt: c.StartedAt, FinishedAt: c.FinishedAt, Error: c.Error, OwnerID: c.OwnerID, FencingToken: c.FencingToken, RequestSequence: c.RequestSequence, Attempt: c.Attempt, cancelRequested: c.CancelRequested}
|
||||
}
|
||||
|
||||
func recordFromJob(j *Job) store.JobRecord {
|
||||
j.mu.Lock()
|
||||
defer j.mu.Unlock()
|
||||
return store.JobRecord{ID: j.ID, TenantID: j.TenantID, Kind: j.Kind, Status: j.Status, IdempotencyKey: j.IdempotencyKey, ModuleID: j.ModuleID, Meta: cloneMeta(j.Meta), CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, Error: j.Error, OwnerID: j.OwnerID, FencingToken: j.FencingToken, RequestSequence: j.RequestSequence, Attempt: j.Attempt, CancelRequested: j.cancelRequested}
|
||||
}
|
||||
|
||||
type idempoKey struct {
|
||||
@@ -310,6 +340,24 @@ func registryMaxConcurrentJobs() int {
|
||||
return 8
|
||||
}
|
||||
|
||||
func jobRegistryPendingLimit() int {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(os.Getenv("EVOBGP_JOB_MAX_PENDING"))); err == nil && n > 0 {
|
||||
return n
|
||||
}
|
||||
return 10000
|
||||
}
|
||||
|
||||
func (r *Registry) ActiveCountUnsafe() int {
|
||||
n := 0
|
||||
for _, j := range r.byID {
|
||||
st := j.statusLocked()
|
||||
if st == StatusQueued || st == StatusRunning {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// pruneTerminalIfOver удаляет самые старые завершённые джобы (succeeded/failed/cancelled), пока len(byID) > maxJobs.
|
||||
func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
||||
if r == nil || maxJobs <= 0 || len(r.byID) <= maxJobs {
|
||||
@@ -352,6 +400,33 @@ func (r *Registry) pruneTerminalIfOver(maxJobs int) {
|
||||
|
||||
// Enqueue creates a job or returns an existing one for the same idempotency key.
|
||||
func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||
return r.EnqueueContext(context.Background(), tenantID, kind, idempotencyKey, moduleID, meta)
|
||||
}
|
||||
|
||||
// EnqueueContext commits a durable job before returning it to an HTTP caller.
|
||||
func (r *Registry) EnqueueContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta map[string]any) (*Job, bool, error) {
|
||||
return r.EnqueueRequestContext(ctx, tenantID, kind, idempotencyKey, moduleID, meta, nil)
|
||||
}
|
||||
|
||||
// EnqueueRequestContext preserves idempotency of dynamically resolved selectors.
|
||||
func (r *Registry) EnqueueRequestContext(ctx context.Context, tenantID, kind string, idempotencyKey *string, moduleID *string, meta, request map[string]any) (*Job, bool, error) {
|
||||
if r.queue != nil {
|
||||
r.durableMu.Lock()
|
||||
stopped := r.durableStop
|
||||
r.durableMu.Unlock()
|
||||
if stopped {
|
||||
return nil, false, fmt.Errorf("jobs: shutting down")
|
||||
}
|
||||
c, created, err := r.queue.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return jobFromRecord(c), created, nil
|
||||
}
|
||||
hash, err := store.JobRequestHash(store.JobEnqueue{TenantID: tenantID, Kind: kind, IdempotencyKey: idempotencyKey, ModuleID: moduleID, Meta: meta, Request: request})
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
r.mu.Lock()
|
||||
maxJobs := registryMaxJobsFromEnv()
|
||||
r.pruneTerminalIfOver(maxJobs)
|
||||
@@ -359,14 +434,17 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||
k := idempoKey{tenant: tenantID, key: *idempotencyKey}
|
||||
if existing, ok := r.byIdempo[k]; ok {
|
||||
st := existing.statusLocked()
|
||||
if st == StatusQueued || st == StatusRunning {
|
||||
r.mu.Unlock()
|
||||
return existing, false, nil
|
||||
r.mu.Unlock()
|
||||
if existing.requestHash != hash {
|
||||
return nil, false, store.ErrJobConflict
|
||||
}
|
||||
delete(r.byIdempo, k)
|
||||
return existing, false, nil
|
||||
}
|
||||
}
|
||||
if r.ActiveCountUnsafe() >= jobRegistryPendingLimit() {
|
||||
r.mu.Unlock()
|
||||
return nil, false, store.ErrJobQueueFull
|
||||
}
|
||||
|
||||
j := &Job{
|
||||
ID: uuid.NewString(),
|
||||
@@ -377,6 +455,7 @@ func (r *Registry) Enqueue(tenantID, kind string, idempotencyKey *string, module
|
||||
ModuleID: moduleID,
|
||||
CreatedAt: time.Now().UTC(),
|
||||
Meta: cloneMeta(meta),
|
||||
requestHash: hash,
|
||||
}
|
||||
if idempotencyKey != nil && *idempotencyKey != "" {
|
||||
r.byIdempo[idempoKey{tenant: tenantID, key: *idempotencyKey}] = j
|
||||
@@ -571,6 +650,11 @@ func (r *Registry) RequestCancelAll() int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
if r.queue != nil {
|
||||
r.durableMu.Lock()
|
||||
r.durableStop = true
|
||||
r.durableMu.Unlock()
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
n := 0
|
||||
@@ -585,6 +669,9 @@ func (r *Registry) RequestCancelAll() int {
|
||||
j.RequestCancel()
|
||||
n++
|
||||
}
|
||||
if r.queue != nil && len(r.workerSem) > n {
|
||||
n = len(r.workerSem)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -614,6 +701,16 @@ func (r *Registry) Drain(ctx context.Context) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
if r.queue != nil {
|
||||
done := make(chan struct{})
|
||||
go func() { r.durableLoopWG.Wait(); r.durableWG.Wait(); close(done) }()
|
||||
select {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
ticker := time.NewTicker(50 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
// FindIdempotentContext looks up a retained logical operation without inserting.
|
||||
func (r *Registry) FindIdempotentContext(ctx context.Context, tenant, key string) (*Job, error) {
|
||||
if r.queue != nil {
|
||||
c, err := r.queue.FindJobByIdempotencyKey(ctx, tenant, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
j := jobFromRecord(c)
|
||||
j.requestHash = c.RequestHash
|
||||
return j, nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
defer r.mu.RUnlock()
|
||||
j := r.byIdempo[idempoKey{tenant: tenant, key: key}]
|
||||
if j == nil {
|
||||
return nil, store.ErrJobNotFound
|
||||
}
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// ReplayContext validates an immutable request before expensive live resolution.
|
||||
func (r *Registry) ReplayContext(ctx context.Context, in store.JobEnqueue) (*Job, bool, error) {
|
||||
if in.IdempotencyKey == nil || *in.IdempotencyKey == "" {
|
||||
return nil, false, nil
|
||||
}
|
||||
j, err := r.FindIdempotentContext(ctx, in.TenantID, *in.IdempotencyKey)
|
||||
if errors.Is(err, store.ErrJobNotFound) {
|
||||
return nil, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
h, err := store.JobRequestHash(in)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if j.requestHash != h {
|
||||
return nil, false, store.ErrJobConflict
|
||||
}
|
||||
return j, true, nil
|
||||
}
|
||||
|
||||
// GetContext reads a job from the shared store when durability is enabled.
|
||||
func (r *Registry) GetContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||
if r.queue == nil {
|
||||
return r.Get(tenant, id)
|
||||
}
|
||||
c, err := r.queue.GetJob(ctx, tenant, id)
|
||||
if errors.Is(err, store.ErrJobNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return jobFromRecord(c), nil
|
||||
}
|
||||
|
||||
// ListContext returns a bounded tenant page from the authoritative queue.
|
||||
func (r *Registry) ListContext(ctx context.Context, tenant, status, kind, cursor string, limit int) ([]*Job, string, bool, error) {
|
||||
if r.queue == nil {
|
||||
j, n, m := r.List(tenant, status, kind, cursor, limit)
|
||||
return j, n, m, nil
|
||||
}
|
||||
c, next, more, err := r.queue.ListJobs(ctx, tenant, status, kind, cursor, limit)
|
||||
if err != nil {
|
||||
return nil, "", false, err
|
||||
}
|
||||
out := make([]*Job, 0, len(c))
|
||||
for _, v := range c {
|
||||
out = append(out, jobFromRecord(v))
|
||||
}
|
||||
return out, next, more, nil
|
||||
}
|
||||
|
||||
// RequestCancelContext persists a cancel request visible to any worker replica.
|
||||
func (r *Registry) RequestCancelContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||
if r.queue == nil {
|
||||
return r.RequestCancel(tenant, id)
|
||||
}
|
||||
c, err := r.queue.CancelJob(ctx, tenant, id)
|
||||
if errors.Is(err, store.ErrJobNotFound) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.mu.RLock()
|
||||
local := r.byID[id]
|
||||
r.mu.RUnlock()
|
||||
if local != nil {
|
||||
local.RequestCancel()
|
||||
}
|
||||
return jobFromRecord(c), nil
|
||||
}
|
||||
|
||||
// RetryContext schedules an explicit retry of a failed shared operation.
|
||||
func (r *Registry) RetryContext(ctx context.Context, tenant, id string) (*Job, error) {
|
||||
if r.queue == nil {
|
||||
return nil, store.ErrJobConflict
|
||||
}
|
||||
c, err := r.queue.RetryJob(ctx, tenant, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return jobFromRecord(c), nil
|
||||
}
|
||||
+590
-142
@@ -2,6 +2,7 @@ package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
@@ -66,13 +67,18 @@ const (
|
||||
|
||||
// Worker executes queued jobs against store.Backend (memory or SQL).
|
||||
type Worker struct {
|
||||
Store store.Backend
|
||||
PgPool *pgxpool.Pool
|
||||
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
|
||||
Store store.Backend
|
||||
PgPool *pgxpool.Pool
|
||||
HTTPClient *http.Client // optional; CDN refresh uses this (default 45s timeout).
|
||||
NodeHTTPClient *http.Client // optional; agent dispatch only, never shared with CDN transport.
|
||||
// Registry is set after BootstrapWorkers creates the job queue; used to chain deploy_apply after refresh/rollback.
|
||||
Registry *Registry
|
||||
// refreshGate serializes deploy_apply gating after module_refresh per tenant (see finishModuleRefreshSuccess).
|
||||
refreshGate sync.Map // map[string]*sync.Mutex
|
||||
// refreshGate decides which module refresh renders for a tenant.
|
||||
// The mutex covers only that decision; the render itself runs outside the lock.
|
||||
refreshGate sync.Map // map[string]*tenantRefreshGate
|
||||
// dispatchGate serializes publish and deployment state for one speaker in this process.
|
||||
// Agent wake runs outside the mutex.
|
||||
dispatchGate sync.Map // map[string]*sync.Mutex
|
||||
}
|
||||
|
||||
type revisionLogEntry struct {
|
||||
@@ -116,6 +122,23 @@ func (w *Worker) Process(j *Job) {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
// A committed child is the durable checkpoint for a parent that crashed
|
||||
// before its own terminal write. Reuse it before rendering another revision.
|
||||
if w.Registry != nil && (isRefreshKind(j.Kind) || j.Kind == KindRevisionRollback || j.Kind == KindPeerReconcile) {
|
||||
child, err := w.Registry.FindIdempotentContext(context.Background(), j.TenantID, "deploy_from:"+j.ID)
|
||||
if err == nil {
|
||||
j.mergeMeta(map[string]any{"revision_id": child.metaString("revision_id"), "deploy_apply_job_id": child.ID, "recovered_child": true})
|
||||
if isRefreshKind(j.Kind) {
|
||||
w.Registry.finalizeRefreshCoalesce(j.TenantID)
|
||||
}
|
||||
j.Succeed()
|
||||
return
|
||||
}
|
||||
if !errors.Is(err, store.ErrJobNotFound) {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
switch j.Kind {
|
||||
case KindModuleRefresh:
|
||||
@@ -185,50 +208,24 @@ func (w *Worker) runPeerReconcile(j *Job) {
|
||||
const peerJobTitle = "Обновление BGP пиров"
|
||||
j.mergeMeta(map[string]any{"job_title": peerJobTitle})
|
||||
|
||||
var revID string
|
||||
latest, _, _ := w.Store.ListRevisions(j.TenantID, "", "", 1)
|
||||
triggerModuleID, err := w.peerTriggerModuleID(j.TenantID, latest)
|
||||
if err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
if len(latest) == 0 {
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
rid, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
j.Fail(err.Error())
|
||||
// Peer edits bump render generation. Render from module snapshots so the
|
||||
// new revision includes current prefixes without paging the previous one.
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
revID, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
revID = rid
|
||||
} else {
|
||||
baseRevID := latest[0].ID
|
||||
rows := make([]store.PrefixRow, 0, 1024)
|
||||
cursor := ""
|
||||
for {
|
||||
page, next, more := w.Store.ListRevisionPrefixes(j.TenantID, baseRevID, cursor, 2000)
|
||||
rows = append(rows, page...)
|
||||
if !more || strings.TrimSpace(next) == "" {
|
||||
break
|
||||
}
|
||||
cursor = next
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
rid, err := pipeline.RenderTenantRevisionFromPrefixes(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID, rows)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
revID = rid
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.mergeMeta(map[string]any{"revision_id": revID})
|
||||
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, revID); err == nil {
|
||||
@@ -240,7 +237,10 @@ func (w *Worker) runPeerReconcile(j *Job) {
|
||||
} else {
|
||||
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
||||
}
|
||||
w.enqueueDeployAllSpeakers(j, j.TenantID, revID)
|
||||
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, revID); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
@@ -269,8 +269,23 @@ func (w *Worker) peerTriggerModuleID(tenantID string, latest []*store.Revision)
|
||||
return "", fmt.Errorf("missing module_id for peer reconcile")
|
||||
}
|
||||
|
||||
type tenantRefreshGate struct {
|
||||
mu sync.Mutex
|
||||
rendering bool
|
||||
dirty bool
|
||||
}
|
||||
|
||||
func (w *Worker) tenantRefreshGate(tenantID string) *tenantRefreshGate {
|
||||
v, _ := w.refreshGate.LoadOrStore(tenantID, &tenantRefreshGate{})
|
||||
return v.(*tenantRefreshGate)
|
||||
}
|
||||
|
||||
func (w *Worker) tenantRefreshMu(tenantID string) *sync.Mutex {
|
||||
v, _ := w.refreshGate.LoadOrStore(tenantID, &sync.Mutex{})
|
||||
return &w.tenantRefreshGate(tenantID).mu
|
||||
}
|
||||
|
||||
func (w *Worker) speakerDispatchMu(speakerID string) *sync.Mutex {
|
||||
v, _ := w.dispatchGate.LoadOrStore(speakerID, &sync.Mutex{})
|
||||
return v.(*sync.Mutex)
|
||||
}
|
||||
|
||||
@@ -327,7 +342,8 @@ func (w *Worker) runTenantRefresh(j *Job) {
|
||||
|
||||
moduleIDs := moduleIDsFromJobMeta(j.metaCopy())
|
||||
if len(moduleIDs) == 0 {
|
||||
j.Fail("missing module_ids in job meta")
|
||||
j.mergeMeta(map[string]any{"modules_refreshed": 0})
|
||||
j.Succeed()
|
||||
return
|
||||
}
|
||||
ctx, cancel := j.workContext()
|
||||
@@ -382,20 +398,19 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
||||
j.Succeed()
|
||||
return
|
||||
}
|
||||
mu := w.tenantRefreshMu(j.TenantID)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
// Determine whether this is the last outstanding refresh for the tenant. The counter is
|
||||
// incremented in Enqueue (under r.mu) and decremented here, so the "last one" decision is
|
||||
// deterministic regardless of ingest timing — unlike the previous status-polling approach
|
||||
// (CountOtherActiveRefresh) which could race under -race.
|
||||
gate := w.tenantRefreshGate(j.TenantID)
|
||||
gate.mu.Lock()
|
||||
// The counter is incremented in Enqueue and decremented here, so the "last one"
|
||||
// decision does not depend on job status polling. Render runs after the unlock.
|
||||
isLastRefresh := true
|
||||
if w.Registry != nil {
|
||||
isLastRefresh = w.Registry.finalizeRefreshCoalesce(j.TenantID)
|
||||
}
|
||||
if !isLastRefresh {
|
||||
if !isLastRefresh || gate.rendering {
|
||||
if gate.rendering {
|
||||
gate.dirty = true
|
||||
}
|
||||
gate.mu.Unlock()
|
||||
j.mergeMeta(map[string]any{
|
||||
"deploy_apply_deferred": true,
|
||||
"deploy_apply_defer_reason": "parallel_module_refresh",
|
||||
@@ -403,16 +418,33 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
||||
j.Succeed()
|
||||
return
|
||||
}
|
||||
gate.rendering = true
|
||||
gate.mu.Unlock()
|
||||
defer func() {
|
||||
gate.mu.Lock()
|
||||
gate.rendering = false
|
||||
gate.mu.Unlock()
|
||||
}()
|
||||
|
||||
rev, err := pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
rev, err := w.renderTenantUntilQuiet(ctx, j, triggerModuleID)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
if errors.Is(err, errRenderSuperseded) {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil || j.IsCancelRequested() {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
if ctx.Err() != nil || j.IsCancelRequested() {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
j.mergeMeta(map[string]any{"revision_id": rev})
|
||||
if entries, total, err := w.buildRevisionLogEntries(j.TenantID, rev); err == nil {
|
||||
j.mergeMeta(map[string]any{
|
||||
@@ -423,50 +455,240 @@ func (w *Worker) finishModuleRefreshSuccess(j *Job, triggerModuleID string) {
|
||||
} else {
|
||||
j.mergeMeta(map[string]any{"log_build_error": err.Error()})
|
||||
}
|
||||
w.enqueueDeployAllSpeakers(j, j.TenantID, rev)
|
||||
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, rev); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
const (
|
||||
// renderQuietPassLimit stops a render loop that never observes a quiet input set.
|
||||
// Hitting the cap fails the job; the last revision must not be published.
|
||||
renderQuietPassLimit = 32
|
||||
// speakerWakeLimit is the maximum number of concurrent agent wakes in one deploy.
|
||||
speakerWakeLimit = 4
|
||||
)
|
||||
|
||||
// errRenderSuperseded marks a render that must not be deployed. A newer snapshot
|
||||
// landed, the job was cancelled while inputs were still changing, or the quiet
|
||||
// pass cap was exhausted.
|
||||
var errRenderSuperseded = errors.New("newer snapshot still pending")
|
||||
|
||||
// renderTenantUntilQuiet builds a revision outside the tenant gate.
|
||||
// It keeps rendering while another ingest marks the gate dirty or the snapshot
|
||||
// generation moves, and returns a revision only after a fresh check shows the
|
||||
// inputs are still the ones just rendered. Cancellation or the safety cap while
|
||||
// inputs are still changing fails the render instead of publishing a stale revision.
|
||||
func (w *Worker) renderTenantUntilQuiet(ctx context.Context, j *Job, triggerModuleID string) (string, error) {
|
||||
gate := w.tenantRefreshGate(j.TenantID)
|
||||
var rev string
|
||||
for pass := 0; pass < renderQuietPassLimit; pass++ {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
if takeRefreshDirty(gate) {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
return "", abort
|
||||
}
|
||||
before, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||
if err != nil {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
rev, err = pipeline.RenderTenantRevision(ctx, w.Store, w.httpClient(), j.TenantID, triggerModuleID)
|
||||
if err != nil {
|
||||
dirty := takeRefreshDirty(gate)
|
||||
if abort := renderJobAborted(ctx, j); abort != nil && dirty {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
return "", abort
|
||||
}
|
||||
if dirty {
|
||||
continue
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
after, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||
if err != nil {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
dirty := takeRefreshDirty(gate)
|
||||
if dirty || renderInputsChanged(before, after) {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
continue
|
||||
}
|
||||
confirm, err := w.captureRenderInputs(ctx, j.TenantID)
|
||||
if err != nil {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil && takeRefreshDirty(gate) {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
if takeRefreshDirty(gate) || renderInputsChanged(after, confirm) {
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
return "", abort
|
||||
}
|
||||
return rev, nil
|
||||
}
|
||||
if abort := renderJobAborted(ctx, j); abort != nil {
|
||||
return "", errRenderWhileDirty(abort)
|
||||
}
|
||||
return "", errRenderWhileDirty(nil)
|
||||
}
|
||||
|
||||
func errRenderWhileDirty(cause error) error {
|
||||
if cause == nil {
|
||||
return fmt.Errorf("jobs: tenant render did not quiesce after %d passes: %w", renderQuietPassLimit, errRenderSuperseded)
|
||||
}
|
||||
return fmt.Errorf("jobs: tenant render cancelled while a newer snapshot was still pending: %w", errRenderSuperseded)
|
||||
}
|
||||
|
||||
func renderJobAborted(ctx context.Context, j *Job) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if j != nil && j.IsCancelRequested() {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func takeRefreshDirty(gate *tenantRefreshGate) bool {
|
||||
if gate == nil {
|
||||
return false
|
||||
}
|
||||
gate.mu.Lock()
|
||||
defer gate.mu.Unlock()
|
||||
dirty := gate.dirty
|
||||
gate.dirty = false
|
||||
return dirty
|
||||
}
|
||||
|
||||
type renderInputView struct {
|
||||
generation int64
|
||||
snapshots []store.ModuleSnapshotUse
|
||||
}
|
||||
|
||||
func (w *Worker) captureRenderInputs(ctx context.Context, tenantID string) (renderInputView, error) {
|
||||
if w == nil || w.Store == nil {
|
||||
return renderInputView{}, fmt.Errorf("jobs: render store unavailable")
|
||||
}
|
||||
gen, err := w.Store.GetRenderGeneration(ctx, tenantID)
|
||||
if err != nil {
|
||||
if !errors.Is(err, store.ErrNotFound) {
|
||||
return renderInputView{}, fmt.Errorf("jobs: read render generation: %w", err)
|
||||
}
|
||||
gen = 0
|
||||
}
|
||||
headers, err := w.Store.ListModuleSnapshotHeaders(ctx, tenantID)
|
||||
if err != nil {
|
||||
return renderInputView{}, fmt.Errorf("jobs: read snapshot headers: %w", err)
|
||||
}
|
||||
return renderInputView{generation: gen, snapshots: headers}, nil
|
||||
}
|
||||
|
||||
func renderInputsChanged(before, after renderInputView) bool {
|
||||
if before.generation != after.generation {
|
||||
return true
|
||||
}
|
||||
return !sameSnapshotHeaders(before.snapshots, after.snapshots)
|
||||
}
|
||||
|
||||
func sameSnapshotHeaders(a, b []store.ModuleSnapshotUse) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
aa := append([]store.ModuleSnapshotUse(nil), a...)
|
||||
bb := append([]store.ModuleSnapshotUse(nil), b...)
|
||||
sort.Slice(aa, func(i, j int) bool { return aa[i].ModuleID < aa[j].ModuleID })
|
||||
sort.Slice(bb, func(i, j int) bool { return bb[i].ModuleID < bb[j].ModuleID })
|
||||
for i := range aa {
|
||||
if aa[i].ModuleID != bb[i].ModuleID || aa[i].SnapshotVersion != bb[i].SnapshotVersion || aa[i].InputHash != bb[i].InputHash {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// enqueueDeployAllSpeakers queues the same work as POST /v1/apply (all speakers, no speaker_id).
|
||||
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) {
|
||||
func (w *Worker) enqueueDeployAllSpeakers(j *Job, tenantID, revID string) error {
|
||||
if w == nil || w.Registry == nil {
|
||||
return
|
||||
return fmt.Errorf("jobs: deploy queue unavailable")
|
||||
}
|
||||
revID = strings.TrimSpace(revID)
|
||||
if revID == "" {
|
||||
return
|
||||
return fmt.Errorf("jobs: missing revision for deploy")
|
||||
}
|
||||
meta := map[string]any{"revision_id": revID}
|
||||
if tr := strings.TrimSpace(j.metaString("trigger")); tr != "" {
|
||||
meta["trigger"] = tr
|
||||
}
|
||||
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, nil, nil, meta)
|
||||
key := "deploy_from:" + j.ID
|
||||
applyJob, _, err := w.Registry.Enqueue(tenantID, KindDeployApply, &key, nil, meta)
|
||||
if err != nil {
|
||||
j.mergeMeta(map[string]any{"deploy_apply_enqueue_error": err.Error()})
|
||||
return
|
||||
return fmt.Errorf("jobs: enqueue deploy: %w", err)
|
||||
}
|
||||
if applyJob != nil {
|
||||
j.mergeMeta(map[string]any{"deploy_apply_job_id": applyJob.ID})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) runDeployApply(j *Job) {
|
||||
revID := strings.TrimSpace(j.metaString("revision_id"))
|
||||
spk := strings.TrimSpace(j.metaString("speaker_id"))
|
||||
hasSpeaker := spk != ""
|
||||
if revID == "" {
|
||||
j.Fail("missing revision_id in job meta")
|
||||
return
|
||||
}
|
||||
if j.OwnerID == "" {
|
||||
if _, developmentMemory := w.Store.(*store.Memory); !developmentMemory {
|
||||
j.Fail("deployment requires a current durable job lease")
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
ctx, cancel := j.workContext()
|
||||
defer cancel()
|
||||
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
|
||||
if activeDir != "" {
|
||||
revObj, err := w.Store.GetRevision(j.TenantID, revID)
|
||||
cancelled := func() bool { return ctx.Err() != nil || j.IsCancelRequested() }
|
||||
var speakers []*store.Speaker
|
||||
if spk != "" {
|
||||
sp, err := w.Store.GetSpeaker(j.TenantID, spk)
|
||||
if err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
speakers = []*store.Speaker{sp}
|
||||
} else {
|
||||
speakers = w.Store.ListSpeakersForTenant(j.TenantID)
|
||||
}
|
||||
activeDir := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_ACTIVE_DIR"))
|
||||
localAttempted := false
|
||||
var localErr error
|
||||
applyLocal := func() error {
|
||||
if localAttempted {
|
||||
return localErr
|
||||
}
|
||||
localAttempted = true
|
||||
revObj, err := w.Store.GetRevision(j.TenantID, revID)
|
||||
if err != nil {
|
||||
localErr = err
|
||||
return err
|
||||
}
|
||||
staging := strings.TrimSpace(os.Getenv("EVOBGP_BIRD_STAGING_DIR"))
|
||||
if staging == "" {
|
||||
staging = os.TempDir() + "/evobgp-bird-staging"
|
||||
@@ -480,99 +702,322 @@ func (w *Worker) runDeployApply(j *Job) {
|
||||
}
|
||||
ctl := &birdfmt.BirdCtl{Bird: cfg.BirdBin, Birdc: cfg.BirdcBin, Socket: cfg.Socket}
|
||||
if err := birddeploy.ApplyRevision(ctx, ctl, revObj, cfg); err != nil {
|
||||
if ctx.Err() != nil {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
applied := make([]string, 0, 8)
|
||||
var dispatchResults []nodedispatch.Result
|
||||
applyOne := func(speakerID string) error {
|
||||
if err := w.Store.SetLastAppliedRevision(j.TenantID, speakerID, revID); err != nil {
|
||||
localErr = err
|
||||
return err
|
||||
}
|
||||
// Replica / node pulls use LatestPublishedRevision; keep pointer in sync with successful deploy.
|
||||
if err := w.Store.PublishRevisionForSpeaker(speakerID, revID); err != nil {
|
||||
return err
|
||||
}
|
||||
applied = append(applied, speakerID)
|
||||
return nil
|
||||
}
|
||||
dispatchSpeaker := func(sp *store.Speaker) {
|
||||
if !nodedispatch.Enabled() || sp == nil {
|
||||
return
|
||||
tally := &deployTally{applied: make([]string, 0, len(speakers))}
|
||||
slots := make(chan struct{}, speakerWakeLimit)
|
||||
var wakeGroup sync.WaitGroup
|
||||
acquireWake := func() bool {
|
||||
select {
|
||||
case slots <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
}
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
if !store.SpeakerNeedsRemoteDispatch(sp.Role, meta) {
|
||||
return
|
||||
}
|
||||
ctx2, cancel := context.WithTimeout(ctx, 35*time.Second)
|
||||
defer cancel()
|
||||
res := nodedispatch.WakeSpeaker(ctx2, sp, nodedispatch.Options{RevisionID: revID})
|
||||
dispatchResults = append(dispatchResults, res)
|
||||
patch := store.SpeakerMeta{
|
||||
LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
LastDispatchStatus: res.Status,
|
||||
}
|
||||
if res.Error != "" {
|
||||
patch.LastDispatchError = res.Error
|
||||
patch.SyncStatus = "error"
|
||||
} else if res.Status == "ok" {
|
||||
patch.LastDispatchError = ""
|
||||
patch.SyncStatus = "synced"
|
||||
}
|
||||
merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
|
||||
_, _ = w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged})
|
||||
}
|
||||
if hasSpeaker && spk != "" {
|
||||
if err := applyOne(spk); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
if sp, err := w.Store.GetSpeaker(j.TenantID, spk); err == nil {
|
||||
dispatchSpeaker(sp)
|
||||
}
|
||||
if len(dispatchResults) > 0 {
|
||||
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
|
||||
"revision_id": revID,
|
||||
"results": dispatchResults,
|
||||
}})
|
||||
}
|
||||
mergeBirdPostApplyMeta(j)
|
||||
j.Succeed()
|
||||
return
|
||||
}
|
||||
speakers := w.Store.ListSpeakersForTenant(j.TenantID)
|
||||
for _, sp := range speakers {
|
||||
if err := applyOne(sp.ID); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
if cancelled() {
|
||||
<-slots
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
for _, sp := range speakers {
|
||||
dispatchSpeaker(sp)
|
||||
if cancelled() {
|
||||
break
|
||||
}
|
||||
if sp == nil {
|
||||
continue
|
||||
}
|
||||
held := false
|
||||
if speakerRemoteWake(sp) {
|
||||
if !acquireWake() {
|
||||
break
|
||||
}
|
||||
held = true
|
||||
}
|
||||
staged := w.stageSpeakerDeploy(ctx, j, sp, revID, activeDir, cancelled, applyLocal)
|
||||
if staged.wake && !staged.stop {
|
||||
wakeGroup.Add(1)
|
||||
go func(sp *store.Speaker, deployment store.DeploymentSpeaker) {
|
||||
defer wakeGroup.Done()
|
||||
defer func() { <-slots }()
|
||||
wakeCtx, cancelWake := context.WithTimeout(ctx, 35*time.Second)
|
||||
res := nodedispatch.WakeSpeaker(wakeCtx, sp, nodedispatch.Options{RevisionID: revID, HTTPClient: w.NodeHTTPClient, MaxRetries: 1})
|
||||
cancelWake()
|
||||
out, include, failedN, unconfirmedN, appliedID := w.finishSpeakerWake(ctx, j, sp, revID, deployment, res, cancelled)
|
||||
tally.add(out, include, 0, failedN, unconfirmedN, appliedID)
|
||||
}(sp, staged.deployment)
|
||||
} else if held {
|
||||
<-slots
|
||||
}
|
||||
tally.add(staged.result, staged.hasResult, staged.published, staged.failed, staged.unconfirmed, staged.appliedID)
|
||||
if staged.stop {
|
||||
break
|
||||
}
|
||||
}
|
||||
wakeGroup.Wait()
|
||||
results, applied, published, failed, unconfirmed := tally.snapshot()
|
||||
j.mergeMeta(map[string]any{
|
||||
"apply_summary": map[string]any{
|
||||
"revision_id": revID,
|
||||
"speakers_count": len(applied),
|
||||
"speaker_ids": applied,
|
||||
"message": fmt.Sprintf("Ревизия %s применена на %d спикерах", shortID(revID), len(applied)),
|
||||
"revision_id": revID,
|
||||
"speakers_count": len(speakers),
|
||||
"published_count": published,
|
||||
"applied_count": len(applied),
|
||||
"unconfirmed_count": unconfirmed,
|
||||
"failed_count": failed,
|
||||
"speaker_ids": applied,
|
||||
"message": fmt.Sprintf("Ревизия %s: подтверждено %d, без подтверждения %d, ошибок %d", shortID(revID), len(applied), unconfirmed, failed),
|
||||
},
|
||||
"node_dispatch": map[string]any{"revision_id": revID, "results": results},
|
||||
})
|
||||
if len(dispatchResults) > 0 {
|
||||
j.mergeMeta(map[string]any{"node_dispatch": map[string]any{
|
||||
"revision_id": revID,
|
||||
"results": dispatchResults,
|
||||
}})
|
||||
if cancelled() {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
mergeBirdPostApplyMeta(j)
|
||||
if cancelled() {
|
||||
j.MarkCancelled()
|
||||
return
|
||||
}
|
||||
if failed > 0 {
|
||||
j.Fail(fmt.Sprintf("deploy delivery failed for %d of %d speakers", failed, len(speakers)))
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
type speakerStage struct {
|
||||
stop bool
|
||||
wake bool
|
||||
deployment store.DeploymentSpeaker
|
||||
result nodedispatch.Result
|
||||
hasResult bool
|
||||
published int
|
||||
failed int
|
||||
unconfirmed int
|
||||
appliedID string
|
||||
}
|
||||
|
||||
type deployTally struct {
|
||||
mu sync.Mutex
|
||||
results []nodedispatch.Result
|
||||
applied []string
|
||||
published int
|
||||
failed int
|
||||
unconfirmed int
|
||||
}
|
||||
|
||||
func (t *deployTally) add(res nodedispatch.Result, include bool, published, failed, unconfirmed int, appliedID string) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if include {
|
||||
t.results = append(t.results, res)
|
||||
}
|
||||
t.published += published
|
||||
t.failed += failed
|
||||
t.unconfirmed += unconfirmed
|
||||
if appliedID != "" {
|
||||
t.applied = append(t.applied, appliedID)
|
||||
}
|
||||
}
|
||||
|
||||
func (t *deployTally) snapshot() (results []nodedispatch.Result, applied []string, published, failed, unconfirmed int) {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
results = append([]nodedispatch.Result(nil), t.results...)
|
||||
applied = append([]string(nil), t.applied...)
|
||||
if applied == nil {
|
||||
applied = []string{}
|
||||
}
|
||||
sort.Slice(results, func(i, j int) bool { return results[i].SpeakerID < results[j].SpeakerID })
|
||||
sort.Strings(applied)
|
||||
return results, applied, t.published, t.failed, t.unconfirmed
|
||||
}
|
||||
|
||||
func speakerRemoteWake(sp *store.Speaker) bool {
|
||||
if sp == nil || !nodedispatch.Enabled() {
|
||||
return false
|
||||
}
|
||||
return store.SpeakerNeedsRemoteDispatch(sp.Role, store.ParseSpeakerMeta(sp.MetaJSON))
|
||||
}
|
||||
|
||||
func (w *Worker) bindDeploymentWriters(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker) (func(string, string, string) error, func(string) error) {
|
||||
fence := deploymentFenceForJob(j)
|
||||
record := func(state, confirmation, message string) error {
|
||||
if j.OwnerID == "" {
|
||||
return nil
|
||||
}
|
||||
return w.Store.RecordDeploymentSpeaker(ctx, fence, deployment, state, confirmation, message)
|
||||
}
|
||||
markApplied := func(confirmation string) error {
|
||||
if j.OwnerID == "" {
|
||||
return w.Store.SetLastAppliedRevision(j.TenantID, sp.ID, revID)
|
||||
}
|
||||
return record("applied", confirmation, "")
|
||||
}
|
||||
return record, markApplied
|
||||
}
|
||||
|
||||
// stageSpeakerDeploy publishes one speaker and records terminal states.
|
||||
// Remote wakes are left to the caller so the speaker lock is not held across the network.
|
||||
func (w *Worker) stageSpeakerDeploy(ctx context.Context, j *Job, sp *store.Speaker, revID, activeDir string, cancelled func() bool, applyLocal func() error) speakerStage {
|
||||
var staged speakerStage
|
||||
mu := w.speakerDispatchMu(sp.ID)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if cancelled() {
|
||||
staged.stop = true
|
||||
return staged
|
||||
}
|
||||
var deployment store.DeploymentSpeaker
|
||||
var publishErr error
|
||||
if j.OwnerID != "" {
|
||||
deployment, publishErr = beginDeployment(ctx, w.Store, deploymentFenceForJob(j), sp.ID, revID)
|
||||
} else {
|
||||
publishErr = w.Store.PublishRevisionForSpeaker(sp.ID, revID)
|
||||
}
|
||||
staged.deployment = deployment
|
||||
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
|
||||
if publishErr != nil {
|
||||
staged.failed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: publishErr.Error()}
|
||||
return staged
|
||||
}
|
||||
staged.published = 1
|
||||
if deployment.State == "applied" {
|
||||
staged.appliedID = sp.ID
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: deployment.Confirmation, AppliedRevisionID: revID}
|
||||
return staged
|
||||
}
|
||||
if cancelled() {
|
||||
staged.stop = true
|
||||
return staged
|
||||
}
|
||||
if !speakerRemoteWake(sp) {
|
||||
if activeDir != "" && strings.EqualFold(sp.Role, "master") {
|
||||
err := applyLocal()
|
||||
if cancelled() {
|
||||
staged.stop = true
|
||||
return staged
|
||||
}
|
||||
if err != nil {
|
||||
_ = record("failed", "unconfirmed", err.Error())
|
||||
staged.failed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||
return staged
|
||||
}
|
||||
if err := markApplied("local_applied"); err != nil {
|
||||
staged.failed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||
return staged
|
||||
}
|
||||
staged.appliedID = sp.ID
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "ok", Confirmation: "local_applied", AppliedRevisionID: revID}
|
||||
return staged
|
||||
}
|
||||
if err := record("unconfirmed", "unconfirmed", "dispatch disabled or speaker has no remote agent"); err != nil {
|
||||
staged.failed = 1
|
||||
}
|
||||
staged.unconfirmed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "skipped", Confirmation: "unconfirmed"}
|
||||
return staged
|
||||
}
|
||||
if err := record("dispatched", "unconfirmed", "awaiting agent acknowledgement"); err != nil {
|
||||
staged.failed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Error: err.Error()}
|
||||
return staged
|
||||
}
|
||||
if deployment.DispatchAttempts >= 3 {
|
||||
_ = record("unknown", "unconfirmed", "delivery attempt budget exhausted")
|
||||
staged.failed = 1
|
||||
staged.hasResult = true
|
||||
staged.result = nodedispatch.Result{SpeakerID: sp.ID, Status: "error", Confirmation: "unconfirmed", Error: "delivery attempt budget exhausted"}
|
||||
return staged
|
||||
}
|
||||
staged.wake = true
|
||||
return staged
|
||||
}
|
||||
|
||||
// finishSpeakerWake records the agent acknowledgement. The speaker lock covers the
|
||||
// state update and is not held while WakeSpeaker is in flight.
|
||||
func (w *Worker) finishSpeakerWake(ctx context.Context, j *Job, sp *store.Speaker, revID string, deployment store.DeploymentSpeaker, res nodedispatch.Result, cancelled func() bool) (nodedispatch.Result, bool, int, int, string) {
|
||||
mu := w.speakerDispatchMu(sp.ID)
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if cancelled != nil && cancelled() {
|
||||
return nodedispatch.Result{SpeakerID: sp.ID, Endpoint: res.Endpoint, Status: "cancelled", Confirmation: "unconfirmed"}, true, 0, 0, ""
|
||||
}
|
||||
record, markApplied := w.bindDeploymentWriters(ctx, j, sp, revID, deployment)
|
||||
appliedID := ""
|
||||
if res.Status == "ok" && res.AppliedRevisionID != "" {
|
||||
if res.AppliedRevisionID != revID {
|
||||
res.Status = "error"
|
||||
res.Confirmation = "unconfirmed"
|
||||
res.Error = "agent reported a different applied revision"
|
||||
} else if latest, _, err := w.Store.LatestPublishedRevision(sp.ID); err != nil {
|
||||
res.Status = "error"
|
||||
res.Confirmation = "unconfirmed"
|
||||
res.Error = err.Error()
|
||||
} else if latest != revID {
|
||||
res.Status = "error"
|
||||
res.Confirmation = "unconfirmed"
|
||||
res.Error = "stale agent acknowledgement: desired revision changed"
|
||||
} else if err := markApplied("agent_reported"); err != nil {
|
||||
res.Status = "error"
|
||||
res.Confirmation = "unconfirmed"
|
||||
res.Error = err.Error()
|
||||
} else {
|
||||
appliedID = sp.ID
|
||||
}
|
||||
}
|
||||
if res.Status != "ok" || res.Confirmation != "agent_reported" {
|
||||
state := "unconfirmed"
|
||||
if res.Status != "ok" {
|
||||
state = "unknown"
|
||||
}
|
||||
if err := record(state, "unconfirmed", res.Error); err != nil {
|
||||
res.Status = "error"
|
||||
res.Error = err.Error()
|
||||
}
|
||||
}
|
||||
patch := store.SpeakerMeta{
|
||||
LastDispatchAt: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
LastDispatchStatus: res.Status,
|
||||
}
|
||||
if res.Error != "" {
|
||||
patch.LastDispatchError = res.Error
|
||||
patch.SyncStatus = "error"
|
||||
} else if res.Status == "ok" && res.Confirmation == "agent_reported" {
|
||||
patch.LastDispatchError = ""
|
||||
patch.SyncStatus = "synced"
|
||||
} else if res.Status == "ok" {
|
||||
patch.SyncStatus = "unconfirmed"
|
||||
}
|
||||
merged := store.MergeSpeakerMetaJSON(sp.MetaJSON, patch)
|
||||
if j.OwnerID == "" {
|
||||
if _, err := w.Store.UpdateSpeaker(j.TenantID, sp.ID, &store.SpeakerPatch{MetaJSON: &merged}); err != nil {
|
||||
res.Status = "error"
|
||||
res.Error = "dispatch metadata update: " + err.Error()
|
||||
}
|
||||
}
|
||||
failed := 0
|
||||
unconfirmed := 0
|
||||
if res.Status == "ok" && res.Confirmation == "unconfirmed" {
|
||||
unconfirmed = 1
|
||||
} else if res.Status != "ok" {
|
||||
failed = 1
|
||||
}
|
||||
return res, true, failed, unconfirmed, appliedID
|
||||
}
|
||||
|
||||
func (w *Worker) runRollback(j *Job) {
|
||||
src := strings.TrimSpace(j.metaString("source_revision_id"))
|
||||
if src == "" {
|
||||
@@ -592,7 +1037,10 @@ func (w *Worker) runRollback(j *Job) {
|
||||
"message": fmt.Sprintf("Откат %s → %s", shortID(src), shortID(newID)),
|
||||
},
|
||||
})
|
||||
w.enqueueDeployAllSpeakers(j, j.TenantID, newID)
|
||||
if err := w.enqueueDeployAllSpeakers(j, j.TenantID, newID); err != nil {
|
||||
j.Fail(err.Error())
|
||||
return
|
||||
}
|
||||
j.Succeed()
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package jobs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -185,3 +188,187 @@ func TestPeerReconcile_RendersFromLatestRevisionAndQueuesDeploy(t *testing.T) {
|
||||
t.Fatalf("expected deploy_apply_job_id in peer_reconcile meta, got %#v", meta)
|
||||
}
|
||||
}
|
||||
|
||||
type renderHookStore struct {
|
||||
*store.Memory
|
||||
onCreate func(prefixes []store.PrefixRow)
|
||||
}
|
||||
|
||||
func (s *renderHookStore) CreateRenderRevision(ctx context.Context, revisionID, tenantID, moduleID string, parent *string, contentHash string, preview map[string]string, prefixes []store.PrefixRow) error {
|
||||
if err := s.Memory.CreateRenderRevision(ctx, revisionID, tenantID, moduleID, parent, contentHash, preview, prefixes); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.onCreate != nil {
|
||||
copied := append([]store.PrefixRow(nil), prefixes...)
|
||||
s.onCreate(copied)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func newIPRenderFixture(t *testing.T) (tenant, module string, mem *store.Memory) {
|
||||
t.Helper()
|
||||
t.Setenv("EVOBGP_ASN_RESOLVE", "0")
|
||||
t.Setenv("EVOBGP_BIRD_ACTIVE_DIR", "")
|
||||
mem = store.NewMemory()
|
||||
mem.SeedDemo()
|
||||
var modIP string
|
||||
tenant, _, modIP, _, _ = mem.DemoIDs()
|
||||
module = modIP
|
||||
for _, mod := range mem.ListModules(tenant) {
|
||||
if mod == nil || mod.ID == module {
|
||||
continue
|
||||
}
|
||||
disabled := false
|
||||
if _, err := mem.UpdateModule(tenant, mod.ID, &store.ModulePatch{Enabled: &disabled}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
return tenant, module, mem
|
||||
}
|
||||
|
||||
func installModuleSnapshot(t *testing.T, mem *store.Memory, tenant, module, prefix string) {
|
||||
t.Helper()
|
||||
err := mem.SetModulePrefixSnapshot(context.Background(), tenant, module, "snap-"+prefix, []store.PrefixRow{{
|
||||
Prefix: prefix,
|
||||
Source: "ip_range",
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatalf("snapshot %s: %v", prefix, err)
|
||||
}
|
||||
}
|
||||
|
||||
func revisionHasPrefix(t *testing.T, mem *store.Memory, tenant, revID, prefix string) bool {
|
||||
t.Helper()
|
||||
rows, _, _ := mem.ListRevisionPrefixes(tenant, revID, "", 100)
|
||||
for _, row := range rows {
|
||||
if row.Prefix == prefix {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func TestRenderTenantUntilQuietOverlappingIngests(t *testing.T) {
|
||||
t.Run("latest snapshot", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
prefixes := []string{"10.8.0.0/24", "10.9.0.0/24", "10.10.0.0/24", "10.11.0.0/24"}
|
||||
installModuleSnapshot(t, mem, tenant, module, prefixes[0])
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
step := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
step++
|
||||
if step >= len(prefixes) {
|
||||
return
|
||||
}
|
||||
installModuleSnapshot(t, mem, tenant, module, prefixes[step])
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j := &Job{ID: "refresh-overlap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusSucceeded {
|
||||
errText := ""
|
||||
if j.Error != nil {
|
||||
errText = *j.Error
|
||||
}
|
||||
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||
}
|
||||
if step != len(prefixes) {
|
||||
t.Fatalf("render commits=%d, want %d so three overlapping ingests are included", step, len(prefixes))
|
||||
}
|
||||
rev := j.metaString("revision_id")
|
||||
if !revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-1]) {
|
||||
t.Fatalf("deployed revision %s missing latest prefix %s", rev, prefixes[len(prefixes)-1])
|
||||
}
|
||||
if revisionHasPrefix(t, mem, tenant, rev, prefixes[len(prefixes)-2]) {
|
||||
t.Fatalf("deployed revision %s still has stale prefix %s", rev, prefixes[len(prefixes)-2])
|
||||
}
|
||||
deployID := j.metaString("deploy_apply_job_id")
|
||||
deploy, err := wk.Registry.Get(tenant, deployID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if deploy.metaString("revision_id") != rev {
|
||||
t.Fatalf("deploy revision=%s, want %s", deploy.metaString("revision_id"), rev)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cancel does not deploy", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.12.0.0/24")
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
var j *Job
|
||||
commits := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
commits++
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.13.0.0/24")
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
j.RequestCancel()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j = &Job{ID: "refresh-cancel", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusFailed {
|
||||
t.Fatalf("status=%s, want failed", j.Status)
|
||||
}
|
||||
if j.Error == nil || !strings.Contains(*j.Error, "newer snapshot") {
|
||||
t.Fatalf("error=%v, want superseded render", j.Error)
|
||||
}
|
||||
if commits != 1 {
|
||||
t.Fatalf("render commits=%d, want 1", commits)
|
||||
}
|
||||
if j.metaString("deploy_apply_job_id") != "" || j.metaString("revision_id") != "" {
|
||||
t.Fatalf("cancelled dirty render published meta %#v", j.metaCopy())
|
||||
}
|
||||
deploys, _, _ := wk.Registry.List(tenant, "", KindDeployApply, "", 10)
|
||||
if len(deploys) != 0 {
|
||||
t.Fatalf("deploy jobs=%d, want 0", len(deploys))
|
||||
}
|
||||
revs, _, _ := mem.ListRevisions(tenant, "", "", 50)
|
||||
for _, rev := range revs {
|
||||
if rev != nil && revisionHasPrefix(t, mem, tenant, rev.ID, "10.13.0.0/24") {
|
||||
t.Fatalf("revision %s contains the snapshot that landed during cancel", rev.ID)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("cap does not publish", func(t *testing.T) {
|
||||
tenant, module, mem := newIPRenderFixture(t)
|
||||
installModuleSnapshot(t, mem, tenant, module, "10.20.0.0/24")
|
||||
hook := &renderHookStore{Memory: mem}
|
||||
var wk *Worker
|
||||
commits := 0
|
||||
hook.onCreate = func([]store.PrefixRow) {
|
||||
commits++
|
||||
installModuleSnapshot(t, mem, tenant, module, fmt.Sprintf("10.21.%d.0/24", commits))
|
||||
gate := wk.tenantRefreshGate(tenant)
|
||||
gate.mu.Lock()
|
||||
gate.dirty = true
|
||||
gate.mu.Unlock()
|
||||
}
|
||||
wk = &Worker{Store: hook, Registry: NewRegistry(nil)}
|
||||
j := &Job{ID: "refresh-cap", TenantID: tenant, Kind: KindModuleRefresh, Status: StatusRunning}
|
||||
wk.finishModuleRefreshSuccess(j, module)
|
||||
if j.Status != StatusFailed || j.Error == nil || !strings.Contains(*j.Error, "did not quiesce") {
|
||||
errText := ""
|
||||
if j.Error != nil {
|
||||
errText = *j.Error
|
||||
}
|
||||
t.Fatalf("status=%s err=%s", j.Status, errText)
|
||||
}
|
||||
if commits != renderQuietPassLimit {
|
||||
t.Fatalf("render commits=%d, want cap %d", commits, renderQuietPassLimit)
|
||||
}
|
||||
if j.metaString("deploy_apply_job_id") != "" {
|
||||
t.Fatal("safety cap published a deploy")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -107,14 +107,14 @@ func LookupWithResolver(
|
||||
if addr, err := netip.ParseAddr(raw); err == nil {
|
||||
out.QueryKind = KindIP
|
||||
out.Normalized = addr.String()
|
||||
if err := lookupIP(st, tenantID, addr, out, commByID, ""); err != nil {
|
||||
if err := lookupIP(ctx, st, tenantID, addr, out, commByID, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else if pfx, err := netip.ParsePrefix(raw); err == nil {
|
||||
masked := pfx.Masked()
|
||||
out.QueryKind = KindCIDR
|
||||
out.Normalized = masked.String()
|
||||
if err := lookupCIDR(st, tenantID, masked, out, commByID); err != nil {
|
||||
if err := lookupCIDR(ctx, st, tenantID, masked, out, commByID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
@@ -124,7 +124,7 @@ func LookupWithResolver(
|
||||
}
|
||||
out.QueryKind = KindDomain
|
||||
out.Normalized = fqdn
|
||||
if err := lookupDomain(st, tenantID, fqdn, out, commByID); err != nil {
|
||||
if err := lookupDomain(ctx, st, tenantID, fqdn, out, commByID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resolve == nil {
|
||||
@@ -178,7 +178,7 @@ func lookupResolvedIPs(
|
||||
out.ResolvedIPs = make([]string, 0, len(ips))
|
||||
for _, ip := range ips {
|
||||
out.ResolvedIPs = append(out.ResolvedIPs, ip.String())
|
||||
if err := lookupIP(st, tenantID, ip, out, commByID, ip.String()); err != nil {
|
||||
if err := lookupIP(ctx, st, tenantID, ip, out, commByID, ip.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -186,6 +186,7 @@ func lookupResolvedIPs(
|
||||
}
|
||||
|
||||
func lookupIP(
|
||||
ctx context.Context,
|
||||
st store.Backend,
|
||||
tenantID string,
|
||||
addr netip.Addr,
|
||||
@@ -227,7 +228,7 @@ func lookupIP(
|
||||
}
|
||||
}
|
||||
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -269,6 +270,7 @@ func prefixCoversQuery(listPfx, query netip.Prefix) bool {
|
||||
}
|
||||
|
||||
func lookupCIDR(
|
||||
ctx context.Context,
|
||||
st store.Backend,
|
||||
tenantID string,
|
||||
query netip.Prefix,
|
||||
@@ -308,7 +310,7 @@ func lookupCIDR(
|
||||
}
|
||||
}
|
||||
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mod.ID)
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -338,7 +340,7 @@ func lookupCIDR(
|
||||
return nil
|
||||
}
|
||||
|
||||
func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error {
|
||||
func lookupDomain(ctx context.Context, st store.Backend, tenantID, fqdn string, out *Result, commByID map[string]*store.Community) error {
|
||||
matchedModuleIDs := make(map[string]*store.Module)
|
||||
|
||||
for _, mod := range st.ListModules(tenantID) {
|
||||
@@ -372,7 +374,7 @@ func lookupDomain(st store.Backend, tenantID, fqdn string, out *Result, commByID
|
||||
}
|
||||
|
||||
for mid, mod := range matchedModuleIDs {
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(tenantID, mid)
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mid)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestLookupIPEntryAndSnapshot(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash1", []store.PrefixRow{
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash1", []store.PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -129,7 +129,7 @@ func TestLookupDomainEntryAndSnapshot(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.SetModulePrefixSnapshot(tenant, mod.ID, "hash-d", []store.PrefixRow{
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash-d", []store.PrefixRow{
|
||||
{Prefix: "198.51.100.1/32", CommunityID: &cid, Source: "domain"},
|
||||
{Prefix: "203.0.113.9/32", CommunityID: &cid, Source: "other"},
|
||||
}); err != nil {
|
||||
@@ -183,7 +183,7 @@ func TestLookupDomainResolvedIPAgainstRanges(t *testing.T) {
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-r", []store.PrefixRow{
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-r", []store.PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -234,7 +234,7 @@ func TestLookupCIDREntryAndSnapshot(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.SetModulePrefixSnapshot(tenant, modIP, "hash-cidr", []store.PrefixRow{
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, modIP, "hash-cidr", []store.PrefixRow{
|
||||
{Prefix: "203.0.113.0/24", CommunityID: &cid, Source: "ip_range"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -61,13 +61,17 @@ func nodeHTTPClient() *http.Client {
|
||||
}
|
||||
|
||||
func fetchLatestRevision(base, token, speaker string) (string, error) {
|
||||
return fetchLatestRevisionContext(context.Background(), base, token, speaker)
|
||||
}
|
||||
|
||||
func fetchLatestRevisionContext(parent context.Context, base, token, speaker string) (string, error) {
|
||||
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/revisions/latest"
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||
ctx, cancel := context.WithTimeout(parent, 60*time.Second)
|
||||
defer cancel()
|
||||
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||
if err != nil {
|
||||
@@ -75,13 +79,17 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return "", fmt.Errorf("latest revision: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
var out struct {
|
||||
RevisionID string `json:"revision_id"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
|
||||
body, err := readHTTPBytes(resp, 64<<10)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := json.Unmarshal(body, &out); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if out.RevisionID == "" {
|
||||
@@ -91,13 +99,21 @@ func fetchLatestRevision(base, token, speaker string) (string, error) {
|
||||
}
|
||||
|
||||
func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
||||
return fetchBundleContext(context.Background(), base, token, speaker, revision, bundle.Limits{})
|
||||
}
|
||||
|
||||
func fetchBundleContext(parent context.Context, base, token, speaker, revision string, limits bundle.Limits) ([]byte, error) {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u := strings.TrimRight(base, "/") + "/v1/speakers/" + speaker + "/bundle/" + revision
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
ctx, cancel := context.WithTimeout(parent, 120*time.Second)
|
||||
defer cancel()
|
||||
resp, err := httpclient.DoWithRetry(ctx, nodeHTTPClient(), req, 3)
|
||||
if err != nil {
|
||||
@@ -105,10 +121,37 @@ func fetchBundle(base, token, speaker, revision string) ([]byte, error) {
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return nil, fmt.Errorf("bundle: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
return io.ReadAll(resp.Body)
|
||||
return readHTTPBytes(resp, l.MaxCompressedBytes)
|
||||
}
|
||||
|
||||
func readHTTPBytes(resp *http.Response, limit int64) ([]byte, error) {
|
||||
if resp.ContentLength > limit {
|
||||
return nil, bundle.ErrLimit
|
||||
}
|
||||
return bundle.ReadCompressed(resp.Body, bundle.Limits{MaxCompressedBytes: limit})
|
||||
}
|
||||
|
||||
func readBundleFile(path string, limits bundle.Limits) ([]byte, error) {
|
||||
l, err := limits.Normalized()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = f.Close() }()
|
||||
info, err := f.Stat()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if info.Size() > l.MaxCompressedBytes {
|
||||
return nil, bundle.ErrLimit
|
||||
}
|
||||
return bundle.ReadCompressed(f, l)
|
||||
}
|
||||
|
||||
func loadPubKey(pubB64, pubHex string) (ed25519.PublicKey, error) {
|
||||
@@ -146,7 +189,7 @@ func CmdVerifyBundle(args []string) int {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
raw, err := os.ReadFile(*path)
|
||||
raw, err := readBundleFile(*path, bundle.Limits{})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
@@ -182,7 +225,7 @@ func CmdApplyBundle(args []string) int {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 2
|
||||
}
|
||||
raw, err := os.ReadFile(*path)
|
||||
raw, err := readBundleFile(*path, bundle.Limits{})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package nodecli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
)
|
||||
|
||||
type countingBody struct {
|
||||
io.Reader
|
||||
read int
|
||||
}
|
||||
|
||||
func (b *countingBody) Read(p []byte) (int, error) {
|
||||
n, e := b.Reader.Read(p)
|
||||
b.read += n
|
||||
return n, e
|
||||
}
|
||||
func (b *countingBody) Close() error { return nil }
|
||||
|
||||
func TestHTTPBundleBoundKnownAndChunked(t *testing.T) {
|
||||
for _, length := range []int64{-1, 2048} {
|
||||
body := &countingBody{Reader: bytes.NewReader(bytes.Repeat([]byte{'x'}, 2048))}
|
||||
_, err := readHTTPBytes(&http.Response{Body: body, ContentLength: length}, 1024)
|
||||
if !errors.Is(err, bundle.ErrLimit) {
|
||||
t.Fatalf("length=%d limit=%v", length, err)
|
||||
}
|
||||
if body.read > 1025 || (length > 1024 && body.read != 0) {
|
||||
t.Fatalf("unbounded read %d for length %d", body.read, length)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLocalBundleSizeBound(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "bundle.tar.gz")
|
||||
if err := os.WriteFile(path, bytes.Repeat([]byte{'x'}, 2048), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := readBundleFile(path, bundle.Limits{MaxCompressedBytes: 1024}); !errors.Is(err, bundle.ErrLimit) {
|
||||
t.Fatalf("local input cap: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleHTTPMetadataAndDiagnosticsBound(t *testing.T) {
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case strings.Contains(r.URL.Path, "/revisions/latest"):
|
||||
_, _ = io.WriteString(w, strings.Repeat(" ", 65<<10))
|
||||
default:
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = io.WriteString(w, strings.Repeat("x", 64<<10))
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
if _, err := fetchLatestRevision(server.URL, "key", "speaker"); !errors.Is(err, bundle.ErrLimit) {
|
||||
t.Fatalf("metadata unbounded: %v", err)
|
||||
}
|
||||
if _, err := fetchBundle(server.URL, "key", "speaker", "revision"); err == nil || len(err.Error()) > 5<<10 {
|
||||
t.Fatalf("diagnostic unbounded: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSyncFetchRespectsCallerCancellation(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if _, err := fetchBundleContext(ctx, "http://127.0.0.1:1", "key", "speaker", "revision", bundle.Limits{}); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("download ignored cancel: %v", err)
|
||||
}
|
||||
if _, err := fetchLatestRevisionContext(ctx, "http://127.0.0.1:1", "key", "speaker"); !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("metadata ignored cancel: %v", err)
|
||||
}
|
||||
}
|
||||
+17
-17
@@ -12,23 +12,23 @@ import (
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/signing"
|
||||
)
|
||||
|
||||
// SyncConfig drives pull → verify → apply on a replica node.
|
||||
type SyncConfig struct {
|
||||
BaseURL string
|
||||
Token string
|
||||
SpeakerID string
|
||||
RevisionID string // empty = latest published on CP
|
||||
PubKeyB64 string
|
||||
PubKeyHex string
|
||||
ExtractDir string
|
||||
BundlePath string // temp file; default os.TempDir()/evobgp-bundle.tar.gz
|
||||
BirdBin string
|
||||
BirdcBin string
|
||||
Socket string
|
||||
HTTPClient interface {
|
||||
BundleLimits bundle.Limits // zero selects bounded large-config defaults
|
||||
BaseURL string
|
||||
Token string
|
||||
SpeakerID string
|
||||
RevisionID string // empty = latest published on CP
|
||||
PubKeyB64 string
|
||||
PubKeyHex string
|
||||
ExtractDir string
|
||||
BundlePath string // temp file; default os.TempDir()/evobgp-bundle.tar.gz
|
||||
BirdBin string
|
||||
BirdcBin string
|
||||
Socket string
|
||||
HTTPClient interface {
|
||||
Do(req interface{}) (interface{}, error)
|
||||
}
|
||||
Timeout time.Duration
|
||||
@@ -59,12 +59,12 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
||||
rev := strings.TrimSpace(cfg.RevisionID)
|
||||
if rev == "" {
|
||||
var err error
|
||||
rev, err = fetchLatestRevision(cfg.BaseURL, cfg.Token, cfg.SpeakerID)
|
||||
rev, err = fetchLatestRevisionContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID)
|
||||
if err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
}
|
||||
raw, err := fetchBundle(cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev)
|
||||
raw, err := fetchBundleContext(ctx, cfg.BaseURL, cfg.Token, cfg.SpeakerID, rev, cfg.BundleLimits)
|
||||
if err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
@@ -75,7 +75,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
||||
if err := os.WriteFile(bundlePath, raw, 0o644); err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
v, err := signing.VerifyGzippedTar(raw, pub)
|
||||
v, err := bundle.VerifyGzippedTarWithLimits(raw, pub, cfg.BundleLimits)
|
||||
if err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
@@ -83,7 +83,7 @@ func SyncBundle(ctx context.Context, cfg SyncConfig) (SyncResult, error) {
|
||||
if err := os.MkdirAll(root, 0o755); err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
if err := bundle.WriteExtractedFiles(root, v); err != nil {
|
||||
if err := bundle.WriteExtractedFilesWithLimits(root, v, cfg.BundleLimits); err != nil {
|
||||
return SyncResult{}, err
|
||||
}
|
||||
mainRel := v.FindMainBirdConf()
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/birdfmt"
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
@@ -52,11 +53,16 @@ func FetchBirdProtocols(ctx context.Context, sp *store.Speaker, opts Options) Bi
|
||||
return res
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
res.Error = fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b)))
|
||||
return res
|
||||
}
|
||||
b, err := httpclient.ReadResponseBody(resp, 8<<20)
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
return res
|
||||
}
|
||||
var out struct {
|
||||
Sessions []birdfmt.BGPSession `json:"sessions"`
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ type Result struct {
|
||||
SpeakerID string `json:"speaker_id"`
|
||||
Endpoint string `json:"endpoint,omitempty"`
|
||||
Status string `json:"status"`
|
||||
Confirmation string `json:"confirmation,omitempty"`
|
||||
AppliedRevisionID string `json:"applied_revision_id,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
@@ -62,12 +63,13 @@ func Enabled() bool {
|
||||
|
||||
// WakeSpeaker POSTs /v1/agent/sync to a replica agent (HTTPS via Traefik).
|
||||
func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
|
||||
res := Result{SpeakerID: sp.ID}
|
||||
res := Result{}
|
||||
if sp == nil {
|
||||
res.Status = "error"
|
||||
res.Error = "nil speaker"
|
||||
return res
|
||||
}
|
||||
res.SpeakerID = sp.ID
|
||||
meta := store.ParseSpeakerMeta(sp.MetaJSON)
|
||||
url := store.AgentSyncURL(meta)
|
||||
if url == "" {
|
||||
@@ -117,13 +119,31 @@ func WakeSpeaker(ctx context.Context, sp *store.Speaker, opts Options) Result {
|
||||
_ = resp.Body.Close()
|
||||
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
|
||||
var out struct {
|
||||
OK *bool `json:"ok"`
|
||||
AppliedRevisionID string `json:"applied_revision_id"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if len(bytes.TrimSpace(b)) > 0 {
|
||||
if err := json.Unmarshal(b, &out); err != nil {
|
||||
res.Status = "error"
|
||||
res.Error = "invalid agent sync response"
|
||||
return res
|
||||
}
|
||||
}
|
||||
if (out.OK != nil && !*out.OK) || strings.TrimSpace(out.Error) != "" {
|
||||
res.Status = "error"
|
||||
res.Error = strings.TrimSpace(out.Error)
|
||||
if res.Error == "" {
|
||||
res.Error = "agent sync reported failure"
|
||||
}
|
||||
return res
|
||||
}
|
||||
_ = json.Unmarshal(b, &out)
|
||||
res.Status = "ok"
|
||||
res.AppliedRevisionID = strings.TrimSpace(out.AppliedRevisionID)
|
||||
if res.AppliedRevisionID == "" {
|
||||
res.AppliedRevisionID = strings.TrimSpace(opts.RevisionID)
|
||||
if res.AppliedRevisionID != "" {
|
||||
res.Confirmation = "agent_reported"
|
||||
} else {
|
||||
res.Confirmation = "unconfirmed"
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"evobgp/internal/httpclient"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
@@ -43,12 +44,16 @@ func FetchAgentHealth(ctx context.Context, sp *store.Speaker, opts Options) Agen
|
||||
return AgentHealthResult{Error: err.Error()}
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<10))
|
||||
return AgentHealthResult{
|
||||
Error: fmt.Sprintf("HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(b))),
|
||||
}
|
||||
}
|
||||
b, err := httpclient.ReadResponseBody(resp, 64<<10)
|
||||
if err != nil {
|
||||
return AgentHealthResult{Error: err.Error()}
|
||||
}
|
||||
var out struct {
|
||||
OK bool `json:"ok"`
|
||||
LastAppliedRevisionID string `json:"last_applied_revision_id"`
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
package nodedispatch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/bundle"
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("EVOBGP_TEST_LEGACY_BIRD_HELPER") != "1" {
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
args := os.Args[1:]
|
||||
log, err := os.OpenFile(os.Getenv("EVOBGP_TEST_BIRD_LOG"), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
os.Exit(2)
|
||||
}
|
||||
_, _ = fmt.Fprintln(log, strings.Join(args, " "))
|
||||
_ = log.Close()
|
||||
switch {
|
||||
case len(args) == 3 && args[0] == "-c" && args[2] == "-p":
|
||||
if _, err := os.Stat(args[1]); err == nil {
|
||||
os.Exit(0)
|
||||
}
|
||||
case len(args) == 3 && args[0] == "-s" && args[2] == "configure":
|
||||
os.Exit(0)
|
||||
case len(args) == 5 && args[0] == "-s" && strings.Join(args[2:], " ") == "show protocols all":
|
||||
fmt.Println("peer_test BGP master up 2026-01-01 Established")
|
||||
os.Exit(0)
|
||||
}
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
// TestLegacyAgentBinaryCurrentDispatch runs the historical agent as a separate
|
||||
// process. All network traffic stays on loopback and BIRD executables are fakes.
|
||||
func TestLegacyAgentBinaryCurrentDispatch(t *testing.T) {
|
||||
binary := os.Getenv("EVOBGP_TEST_LEGACY_AGENT_BIN")
|
||||
if binary == "" {
|
||||
t.Skip("EVOBGP_TEST_LEGACY_AGENT_BIN not set")
|
||||
}
|
||||
binary, err := filepath.Abs(binary)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(binary); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
const speakerID = "legacy-speaker"
|
||||
const revisionID = "legacy-revision"
|
||||
const nodeToken = "local-node-token"
|
||||
const agentSecret = "local-agent-secret"
|
||||
pub, priv, err := ed25519.GenerateKey(rand.Reader)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
archive, err := bundle.BuildGzippedTar(revisionID, speakerID, map[string]string{
|
||||
"bird.conf": "router id 192.0.2.1;\n",
|
||||
}, priv)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var bundleGets atomic.Int32
|
||||
controlPlane := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer "+nodeToken {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/v1/speakers/" + speakerID + "/bundle/" + revisionID:
|
||||
bundleGets.Add(1)
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
_, _ = w.Write(archive)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer controlPlane.Close()
|
||||
|
||||
root := t.TempDir()
|
||||
logPath := filepath.Join(root, "bird-invocations")
|
||||
helper, err := os.Executable()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
agentAddr := listener.Addr().String()
|
||||
_ = listener.Close()
|
||||
outputPath := filepath.Join(root, "agent.log")
|
||||
output, err := os.Create(outputPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = output.Close() }()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, binary, "-listen", agentAddr, "serve")
|
||||
cmd.Stdout, cmd.Stderr = output, output
|
||||
cmd.Env = append(os.Environ(),
|
||||
"EVOBGP_AGENT_SECRET="+agentSecret,
|
||||
"EVOBGP_CONTROL_PLANE_URL="+controlPlane.URL,
|
||||
"EVOBGP_NODE_TOKEN="+nodeToken,
|
||||
"EVOBGP_SPEAKER_ID="+speakerID,
|
||||
"EVOBGP_BUNDLE_PUBKEY_BASE64="+base64.StdEncoding.EncodeToString(pub),
|
||||
"EVOBGP_BIRD_EXTRACT_DIR="+filepath.Join(root, "extract"),
|
||||
"EVOBGP_BIRD_BIN="+helper,
|
||||
"EVOBGP_BIRDC_BIN="+helper,
|
||||
"EVOBGP_BIRDC_SOCKET="+filepath.Join(root, "fake.ctl"),
|
||||
"EVOBGP_TEST_LEGACY_BIRD_HELPER=1",
|
||||
"EVOBGP_TEST_BIRD_LOG="+logPath,
|
||||
"TMPDIR="+root,
|
||||
"TMP="+root,
|
||||
"TEMP="+root,
|
||||
)
|
||||
if err := cmd.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() {
|
||||
cancel()
|
||||
_ = cmd.Wait()
|
||||
}()
|
||||
|
||||
transport := http.DefaultTransport.(*http.Transport).Clone()
|
||||
// Under -race each helper process can spend a second flushing race output at
|
||||
// exit. Sync executes parse-check and configure before returning its ACK.
|
||||
client := &http.Client{Timeout: 10 * time.Second, Transport: legacyAgentTransport{addr: agentAddr, base: transport}}
|
||||
defer transport.CloseIdleConnections()
|
||||
sp := &store.Speaker{ID: speakerID, Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
AgentDomain: "legacy-agent.example.test", AgentSecret: agentSecret,
|
||||
})}
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||
if health.OK {
|
||||
if health.LastAppliedRevisionID != "" {
|
||||
t.Fatalf("initial health has applied revision: %+v", health)
|
||||
}
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
log, _ := os.ReadFile(outputPath)
|
||||
t.Fatalf("legacy agent did not become healthy: %+v; output: %s", health, log)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
|
||||
// The historical agent must enforce the bearer token before reaching sync.
|
||||
badReq, err := http.NewRequest(http.MethodPost, "https://legacy-agent.example.test/v1/agent/sync", strings.NewReader(`{"revision_id":"legacy-revision"}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
badReq.Header.Set("Authorization", "Bearer wrong-secret")
|
||||
badResp, err := client.Do(badReq)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = badResp.Body.Close()
|
||||
if badResp.StatusCode != http.StatusUnauthorized || bundleGets.Load() != 0 {
|
||||
t.Fatalf("unauthorized sync: status=%d bundle_gets=%d", badResp.StatusCode, bundleGets.Load())
|
||||
}
|
||||
|
||||
result := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{
|
||||
HTTPClient: client, MaxRetries: 1, RevisionID: revisionID,
|
||||
})
|
||||
if result.Status != "ok" || result.Confirmation != "agent_reported" || result.AppliedRevisionID != revisionID {
|
||||
log, _ := os.ReadFile(outputPath)
|
||||
t.Fatalf("current dispatch to legacy agent: %+v; output: %s", result, log)
|
||||
}
|
||||
if bundleGets.Load() != 1 {
|
||||
t.Fatalf("signed bundle gets = %d, want 1", bundleGets.Load())
|
||||
}
|
||||
health := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||
if !health.OK || health.LastAppliedRevisionID != revisionID || health.LastSyncAt == "" {
|
||||
t.Fatalf("post-sync legacy health: %+v", health)
|
||||
}
|
||||
protocols := nodedispatch.FetchBirdProtocols(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||
if protocols.Error != "" || len(protocols.Sessions) != 1 || protocols.Sessions[0].State != "Established" {
|
||||
t.Fatalf("legacy bird protocols: %+v", protocols)
|
||||
}
|
||||
mainConfig, err := os.ReadFile(filepath.Join(root, "extract", "bird.conf"))
|
||||
if err != nil || string(mainConfig) != "router id 192.0.2.1;\n" {
|
||||
t.Fatalf("extracted signed bundle: content=%q err=%v", mainConfig, err)
|
||||
}
|
||||
invocations, err := os.ReadFile(logPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wantParse := fmt.Sprintf("-c %s -p", filepath.Join(root, "extract", "bird.conf"))
|
||||
if !strings.Contains(string(invocations), wantParse) || !strings.Contains(string(invocations), "-s "+filepath.Join(root, "fake.ctl")+" configure") {
|
||||
t.Fatalf("legacy agent did not parse-check and configure via stand-ins: %s", invocations)
|
||||
}
|
||||
}
|
||||
|
||||
type legacyAgentTransport struct {
|
||||
addr string
|
||||
base http.RoundTripper
|
||||
}
|
||||
|
||||
func (tr legacyAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
copyReq := req.Clone(req.Context())
|
||||
copyReq.URL.Scheme = "http"
|
||||
copyReq.URL.Host = tr.addr
|
||||
copyReq.Host = tr.addr
|
||||
return tr.base.RoundTrip(copyReq)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package nodedispatch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type legacyRoundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f legacyRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
func TestLegacyCompatibilityAgentSync(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, response, wantApplied string
|
||||
status int
|
||||
}{
|
||||
{"legacy bare success", `{"ok":true}`, "", http.StatusOK},
|
||||
{"legacy empty no-content", "", "", http.StatusNoContent},
|
||||
{"legacy explicit revision", `{"ok":true,"applied_revision_id":"revision-1"}`, "revision-1", http.StatusOK},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var gotAuth, gotType string
|
||||
var gotBody map[string]json.RawMessage
|
||||
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Method != http.MethodPost || r.URL.Path != "/v1/agent/sync" {
|
||||
t.Errorf("unexpected sync request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
gotAuth, gotType = r.Header.Get("Authorization"), r.Header.Get("Content-Type")
|
||||
body, _ := io.ReadAll(r.Body)
|
||||
if err := json.Unmarshal(body, &gotBody); err != nil {
|
||||
t.Errorf("sync JSON: %v", err)
|
||||
}
|
||||
return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.response)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
|
||||
})}
|
||||
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
|
||||
if got.Status != "ok" || got.AppliedRevisionID != tc.wantApplied {
|
||||
t.Fatalf("sync result: %+v", got)
|
||||
}
|
||||
wantConfirmation := "unconfirmed"
|
||||
if tc.wantApplied != "" {
|
||||
wantConfirmation = "agent_reported"
|
||||
}
|
||||
if got.Confirmation != wantConfirmation {
|
||||
t.Fatalf("confirmation = %q, want %q", got.Confirmation, wantConfirmation)
|
||||
}
|
||||
if gotAuth != "Bearer legacy-secret" || gotType != "application/json" {
|
||||
t.Fatalf("legacy headers: authorization=%q content-type=%q", gotAuth, gotType)
|
||||
}
|
||||
if !reflect.DeepEqual(gotBody, map[string]json.RawMessage{"revision_id": json.RawMessage(`"revision-1"`)}) {
|
||||
t.Fatalf("unexpected fields sent to strict legacy decoder: %s", gotBody)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentSyncRejectsMalformedSuccess(t *testing.T) {
|
||||
for _, response := range []string{`{"ok":false}`, `{"ok":true`, `{"ok":false,"applied_revision_id":"revision-1"}`} {
|
||||
t.Run(response, func(t *testing.T) {
|
||||
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
sp := &store.Speaker{ID: "speaker-1", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
|
||||
got := nodedispatch.WakeSpeaker(context.Background(), sp, nodedispatch.Options{HTTPClient: client, MaxRetries: 1, RevisionID: "revision-1"})
|
||||
if got.Status != "error" || got.AppliedRevisionID != "" {
|
||||
t.Fatalf("invalid 2xx response accepted: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyCompatibilityAgentHealthMissingNewFields(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, body, wantRevision, wantSync string
|
||||
}{
|
||||
{"bare legacy health", `{"ok":true}`, "", ""},
|
||||
{"revision without time", `{"ok":true,"last_applied_revision_id":"revision-1"}`, "revision-1", ""},
|
||||
{"revision with time", `{"ok":true,"last_applied_revision_id":"revision-1","last_sync_at":"2026-10-26T00:00:00Z"}`, "revision-1", "2026-10-26T00:00:00Z"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
client := &http.Client{Transport: legacyRoundTripFunc(func(r *http.Request) (*http.Response, error) {
|
||||
if r.Method != http.MethodGet || r.URL.Path != "/v1/agent/health" || r.Header.Get("Authorization") != "Bearer legacy-secret" {
|
||||
t.Errorf("unexpected health request: %s %s", r.Method, r.URL)
|
||||
}
|
||||
return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(tc.body)), Header: make(http.Header)}, nil
|
||||
})}
|
||||
sp := &store.Speaker{ID: "speaker-1", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{
|
||||
AgentDomain: "agent.example.test", AgentSecret: "legacy-secret",
|
||||
})}
|
||||
got := nodedispatch.FetchAgentHealth(context.Background(), sp, nodedispatch.Options{HTTPClient: client})
|
||||
if !got.OK || got.Error != "" || got.LastAppliedRevisionID != tc.wantRevision || got.LastSyncAt != tc.wantSync {
|
||||
t.Fatalf("legacy health: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package nodedispatch_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/nodedispatch"
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type telemetryTestBody struct {
|
||||
io.Reader
|
||||
read int
|
||||
closed bool
|
||||
}
|
||||
|
||||
func (b *telemetryTestBody) Read(p []byte) (int, error) {
|
||||
n, err := b.Reader.Read(p)
|
||||
b.read += n
|
||||
return n, err
|
||||
}
|
||||
func (b *telemetryTestBody) Close() error { b.closed = true; return nil }
|
||||
|
||||
type telemetryErrorReader struct{}
|
||||
|
||||
func (telemetryErrorReader) Read([]byte) (int, error) { return 0, errors.New("injected body failure") }
|
||||
|
||||
func TestAgentTelemetryCompleteBodyLimits(t *testing.T) {
|
||||
sp := &store.Speaker{ID: "legacy", Role: "replica", MetaJSON: store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})}
|
||||
for _, tc := range []struct {
|
||||
name, prefix string
|
||||
limit int
|
||||
call func(nodedispatch.Options) string
|
||||
}{
|
||||
{"health", `{"ok":true}`, 64 << 10, func(opts nodedispatch.Options) string {
|
||||
return nodedispatch.FetchAgentHealth(context.Background(), sp, opts).Error
|
||||
}},
|
||||
{"protocols", `{"sessions":[]}`, 8 << 20, func(opts nodedispatch.Options) string {
|
||||
return nodedispatch.FetchBirdProtocols(context.Background(), sp, opts).Error
|
||||
}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
for _, known := range []bool{true, false} {
|
||||
for _, extra := range []int{0, 1} {
|
||||
data := tc.prefix + strings.Repeat(" ", tc.limit-len(tc.prefix)+extra)
|
||||
body := &telemetryTestBody{Reader: strings.NewReader(data)}
|
||||
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
length := int64(-1)
|
||||
if known {
|
||||
length = int64(len(data))
|
||||
}
|
||||
return &http.Response{StatusCode: 200, ContentLength: length, Body: body, Header: make(http.Header)}, nil
|
||||
})}
|
||||
err := tc.call(nodedispatch.Options{HTTPClient: client})
|
||||
if (extra == 0 && err != "") || (extra == 1 && err == "") || body.read > tc.limit+1 || !body.closed {
|
||||
t.Fatalf("known=%v extra=%d read=%d closed=%v error=%s", known, extra, body.read, body.closed, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, status := range []int{200, 400} {
|
||||
reader := io.MultiReader(strings.NewReader(tc.prefix), telemetryErrorReader{})
|
||||
if status == 400 {
|
||||
reader = io.MultiReader(strings.NewReader(strings.Repeat("x", 16000)))
|
||||
}
|
||||
body := &telemetryTestBody{Reader: reader}
|
||||
client := &http.Client{Transport: legacyRoundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{StatusCode: status, ContentLength: -1, Body: body, Header: make(http.Header)}, nil
|
||||
})}
|
||||
err := tc.call(nodedispatch.Options{HTTPClient: client})
|
||||
if err == "" || !body.closed || (status == 400 && body.read > 4<<10) {
|
||||
t.Fatalf("status=%d read=%d closed=%v error=%s", status, body.read, body.closed, err)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,10 +20,12 @@ import (
|
||||
const namespace = "evobgp"
|
||||
|
||||
var (
|
||||
metricsStore atomic.Value // store.Backend
|
||||
metricsStore atomic.Value // storeBackendHolder
|
||||
registerCollectorOnce sync.Once
|
||||
)
|
||||
|
||||
type storeBackendHolder struct{ backend store.Backend }
|
||||
|
||||
var (
|
||||
httpRequests = promauto.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
@@ -117,7 +119,7 @@ var (
|
||||
upstreamBreakerOpen = promauto.NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: "upstream_breaker_open",
|
||||
Help: "1 if per-host HTTP circuit breaker is open.",
|
||||
Help: "Last request-time observation of the per-host circuit breaker (1=open, 0=closed); idle host series expire with the bounded cache.",
|
||||
}, []string{"host"})
|
||||
)
|
||||
|
||||
@@ -173,6 +175,9 @@ func RecordJobTerminal(kind, status string) {
|
||||
|
||||
// RecordJobAuditDepth updates durable job_audit depth gauges.
|
||||
func RecordJobAuditDepth(counts map[string]int64) {
|
||||
for _, status := range []string{"queued", "running", "succeeded", "failed", "cancelled"} {
|
||||
jobAuditDepth.WithLabelValues(status).Set(float64(counts[status]))
|
||||
}
|
||||
for status, n := range counts {
|
||||
if status == "" {
|
||||
status = "unknown"
|
||||
@@ -193,6 +198,15 @@ func SetUpstreamBreakerOpen(host string, open bool) {
|
||||
upstreamBreakerOpen.WithLabelValues(host).Set(v)
|
||||
}
|
||||
|
||||
// RemoveUpstreamBreaker removes an expired host series together with its cache entry.
|
||||
// Callers must retain active entries until their final metric update completes.
|
||||
func RemoveUpstreamBreaker(host string) {
|
||||
if host == "" {
|
||||
host = "_"
|
||||
}
|
||||
upstreamBreakerOpen.DeleteLabelValues(host)
|
||||
}
|
||||
|
||||
// SetBuildInfo sets evobgp_build_info gauge (idempotent labels).
|
||||
func SetBuildInfo(version, gitSHA string) {
|
||||
if version == "" {
|
||||
@@ -205,34 +219,49 @@ func SetBuildInfo(version, gitSHA string) {
|
||||
}
|
||||
|
||||
type memoryStoreCollector struct {
|
||||
prefixMaxDesc *prometheus.Desc
|
||||
prefixSumDesc *prometheus.Desc
|
||||
peersDesc *prometheus.Desc
|
||||
peerStateDesc *prometheus.Desc
|
||||
prefixMaxDesc *prometheus.Desc
|
||||
prefixSumDesc *prometheus.Desc
|
||||
peersDesc *prometheus.Desc
|
||||
peerStateDesc *prometheus.Desc
|
||||
storedMaxDesc *prometheus.Desc
|
||||
storedSumDesc *prometheus.Desc
|
||||
inventoryDesc *prometheus.Desc
|
||||
storedStateDesc *prometheus.Desc
|
||||
collectionSuccessDesc *prometheus.Desc
|
||||
}
|
||||
|
||||
func newMemoryStoreCollector() *memoryStoreCollector {
|
||||
return &memoryStoreCollector{
|
||||
prefixMaxDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_max"),
|
||||
"Maximum materialized_prefix_count among all revisions in the store.",
|
||||
"Deprecated: maximum materialized_prefix_count among all stored revisions; use stored_revision_materialized_prefixes_max.",
|
||||
nil, nil,
|
||||
),
|
||||
prefixSumDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "materialized_prefixes_sum"),
|
||||
"Sum of materialized_prefix_count over revisions (development aggregate).",
|
||||
"Deprecated: sum of materialized_prefix_count over all stored revisions; use stored_revision_materialized_prefixes_sum.",
|
||||
nil, nil,
|
||||
),
|
||||
peersDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "bgp_peers_configured_total"),
|
||||
"BGP peers configured in the control-plane store.",
|
||||
"Deprecated: BGP peers configured in the control-plane store; use inventory_bgp_peers.",
|
||||
nil, nil,
|
||||
),
|
||||
peerStateDesc: prometheus.NewDesc(
|
||||
prometheus.BuildFQName(namespace, "", "bgp_peer_sessions"),
|
||||
"Configured BGP peers in the store by session_state (intent / last known, not live BIRD).",
|
||||
"Deprecated: configured BGP peers by persisted session_state, not live BIRD; use stored_bgp_peer_session_state.",
|
||||
[]string{"state"}, nil,
|
||||
),
|
||||
storedMaxDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_max"),
|
||||
"Maximum materialized prefix count across all stored revisions; historical, not current routes.", nil, nil),
|
||||
storedSumDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_revision_materialized_prefixes_sum"),
|
||||
"Sum of materialized prefix counts across all stored revisions; historical, not current routes.", nil, nil),
|
||||
inventoryDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "inventory_bgp_peers"),
|
||||
"Configured BGP peer inventory in the control-plane store, regardless of live state.", nil, nil),
|
||||
storedStateDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "stored_bgp_peer_session_state"),
|
||||
"BGP peer inventory grouped by persisted session_state; not a live observation.", []string{"state"}, nil),
|
||||
collectionSuccessDesc: prometheus.NewDesc(prometheus.BuildFQName(namespace, "", "store_metrics_collection_success"),
|
||||
"1 if the latest store metrics collection succeeded, 0 on database aggregation error; other store metrics are omitted on error.", nil, nil),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +270,11 @@ func (c *memoryStoreCollector) Describe(ch chan<- *prometheus.Desc) {
|
||||
ch <- c.prefixSumDesc
|
||||
ch <- c.peersDesc
|
||||
ch <- c.peerStateDesc
|
||||
ch <- c.storedMaxDesc
|
||||
ch <- c.storedSumDesc
|
||||
ch <- c.inventoryDesc
|
||||
ch <- c.storedStateDesc
|
||||
ch <- c.collectionSuccessDesc
|
||||
}
|
||||
|
||||
func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
@@ -248,19 +282,44 @@ func (c *memoryStoreCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
if v == nil {
|
||||
return
|
||||
}
|
||||
b, ok := v.(store.Backend)
|
||||
if !ok || b == nil {
|
||||
holder, ok := v.(storeBackendHolder)
|
||||
if !ok || holder.backend == nil {
|
||||
return
|
||||
}
|
||||
maxN, sumN := b.MaterializedPrefixStats()
|
||||
b := holder.backend
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
maxN, sumN, peers, states, err := b.ObserveStoreMetrics(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 0)
|
||||
return
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.collectionSuccessDesc, prometheus.GaugeValue, 1)
|
||||
ch <- prometheus.MustNewConstMetric(c.prefixMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||
ch <- prometheus.MustNewConstMetric(c.prefixSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(b.PeerCount()))
|
||||
for state, n := range b.PeerSessionCountsByState() {
|
||||
ch <- prometheus.MustNewConstMetric(c.storedMaxDesc, prometheus.GaugeValue, float64(maxN))
|
||||
ch <- prometheus.MustNewConstMetric(c.storedSumDesc, prometheus.GaugeValue, float64(sumN))
|
||||
ch <- prometheus.MustNewConstMetric(c.peersDesc, prometheus.GaugeValue, float64(peers))
|
||||
ch <- prometheus.MustNewConstMetric(c.inventoryDesc, prometheus.GaugeValue, float64(peers))
|
||||
canonicalStates := make(map[string]int)
|
||||
for state, n := range states {
|
||||
if state == "" {
|
||||
state = "unknown"
|
||||
}
|
||||
ch <- prometheus.MustNewConstMetric(c.peerStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||
canonicalStates[canonicalPeerState(state)] += n
|
||||
}
|
||||
for state, n := range canonicalStates {
|
||||
ch <- prometheus.MustNewConstMetric(c.storedStateDesc, prometheus.GaugeValue, float64(n), state)
|
||||
}
|
||||
}
|
||||
|
||||
func canonicalPeerState(state string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(state)) {
|
||||
case "idle", "connect", "active", "opensent", "openconfirm", "established", "unknown":
|
||||
return strings.ToLower(strings.TrimSpace(state))
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -269,7 +328,7 @@ func RegisterStoreBackend(b store.Backend) {
|
||||
if b == nil {
|
||||
return
|
||||
}
|
||||
metricsStore.Store(b)
|
||||
metricsStore.Store(storeBackendHolder{backend: b})
|
||||
registerCollectorOnce.Do(func() {
|
||||
prometheus.DefaultRegisterer.MustRegister(newMemoryStoreCollector())
|
||||
})
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type failingMetricsBackend struct{ store.Backend }
|
||||
|
||||
func (failingMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||
return 0, 0, 0, nil, errors.New("database unavailable")
|
||||
}
|
||||
|
||||
type storedMetricsBackend struct{ store.Backend }
|
||||
|
||||
func (storedMetricsBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||
return 12, 24, 1, map[string]int{"established": 1}, nil
|
||||
}
|
||||
|
||||
type legacyStateBackend struct{ store.Backend }
|
||||
|
||||
func (legacyStateBackend) ObserveStoreMetrics(context.Context) (int, int, int, map[string]int, error) {
|
||||
return 0, 0, 2, map[string]int{" Established ": 1, "established": 1}, nil
|
||||
}
|
||||
|
||||
func TestStoreMetricsCollectionErrorOmitsAggregates(t *testing.T) {
|
||||
metricsStore.Store(storeBackendHolder{backend: failingMetricsBackend{}})
|
||||
registry := prometheus.NewRegistry()
|
||||
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(families) != 1 || families[0].GetName() != "evobgp_store_metrics_collection_success" {
|
||||
t.Fatalf("expected only collection status on error, got %v", families)
|
||||
}
|
||||
if got := families[0].Metric[0].Gauge.GetValue(); got != 0 {
|
||||
t.Fatalf("collection status = %v, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStoredRevisionMetricsDoNotClaimCurrentRoutes(t *testing.T) {
|
||||
metricsStore.Store(storeBackendHolder{backend: storedMetricsBackend{}})
|
||||
registry := prometheus.NewRegistry()
|
||||
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values := make(map[string]float64)
|
||||
for _, family := range families {
|
||||
if len(family.Metric) > 0 && family.Metric[0].Gauge != nil {
|
||||
values[family.GetName()] = family.Metric[0].Gauge.GetValue()
|
||||
}
|
||||
}
|
||||
if values["evobgp_stored_revision_materialized_prefixes_sum"] != 24 || values["evobgp_materialized_prefixes_sum"] != 24 {
|
||||
t.Fatalf("stored revision and legacy sums must match: %v", values)
|
||||
}
|
||||
if values["evobgp_store_metrics_collection_success"] != 1 {
|
||||
t.Fatalf("successful collection not reported: %v", values)
|
||||
}
|
||||
if _, ok := values["evobgp_current_routes"]; ok {
|
||||
t.Fatal("historical revision sum must not be published as current routes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCanonicalPeerStateBoundsNewLabelValues(t *testing.T) {
|
||||
if got := canonicalPeerState("Established"); got != "established" {
|
||||
t.Fatalf("state = %q", got)
|
||||
}
|
||||
if got := canonicalPeerState("unexpected peer text"); got != "other" {
|
||||
t.Fatalf("state = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLegacyPeerStateLabelPreservedWhileNewLabelIsCanonical(t *testing.T) {
|
||||
metricsStore.Store(storeBackendHolder{backend: legacyStateBackend{}})
|
||||
registry := prometheus.NewRegistry()
|
||||
if err := registry.Register(newMemoryStoreCollector()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
families, err := registry.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
legacyRaw, newCanonical := false, false
|
||||
for _, family := range families {
|
||||
for _, metric := range family.Metric {
|
||||
if len(metric.Label) != 1 {
|
||||
continue
|
||||
}
|
||||
switch family.GetName() {
|
||||
case "evobgp_bgp_peer_sessions":
|
||||
if metric.Label[0].GetValue() == " Established " && metric.Gauge.GetValue() == 1 {
|
||||
legacyRaw = true
|
||||
}
|
||||
case "evobgp_stored_bgp_peer_session_state":
|
||||
if metric.Label[0].GetValue() == "established" && metric.Gauge.GetValue() == 2 {
|
||||
newCanonical = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !legacyRaw || !newCanonical {
|
||||
t.Fatalf("legacy raw state preserved=%v, new canonical state aggregated=%v", legacyRaw, newCanonical)
|
||||
}
|
||||
}
|
||||
@@ -86,20 +86,26 @@ func aggregateTenantPrefixRowsAll(ctx context.Context, st store.Backend, hc *htt
|
||||
}
|
||||
|
||||
func rowsForModule(ctx context.Context, st store.Backend, hc *http.Client, tenantID string, mod *store.Module) ([]store.PrefixRow, error) {
|
||||
if rows, ok, err := moduleRowsFromSnapshot(st, tenantID, mod); err != nil {
|
||||
return nil, err
|
||||
} else if ok {
|
||||
return rows, nil
|
||||
}
|
||||
var prior []store.PrefixRow
|
||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
prior = snap.Prefixes
|
||||
}
|
||||
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, prior)
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, mod.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := persistModuleSnapshot(st, tenantID, mod, rows); err != nil {
|
||||
if ok && snap != nil {
|
||||
if rows, matched, err := snapshotRowsIfCurrent(ctx, st, tenantID, mod, snap); err != nil {
|
||||
return nil, err
|
||||
} else if matched {
|
||||
return rows, nil
|
||||
}
|
||||
}
|
||||
var priorRows []store.PrefixRow
|
||||
if ok && snap != nil {
|
||||
priorRows = snap.Prefixes
|
||||
}
|
||||
rows, err := collectModulePrefixRows(ctx, st, hc, tenantID, mod, modulePrior{provided: true, rows: priorRows})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := persistModuleSnapshot(ctx, st, tenantID, mod, rows); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rows, nil
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
)
|
||||
|
||||
const cdnBodyLimit = 8 << 20
|
||||
|
||||
// ErrCDNBodyTooLarge reports a CDN response larger than the 8 MiB decoded-body limit.
|
||||
var ErrCDNBodyTooLarge = errors.New("pipeline: cdn response body exceeds 8 MiB")
|
||||
|
||||
type limitedCDNBody struct {
|
||||
io.ReadCloser
|
||||
remaining int64
|
||||
}
|
||||
|
||||
func (b *limitedCDNBody) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if b.remaining == 0 {
|
||||
var one [1]byte
|
||||
n, err := b.ReadCloser.Read(one[:])
|
||||
if n > 0 {
|
||||
return 0, ErrCDNBodyTooLarge
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
if int64(len(p)) > b.remaining {
|
||||
p = p[:int(b.remaining)]
|
||||
}
|
||||
n, err := b.ReadCloser.Read(p)
|
||||
b.remaining -= int64(n)
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadCDNBody reads a CDN response and rejects decoded bodies above 8 MiB.
|
||||
// The extra byte distinguishes an exact-limit response from a truncated one.
|
||||
func ReadCDNBody(body io.Reader) ([]byte, error) {
|
||||
data, err := io.ReadAll(io.LimitReader(body, cdnBodyLimit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(data) > cdnBodyLimit {
|
||||
return nil, ErrCDNBodyTooLarge
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
func TestReadCDNBodyLimit(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extra int64
|
||||
large bool
|
||||
}{
|
||||
{"exact limit", 0, false},
|
||||
{"one byte too large", 1, true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
reader := io.MultiReader(io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit)), cdnBodyLimit), strings.NewReader(strings.Repeat("y", int(tc.extra))))
|
||||
body, err := ReadCDNBody(reader)
|
||||
if tc.large {
|
||||
if !errors.Is(err, ErrCDNBodyTooLarge) || body != nil {
|
||||
t.Fatalf("body=%d err=%v", len(body), err)
|
||||
}
|
||||
} else if err != nil || len(body) != cdnBodyLimit {
|
||||
t.Fatalf("body=%d err=%v", len(body), err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchCDNSourceRowsRejectsOversizedBody(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_, _ = io.Copy(w, strings.NewReader(strings.Repeat("x", cdnBodyLimit+1)))
|
||||
}))
|
||||
defer srv.Close()
|
||||
_, err := fetchCDNSourceRows(context.Background(), nil, srv.Client(), "", "", nil,
|
||||
&store.CDNSource{ID: "source", URL: srv.URL, SourceKind: "plain"}, modulePrior{}, time.Now())
|
||||
if !errors.Is(err, ErrCDNBodyTooLarge) {
|
||||
t.Fatalf("oversized CDN source must fail: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCDNTransportBoundsDecodedGzipBody(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Encoding", "gzip")
|
||||
zw := gzip.NewWriter(w)
|
||||
_, _ = io.Copy(zw, io.LimitReader(strings.NewReader(strings.Repeat("x", cdnBodyLimit+1)), cdnBodyLimit+1))
|
||||
_ = zw.Close()
|
||||
}))
|
||||
defer srv.Close()
|
||||
|
||||
client, err := secureCDNClient(srv.Client())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := client.Get(srv.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if !resp.Uncompressed {
|
||||
t.Fatal("expected net/http gzip decompression")
|
||||
}
|
||||
if _, err := ReadCDNBody(resp.Body); !errors.Is(err, ErrCDNBodyTooLarge) {
|
||||
t.Fatalf("oversized decoded body: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,7 @@ func TestCollectModulePrefixRows_CDNSendsIfNoneMatch(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -89,11 +89,11 @@ func TestCollectModulePrefixRows_CDN304UsesSnapshot(t *testing.T) {
|
||||
Prefix: "203.0.113.0/24",
|
||||
Source: cdnSourceKey(src.ID),
|
||||
}}
|
||||
if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil {
|
||||
if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, prior)
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{provided: true, rows: prior})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -135,7 +135,7 @@ func TestRefreshModuleIngest_CDN304UsesStoredSnapshot(t *testing.T) {
|
||||
Prefix: "203.0.113.0/24",
|
||||
Source: cdnSourceKey(src.ID),
|
||||
}}
|
||||
if err := mergeCDNSourceIntoModuleSnapshot(m, tenant, mod, src.ID, prior); err != nil {
|
||||
if err := mergeCDNSourceIntoModuleSnapshot(context.Background(), m, tenant, mod, src.ID, prior); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -185,7 +185,7 @@ func TestCollectModulePrefixRows_CDN304RetriesWithoutETag(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, nil)
|
||||
collected, err := collectModulePrefixRows(context.Background(), m, srv.Client(), tenant, mod, modulePrior{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -11,22 +12,86 @@ import (
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
var errCDNSnapshotRead = errors.New("pipeline: CDN snapshot read failed")
|
||||
|
||||
// modulePrior is the module snapshot the caller already loaded.
|
||||
// provided distinguishes that read from a caller that did not load one.
|
||||
type modulePrior struct {
|
||||
provided bool
|
||||
rows []store.PrefixRow
|
||||
}
|
||||
|
||||
func cdnSourceKey(sourceID string) string {
|
||||
return "cdn:" + strings.TrimSpace(sourceID)
|
||||
}
|
||||
|
||||
func cachedCDNPrefixRows(st store.Backend, tenantID, moduleID string, priorSnapshot []store.PrefixRow, sourceKey string) []store.PrefixRow {
|
||||
if cached := prefixRowsForSource(priorSnapshot, sourceKey); len(cached) > 0 {
|
||||
return cached
|
||||
// narrowSnapshotReadCheaper reports whether per-source reads move less data than
|
||||
// one full module snapshot. One skipped source is filtered in SQL. Several skipped
|
||||
// sources share one snapshot read once they are a large share of the module.
|
||||
func narrowSnapshotReadCheaper(skipped, total int) bool {
|
||||
if skipped <= 0 || total <= 0 {
|
||||
return false
|
||||
}
|
||||
if st != nil {
|
||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, moduleID); ok && snap != nil {
|
||||
if cached := prefixRowsForSource(snap.Prefixes, sourceKey); len(cached) > 0 {
|
||||
return cached
|
||||
}
|
||||
if skipped == 1 {
|
||||
return true
|
||||
}
|
||||
return skipped*2 < total
|
||||
}
|
||||
|
||||
func listModuleSourceRows(ctx context.Context, st store.Backend, tenantID, moduleID, source string) ([]store.PrefixRow, error) {
|
||||
if st == nil {
|
||||
return nil, nil
|
||||
}
|
||||
rows, err := st.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func cachedCDNPrefixRows(ctx context.Context, st store.Backend, tenantID, moduleID string, prior modulePrior, sourceKey string) ([]store.PrefixRow, error) {
|
||||
if prior.provided {
|
||||
return prefixRowsForSource(prior.rows, sourceKey), nil
|
||||
}
|
||||
return listModuleSourceRows(ctx, st, tenantID, moduleID, sourceKey)
|
||||
}
|
||||
|
||||
// loadSkippedCDNRows resolves interval-skipped sources without a second full snapshot
|
||||
// read when the caller already passed prior rows.
|
||||
func loadSkippedCDNRows(ctx context.Context, st store.Backend, tenantID, moduleID string, sources []*store.CDNSource, prior modulePrior, now time.Time) (modulePrior, map[string][]store.PrefixRow, error) {
|
||||
if prior.provided || st == nil {
|
||||
return prior, nil, nil
|
||||
}
|
||||
var skipped []*store.CDNSource
|
||||
for _, src := range sources {
|
||||
if src != nil && shouldSkipCDNSourceFetch(src, now) {
|
||||
skipped = append(skipped, src)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
if len(skipped) == 0 {
|
||||
return prior, nil, nil
|
||||
}
|
||||
if narrowSnapshotReadCheaper(len(skipped), len(sources)) {
|
||||
out := make(map[string][]store.PrefixRow, len(skipped))
|
||||
for _, src := range skipped {
|
||||
key := cdnSourceKey(src.ID)
|
||||
rows, err := listModuleSourceRows(ctx, st, tenantID, moduleID, key)
|
||||
if err != nil {
|
||||
return prior, nil, err
|
||||
}
|
||||
out[key] = rows
|
||||
}
|
||||
return prior, out, nil
|
||||
}
|
||||
snap, ok, err := st.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
|
||||
if err != nil {
|
||||
return prior, nil, fmt.Errorf("%w: %w", errCDNSnapshotRead, err)
|
||||
}
|
||||
rows := []store.PrefixRow(nil)
|
||||
if ok && snap != nil {
|
||||
rows = snap.Prefixes
|
||||
}
|
||||
return modulePrior{provided: true, rows: rows}, nil, nil
|
||||
}
|
||||
|
||||
func mergeSnapshotDropSource(rows []store.PrefixRow, sourceKey string) []store.PrefixRow {
|
||||
@@ -69,23 +134,16 @@ func mergeSnapshotKeepSkippedCDN(rows []store.PrefixRow, fetchedSourceIDs, allSo
|
||||
return out
|
||||
}
|
||||
|
||||
// mergeAllCDNSourcesIntoModuleSnapshot replaces fetched CDN source rows in one write.
|
||||
// skipped sources (errors with EVOBGP_CDN_PARTIAL_OK) keep their prior rows.
|
||||
func mergeAllCDNSourcesIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, priorSnapshot []store.PrefixRow, cdnRows []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) error {
|
||||
if st == nil || mod == nil {
|
||||
return nil
|
||||
// mergeCDNSnapshotRows builds the full module snapshot after a CDN collect.
|
||||
// Fetched sources replace their rows. Skipped sources (partial OK) keep prior rows.
|
||||
// Sources absent from allSourceIDs are dropped. Non-CDN rows in prior are kept.
|
||||
// The caller persists the result once; this function does not write.
|
||||
func mergeCDNSnapshotRows(prior, fetched []store.PrefixRow, fetchedSourceIDs, allSourceIDs []string) []store.PrefixRow {
|
||||
kept := mergeSnapshotKeepSkippedCDN(prior, fetchedSourceIDs, allSourceIDs)
|
||||
if len(fetched) == 0 {
|
||||
return kept
|
||||
}
|
||||
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||
defer unlock()
|
||||
var base []store.PrefixRow
|
||||
if len(priorSnapshot) > 0 {
|
||||
base = priorSnapshot
|
||||
} else if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
base = snap.Prefixes
|
||||
}
|
||||
kept := mergeSnapshotKeepSkippedCDN(base, fetchedSourceIDs, allSourceIDs)
|
||||
merged := append(kept, cdnRows...)
|
||||
return persistModuleSnapshot(st, tenantID, mod, merged)
|
||||
return append(kept, fetched...)
|
||||
}
|
||||
|
||||
func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []string) []store.PrefixRow {
|
||||
@@ -106,19 +164,23 @@ func cdnRowsFromParsed(mod *store.Module, src *store.CDNSource, pfxStrings []str
|
||||
}
|
||||
|
||||
// mergeCDNSourceIntoModuleSnapshot replaces rows for one CDN source in the module snapshot.
|
||||
func mergeCDNSourceIntoModuleSnapshot(st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
|
||||
func mergeCDNSourceIntoModuleSnapshot(ctx context.Context, st store.Backend, tenantID string, mod *store.Module, sourceID string, newRows []store.PrefixRow) error {
|
||||
if st == nil || mod == nil {
|
||||
return nil
|
||||
}
|
||||
unlock := st.LockModuleSnapshot(tenantID, mod.ID)
|
||||
defer unlock()
|
||||
sourceKey := cdnSourceKey(sourceID)
|
||||
var base []store.PrefixRow
|
||||
if snap, ok, _ := st.GetModulePrefixSnapshot(tenantID, mod.ID); ok && snap != nil {
|
||||
base = mergeSnapshotDropSource(snap.Prefixes, sourceKey)
|
||||
hash, err := moduleIngestInputHash(st, tenantID, mod)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
merged := append(base, newRows...)
|
||||
return persistModuleSnapshot(st, tenantID, mod, merged)
|
||||
sourceKey := cdnSourceKey(sourceID)
|
||||
return st.UpdateModulePrefixSnapshot(ctx, tenantID, mod.ID, func(current *store.ModulePrefixSnapshot) (*store.ModulePrefixSnapshot, error) {
|
||||
var base []store.PrefixRow
|
||||
if current != nil {
|
||||
base = mergeSnapshotDropSource(current.Prefixes, sourceKey)
|
||||
}
|
||||
merged := append(base, newRows...)
|
||||
return &store.ModulePrefixSnapshot{InputHash: hash, Prefixes: merged}, nil
|
||||
})
|
||||
}
|
||||
|
||||
func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
|
||||
@@ -134,7 +196,7 @@ func parseCDNBody(body string, src *store.CDNSource) ([]string, error) {
|
||||
}
|
||||
|
||||
// fetchCDNSourceRows loads CDN prefixes without persisting the module snapshot (caller merges once).
|
||||
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, priorSnapshot []store.PrefixRow, now time.Time) ([]store.PrefixRow, error) {
|
||||
func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client, tenantID, moduleID string, mod *store.Module, src *store.CDNSource, prior modulePrior, now time.Time) ([]store.PrefixRow, error) {
|
||||
u := strings.TrimSpace(src.URL)
|
||||
if u == "" {
|
||||
return nil, nil
|
||||
@@ -159,7 +221,12 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
|
||||
}
|
||||
|
||||
if resp.StatusCode == http.StatusNotModified {
|
||||
if cached := cachedCDNPrefixRows(st, tenantID, moduleID, priorSnapshot, sourceKey); len(cached) > 0 {
|
||||
cached, cacheErr := cachedCDNPrefixRows(ctx, st, tenantID, moduleID, prior, sourceKey)
|
||||
if cacheErr != nil {
|
||||
_ = resp.Body.Close()
|
||||
return nil, cacheErr
|
||||
}
|
||||
if len(cached) > 0 {
|
||||
_ = resp.Body.Close()
|
||||
return cached, nil
|
||||
}
|
||||
@@ -183,7 +250,7 @@ func fetchCDNSourceRows(ctx context.Context, st store.Backend, hc *http.Client,
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
return nil, fmt.Errorf("cdn url %s: %s", u, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
body, err := ReadCDNBody(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type failingSnapshotUpdate struct{ store.Backend }
|
||||
|
||||
func (f failingSnapshotUpdate) UpdateModulePrefixSnapshot(context.Context, string, string, store.ModuleSnapshotUpdate) error {
|
||||
return context.DeadlineExceeded
|
||||
}
|
||||
|
||||
func TestMergeCDNSourceRefusesWriteWhenAtomicUpdateFails(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-lock-failure", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = mergeCDNSourceIntoModuleSnapshot(context.Background(), failingSnapshotUpdate{m}, tenant, mod, "source-a", []store.PrefixRow{{Prefix: "192.0.2.0/24", Source: "cdn:source-a"}})
|
||||
if !errors.Is(err, context.DeadlineExceeded) {
|
||||
t.Fatalf("expected update failure, got %v", err)
|
||||
}
|
||||
if _, ok, err := m.GetModulePrefixSnapshot(context.Background(), tenant, mod.ID); err != nil || ok {
|
||||
t.Fatalf("snapshot written without lock: ok=%v err=%v", ok, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"evobgp/internal/store"
|
||||
)
|
||||
|
||||
type snapCalls struct {
|
||||
store.Backend
|
||||
full atomic.Int32
|
||||
narrow atomic.Int32
|
||||
}
|
||||
|
||||
func (s *snapCalls) GetModulePrefixSnapshot(ctx context.Context, tenantID, moduleID string) (*store.ModulePrefixSnapshot, bool, error) {
|
||||
s.full.Add(1)
|
||||
return s.Backend.GetModulePrefixSnapshot(ctx, tenantID, moduleID)
|
||||
}
|
||||
|
||||
func (s *snapCalls) ListModuleSnapshotSourceRows(ctx context.Context, tenantID, moduleID, source string) ([]store.PrefixRow, error) {
|
||||
s.narrow.Add(1)
|
||||
return s.Backend.ListModuleSnapshotSourceRows(ctx, tenantID, moduleID, source)
|
||||
}
|
||||
|
||||
func TestNarrowSnapshotReadCheaper(t *testing.T) {
|
||||
if !narrowSnapshotReadCheaper(1, 8) {
|
||||
t.Fatal("one skipped source should use a narrow read")
|
||||
}
|
||||
if !narrowSnapshotReadCheaper(3, 8) {
|
||||
t.Fatal("a minority of skipped sources should use narrow reads")
|
||||
}
|
||||
if narrowSnapshotReadCheaper(6, 8) {
|
||||
t.Fatal("a majority of skipped sources should share one snapshot read")
|
||||
}
|
||||
if narrowSnapshotReadCheaper(0, 4) {
|
||||
t.Fatal("nothing to read")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefreshModuleIngestDoesNotRereadSnapshot(t *testing.T) {
|
||||
t.Setenv("EVOBGP_CDN_ALLOW_PRIVATE", "1")
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-once", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
interval := 3600
|
||||
fresh := time.Now().UTC().Add(-30 * time.Second)
|
||||
src, err := m.CreateCDNSource(tenant, mod.ID, &store.CDNSource{
|
||||
SourceKind: "txt",
|
||||
URL: "https://cdn.example/prefixes.txt",
|
||||
RefreshIntervalSec: &interval,
|
||||
LastRefreshedAt: &fresh,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rows := []store.PrefixRow{{Prefix: "203.0.113.0/24", Source: cdnSourceKey(src.ID)}}
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", rows); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counted := &snapCalls{Backend: m}
|
||||
if err := RefreshModuleIngest(context.Background(), counted, nil, tenant, mod.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if counted.full.Load() != 1 || counted.narrow.Load() != 0 {
|
||||
t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestSkippedCDNSourceUsesNarrowReadWithoutPrior(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-narrow", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
interval := 3600
|
||||
fresh := time.Now().UTC().Add(-30 * time.Second)
|
||||
src := &store.CDNSource{ID: "only", SourceKind: "txt", URL: "https://cdn.example/one", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh}
|
||||
want := []store.PrefixRow{{Prefix: "203.0.113.8/32", Source: "cdn:only"}}
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", append([]store.PrefixRow{{Prefix: "198.51.100.0/24", Source: "cdn:other"}}, want...)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counted := &snapCalls{Backend: m}
|
||||
got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, []*store.CDNSource{src}, modulePrior{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if counted.full.Load() != 0 || counted.narrow.Load() != 1 {
|
||||
t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
|
||||
}
|
||||
if len(got) != 1 || got[0].Prefix != "203.0.113.8/32" || got[0].Source != "cdn:only" {
|
||||
t.Fatalf("rows=%+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManySkippedCDNSourcesShareOneSnapshotRead(t *testing.T) {
|
||||
m := store.NewMemory()
|
||||
m.SeedDemo()
|
||||
tenant, _, _, _, _ := m.DemoIDs()
|
||||
mod, err := m.CreateModule(tenant, &store.Module{Type: "CDN_CIDRS", Name: "cdn-shared", Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
interval := 3600
|
||||
fresh := time.Now().UTC().Add(-30 * time.Second)
|
||||
sources := []*store.CDNSource{
|
||||
{ID: "a", URL: "https://cdn.example/a", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh},
|
||||
{ID: "b", URL: "https://cdn.example/b", RefreshIntervalSec: &interval, LastRefreshedAt: &fresh},
|
||||
}
|
||||
stored := []store.PrefixRow{
|
||||
{Prefix: "203.0.113.1/32", Source: "cdn:a"},
|
||||
{Prefix: "203.0.113.2/32", Source: "cdn:b"},
|
||||
{Prefix: "198.51.100.0/24", Source: "cdn:gone"},
|
||||
}
|
||||
if err := m.SetModulePrefixSnapshot(context.Background(), tenant, mod.ID, "hash", stored); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
counted := &snapCalls{Backend: m}
|
||||
got, err := collectCDNPrefixRows(context.Background(), counted, nil, tenant, mod, sources, modulePrior{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if counted.full.Load() != 1 || counted.narrow.Load() != 0 {
|
||||
t.Fatalf("snapshot reads full=%d narrow=%d", counted.full.Load(), counted.narrow.Load())
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, row := range got {
|
||||
seen[row.Source] = true
|
||||
}
|
||||
if !seen["cdn:a"] || !seen["cdn:b"] || seen["cdn:gone"] {
|
||||
t.Fatalf("rows=%+v", got)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 197 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user