feat(httpapi): harden control plane and speed up prefix refresh

Очередь job получила lease и fencing, телеметрия и прогресс deploy сохраняются, ревизия пишет manifest входов. Обновление префиксов не переписывает неизменённый снимок, большие наборы идут через COPY, а устаревшая ревизия не попадает в deploy. Бандл ноды кэшируется, prune и постановка job больше не держат горячий путь.

Co-authored-by: Cursor <[email protected]>

internal/jobs/deployment_reconcile_test.go

@@ -0,0 +1,140 @@
+package jobs
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"evobgp/internal/store"
+)
+
+func TestDeploymentReconcileDedupAndDesiredOrdering(t *testing.T) {
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, revision, speaker := m.DemoIDs()
+	ctx := context.Background()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	initial, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	claim := func(owner string) *Job {
+		t.Helper()
+		rows, e := m.ClaimJobs(ctx, owner, 1, time.Minute)
+		if e != nil || len(rows) != 1 {
+			t.Fatalf("claim: %+v %v", rows, e)
+		}
+		return jobFromRecord(rows[0])
+	}
+	first := claim("initial")
+	if first.ID != initial.ID {
+		t.Fatal("wrong initial job")
+	}
+	(&Worker{Store: m}).Process(first)
+	if err = m.FinishJob(ctx, recordFromJob(first)); err != nil {
+		t.Fatal(err)
+	}
+	// Enqueue an operator request before the synthetic recovery request. The latter
+	// must retain the earlier desired order despite its own larger queue sequence.
+	operator, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	for range 2 {
+		if err = ReconcileDeployments(ctx, r, m); err != nil {
+			t.Fatal(err)
+		}
+	}
+	all, _, _, err := m.ListJobs(ctx, tenant, "", "", "", 100)
+	if err != nil || len(all) != 3 {
+		t.Fatalf("reconcile dedupe: %d %v", len(all), err)
+	}
+	newer := claim("operator")
+	if newer.ID != operator.ID {
+		t.Fatal("wrong operator job")
+	}
+	recovery := claim("recovery")
+	f := deploymentFenceForJob(recovery)
+	d, err := m.BeginDeploymentSpeaker(ctx, f, speaker, revision)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if d.RequestSequence != first.RequestSequence {
+		t.Fatal("recovery lost original desired order")
+	}
+	if err = m.RecordDeploymentSpeaker(ctx, f, d, "unconfirmed", "unconfirmed", ""); err != nil {
+		t.Fatal(err)
+	}
+	newD, err := m.BeginDeploymentSpeaker(ctx, deploymentFenceForJob(newer), speaker, revision)
+	if err != nil {
+		t.Fatalf("newer operator should supersede recovery despite smaller queue sequence: %v", err)
+	}
+	if newD.RequestSequence != newer.RequestSequence {
+		t.Fatal("operator desired sequence changed")
+	}
+	if _, err = m.BeginDeploymentSpeaker(ctx, f, speaker, revision); err == nil {
+		t.Fatal("stale recovery generation reused")
+	}
+}
+
+func TestDeploymentHealthDetectsDriftAndKeepsLegacyQuality(t *testing.T) {
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "0")
+	m := store.NewMemory()
+	m.SeedDemo()
+	tenant, _, _, revision, speaker := m.DemoIDs()
+	ctx := context.Background()
+	r := NewRegistry(nil)
+	r.UseDurableQueue(m)
+	_, _, err := r.EnqueueContext(ctx, tenant, KindDeployApply, nil, nil, map[string]any{"revision_id": revision, "speaker_id": speaker})
+	if err != nil {
+		t.Fatal(err)
+	}
+	rows, err := m.ClaimJobs(ctx, "owner", 1, time.Minute)
+	if err != nil || len(rows) != 1 {
+		t.Fatal(err)
+	}
+	j := jobFromRecord(rows[0])
+	(&Worker{Store: m}).Process(j)
+	if err = m.FinishJob(ctx, recordFromJob(j)); err != nil {
+		t.Fatal(err)
+	}
+	meta := store.SpeakerMetaJSON(store.SpeakerMeta{AgentDomain: "agent.example.test", AgentSecret: "secret"})
+	if _, err = m.UpdateSpeaker(tenant, speaker, &store.SpeakerPatch{MetaJSON: &meta}); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("EVOBGP_NODE_DISPATCH_ENABLED", "1")
+	response := `{"ok":true,"last_applied_revision_id":"` + revision + `"}`
+	client := &http.Client{Transport: deployRoundTrip(func(req *http.Request) (*http.Response, error) {
+		if req.Header.Get("Authorization") != "Bearer secret" || req.Method != "GET" {
+			t.Error("health must be authenticated GET")
+		}
+		return &http.Response{StatusCode: 200, Body: io.NopCloser(strings.NewReader(response)), Header: make(http.Header)}, nil
+	})}
+	if err = ObserveDeployments(ctx, m, client); err != nil {
+		t.Fatal(err)
+	}
+	progress, err := m.ListDeploymentSpeakers(ctx, tenant, j.ID)
+	if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthReceivedAt == nil || progress[0].HealthSourceAt != "" {
+		t.Fatalf("legacy health confirmation: %+v %v", progress, err)
+	}
+	response = `{"ok":true,"last_applied_revision_id":"older-late-operation"}`
+	if err = ObserveDeployments(ctx, m, client); err != nil {
+		t.Fatal(err)
+	}
+	progress, _ = m.ListDeploymentSpeakers(ctx, tenant, j.ID)
+	if progress[0].State != "unknown" || progress[0].Confirmation != "unconfirmed" {
+		t.Fatalf("late drift hidden: %+v", progress)
+	}
+	if err = ReconcileDeployments(ctx, r, m); err != nil {
+		t.Fatal(err)
+	}
+	jobs, _, _, _ := m.ListJobs(ctx, tenant, "queued", "", "", 100)
+	if len(jobs) != 1 {
+		t.Fatalf("drift recovery jobs: %d", len(jobs))
+	}
+}