EvoBGP/internal/repository/deployment_integration_test.go

package repository

import (
	"context"
	"errors"
	"testing"
	"time"

	"evobgp/internal/store"
	"github.com/google/uuid"
)

func TestDeploymentLeaseGenerationAndRestartIntegration(t *testing.T) {
	p, tenant := queueIntegration(t)
	ctx := context.Background()
	speaker, revision, newer := uuid.NewString(), uuid.NewString(), uuid.NewString()
	for _, id := range []string{revision, newer} {
		if _, err := p.pool.Exec(ctx, `INSERT INTO config_revision(id,tenant_id,content_hash) VALUES($1,$2,'deployment-test')`, id, tenant); err != nil {
			t.Fatal(err)
		}
	}
	if _, err := p.pool.Exec(ctx, `INSERT INTO bgp_speaker(id,tenant_id,role) VALUES($1,$2,'replica')`, speaker, tenant); err != nil {
		t.Fatal(err)
	}
	enqueue := func() store.JobRecord {
		t.Helper()
		j, _, err := p.EnqueueJob(ctx, store.JobEnqueue{TenantID: tenant, Kind: "deploy_apply"})
		if err != nil {
			t.Fatal(err)
		}
		return j
	}
	claim := func(owner string) store.JobRecord {
		t.Helper()
		rows, err := p.ClaimJobs(ctx, owner, 1, time.Minute)
		if err != nil || len(rows) != 1 {
			t.Fatalf("claim: %+v %v", rows, err)
		}
		return rows[0]
	}
	fence := func(j store.JobRecord) store.JobFence {
		return store.JobFence{JobID: j.ID, TenantID: j.TenantID, OwnerID: j.OwnerID, Token: j.FencingToken}
	}
	first := enqueue()
	first = claim("deployment-one")
	d, err := p.BeginDeploymentSpeaker(ctx, fence(first), speaker, revision)
	if err != nil {
		t.Fatal(err)
	}
	second := enqueue()
	second = claim("deployment-two")
	if _, err = p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer); !errors.Is(err, store.ErrDeploymentBusy) {
		t.Fatalf("overlapping network ownership: %v", err)
	}
	if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "unknown", "unconfirmed", "response lost"); err != nil {
		t.Fatal(err)
	}
	if _, err = p.pool.Exec(ctx, `UPDATE job_audit SET lease_until=clock_timestamp()-interval '1 second' WHERE id=$1`, first.ID); err != nil {
		t.Fatal(err)
	}
	if err = p.RecordDeploymentSpeaker(ctx, fence(first), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
		t.Fatalf("expired ACK accepted: %v", err)
	}
	reclaimed := claim("deployment-reclaimed")
	if reclaimed.ID != first.ID || reclaimed.FencingToken <= first.FencingToken {
		t.Fatal("job not reclaimed with fresh token")
	}
	if _, err = p.BeginDeploymentSpeaker(ctx, fence(reclaimed), speaker, revision); !errors.Is(err, store.ErrDeploymentBusy) {
		t.Fatalf("reclaim overlaps uncertain request: %v", err)
	}
	if _, err = p.pool.Exec(ctx, `UPDATE bgp_speaker SET delivery_until=clock_timestamp()-interval '1 second' WHERE id=$1`, speaker); err != nil {
		t.Fatal(err)
	}
	// A fresh repository object sees the committed partial progress after restart.
	restarted := &Postgres{pool: p.pool}
	rows, err := restarted.ListDeploymentSpeakers(ctx, tenant, first.ID)
	if err != nil || len(rows) != 1 || rows[0].State != "unknown" {
		t.Fatalf("restart progress: %+v %v", rows, err)
	}
	d2, err := restarted.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
	if err != nil {
		t.Fatal(err)
	}
	if d2.Generation <= d.Generation {
		t.Fatal("speaker generation did not advance")
	}
	if err = p.RecordDeploymentSpeaker(ctx, fence(reclaimed), d, "applied", "agent_reported", ""); !errors.Is(err, store.ErrJobLeaseLost) {
		t.Fatalf("late old-generation ACK accepted: %v", err)
	}
	if err = p.RecordDeploymentSpeaker(ctx, fence(second), d2, "applied", "agent_reported", ""); err != nil {
		t.Fatal(err)
	}
	var published, applied string
	if err = p.pool.QueryRow(ctx, `SELECT published_revision_id::text,last_applied_revision_id::text FROM bgp_speaker WHERE id=$1`, speaker).Scan(&published, &applied); err != nil || published != newer || applied != newer {
		t.Fatalf("fenced pointers: %s %s %v", published, applied, err)
	}
	d2Replay, err := p.BeginDeploymentSpeaker(ctx, fence(second), speaker, newer)
	if err != nil || d2Replay.State != "applied" || d2Replay.Generation != d2.Generation {
		t.Fatalf("applied replay: %+v %v", d2Replay, err)
	}
	second.Status = "succeeded"
	if err = p.FinishJob(ctx, second); err != nil {
		t.Fatal(err)
	}
	oldPoll, err := p.ListDeploymentObservations(ctx, 10)
	if err != nil || len(oldPoll) != 1 {
		t.Fatalf("old poll %+v %v", oldPoll, err)
	}
	newPoll, err := p.ListDeploymentObservations(ctx, 10)
	if err != nil || len(newPoll) != 1 {
		t.Fatalf("new poll %+v %v", newPoll, err)
	}
	// A stale snapshot must not downgrade the current acknowledged state when
	// the latest poll times out or the old agent omits revision fields.
	newPoll[0].State = "unknown"
	newPoll[0].Confirmation = "unconfirmed"
	if err = p.ObserveDeployment(ctx, newPoll[0], store.DeploymentObservation{Error: "timeout"}); err != nil {
		t.Fatal(err)
	}
	if err = p.ObserveDeployment(ctx, oldPoll[0], store.DeploymentObservation{RevisionID: revision}); !errors.Is(err, store.ErrJobLeaseLost) {
		t.Fatalf("late health accepted: %v", err)
	}
	progress, err := p.ListDeploymentSpeakers(ctx, tenant, second.ID)
	if err != nil || len(progress) != 1 || progress[0].State != "applied" || progress[0].Confirmation != "agent_reported" || progress[0].HealthError != "timeout" {
		t.Fatalf("health lost ACK %+v %v", progress, err)
	}
	// Historical deployment records must survive ordinary revision retention.
	if _, err = p.pool.Exec(ctx, `DELETE FROM config_revision WHERE id=$1::uuid`, revision); err != nil {
		t.Fatalf("history blocks revision pruning: %v", err)
	}
	history, err := p.ListDeploymentSpeakers(ctx, tenant, first.ID)
	if err != nil || len(history) != 1 || history[0].RevisionID != revision {
		t.Fatalf("pruning removed history %+v %v", history, err)
	}
}